@freshworks/shiftleft-tools 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +351 -0
- package/bin/shiftleft.js +95 -0
- package/package.json +57 -0
- package/src/commands/doctor.js +208 -0
- package/src/commands/init-postman.js +298 -0
- package/src/commands/init-rules.js +78 -0
- package/src/commands/link.js +172 -0
- package/src/commands/protect.js +61 -0
- package/src/commands/run-tests.js +182 -0
- package/src/commands/setup-pipeline.js +209 -0
- package/src/commands/update.js +203 -0
- package/src/index.js +4 -0
- package/src/utils/copy-tree.js +98 -0
- package/src/utils/gitignore.js +26 -0
- package/src/utils/logger.js +9 -0
- package/src/utils/manifest.js +145 -0
- package/src/utils/stack.js +80 -0
- package/src/utils/template.js +135 -0
- package/templates/AGENTS.md +109 -0
- package/templates/CLAUDE.md +3 -0
- package/templates/jenkins/Jenkinsfile-java.groovy +432 -0
- package/templates/jenkins/Jenkinsfile-node.groovy +450 -0
- package/templates/postman/.husky/pre-commit +19 -0
- package/templates/postman/.prettierrc.json +5 -0
- package/templates/postman/README.md.ejs +147 -0
- package/templates/postman/collections/01-core.json.ejs +91 -0
- package/templates/postman/config/local.json.ejs +12 -0
- package/templates/postman/config/staging.json.ejs +26 -0
- package/templates/postman/environments/local.postman_environment.json.ejs +31 -0
- package/templates/postman/environments/staging.postman_environment.json.ejs +31 -0
- package/templates/postman/gitignore +16 -0
- package/templates/postman/npmrc +31 -0
- package/templates/postman/package.json.ejs +66 -0
- package/templates/postman/run-all-shim.sh +16 -0
- package/templates/postman/scripts/auth/generate-jwt.sh +113 -0
- package/templates/postman/scripts/auth/get-issuer-secret.sh +140 -0
- package/templates/postman/scripts/infra/start-mocks.sh +138 -0
- package/templates/postman/scripts/infra/stop-mocks.sh +43 -0
- package/templates/postman/scripts/lib/api_coverage.py +1122 -0
- package/templates/postman/scripts/lib/cleanup-reports.sh +101 -0
- package/templates/postman/scripts/lib/cleanup-stryker.sh +44 -0
- package/templates/postman/scripts/lib/report_combined.py +527 -0
- package/templates/postman/scripts/lib/report_consolidated.py +363 -0
- package/templates/postman/scripts/lib/report_generator.py +121 -0
- package/templates/postman/scripts/lib/report_migration.py +156 -0
- package/templates/postman/scripts/lib/report_mutation.py +110 -0
- package/templates/postman/scripts/lib/report_unit.py +353 -0
- package/templates/postman/scripts/lib/report_utils.py +973 -0
- package/templates/postman/scripts/report-generators/generate-consolidated-report.sh +445 -0
- package/templates/postman/scripts/report-generators/java-api-coverage-matrix.sh +257 -0
- package/templates/postman/scripts/report-generators/mutation-report.sh +672 -0
- package/templates/postman/scripts/report-generators/node-api-coverage-matrix.sh +167 -0
- package/templates/postman/scripts/report-generators/stage-report-artifacts.sh +27 -0
- package/templates/postman/scripts/run-all.sh +452 -0
- package/templates/postman/scripts/runners/run-mutation-tests.sh +113 -0
- package/templates/postman/scripts/runners/run-tests-local.sh +936 -0
- package/templates/postman/scripts/runners/run-tests-staging.sh +741 -0
- package/templates/postman-node/README.md.ejs +26 -0
- package/templates/postman-node/collections/crud/01-bootstrap.json.ejs +34 -0
- package/templates/postman-node/config/local.json.ejs +46 -0
- package/templates/postman-node/config/staging.json.ejs +31 -0
- package/templates/postman-node/local.test.env.ejs +3 -0
- package/templates/postman-node/mocks/external.js +14 -0
- package/templates/postman-node/package.json.ejs +39 -0
- package/templates/postman-node/requirements.txt +1 -0
- package/templates/postman-node/scripts/database/cleanup-mysql.sh +12 -0
- package/templates/postman-node/scripts/database/run-migrations.js +29 -0
- package/templates/postman-node/scripts/database/start-mysql.sh +34 -0
- package/templates/postman-node/scripts/database/wait-for-mysql.sh +36 -0
- package/templates/postman-node/scripts/lib/api_coverage_node.py +1137 -0
- package/templates/postman-node/scripts/lib/fetch-jwt.sh +86 -0
- package/templates/postman-node/scripts/lib/run-newman.sh +104 -0
- package/templates/postman-node/scripts/lib/setup-database.sh +55 -0
- package/templates/postman-node/scripts/lib/start-app.sh +48 -0
- package/templates/postman-node/scripts/lib/utils.sh +114 -0
- package/templates/postman-node/scripts/report-generators/stage-report-artifacts.sh +26 -0
- package/templates/postman-node/scripts/run-all.sh +303 -0
- package/templates/postman-node/scripts/runners/run-tests.sh +123 -0
- package/templates/postman-node/scripts/setup-mocks.js.ejs +29 -0
- package/templates/postman-node/stryker.config.js.ejs +51 -0
- package/templates/rules/local-test-setup.mdc +420 -0
- package/templates/rules/testing-node.mdc +66 -0
- package/templates/rules/testing.mdc +248 -0
- package/templates/skills/_shared/postman-standards.md +380 -0
- package/templates/skills/enhance-test-pipeline/SKILL-java.md +483 -0
- package/templates/skills/enhance-test-pipeline/SKILL-node.md +431 -0
- package/templates/skills/enhance-test-pipeline/SKILL.md +9 -0
- package/templates/skills/review-test-suite/SKILL-java.md +137 -0
- package/templates/skills/review-test-suite/SKILL-node.md +78 -0
- package/templates/skills/review-test-suite/SKILL.md +9 -0
- package/templates/skills/run-test-suite/SKILL-java.md +186 -0
- package/templates/skills/run-test-suite/SKILL-node.md +191 -0
- package/templates/skills/run-test-suite/SKILL.md +9 -0
- package/templates/skills/setup-api-tests/SKILL-java.md +1094 -0
- package/templates/skills/setup-api-tests/SKILL-node.md +141 -0
- package/templates/skills/setup-api-tests/SKILL.md +9 -0
- package/templates/skills/setup-mutation-tests/SKILL-java.md +303 -0
- package/templates/skills/setup-mutation-tests/SKILL-node.md +408 -0
- package/templates/skills/setup-mutation-tests/SKILL.md +9 -0
- package/templates/skills/setup-test-pipeline/SKILL-java.md +454 -0
- package/templates/skills/setup-test-pipeline/SKILL-node.md +318 -0
- package/templates/skills/setup-test-pipeline/SKILL.md +9 -0
- package/templates/skills/write-api-tests/SKILL-java.md +115 -0
- package/templates/skills/write-api-tests/SKILL-node.md +83 -0
- package/templates/skills/write-api-tests/SKILL.md +9 -0
- package/templates/stryker.config.js +50 -0
|
@@ -0,0 +1,1122 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
API Coverage Matrix Generator.
|
|
4
|
+
|
|
5
|
+
Analyzes Java Spring Boot controllers and Postman collections to produce
|
|
6
|
+
an HTML coverage report showing which endpoints have test coverage.
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
python3 api_coverage.py --controller-dir DIR --postman-dir DIR --html-output FILE
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import json
|
|
16
|
+
import sys
|
|
17
|
+
from collections import defaultdict
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
from html import escape as esc
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
from report_utils import (
|
|
23
|
+
get_css, get_html_template,
|
|
24
|
+
build_summary_card, build_coverage_item, build_gaps_section,
|
|
25
|
+
build_collapsible_table, build_endpoint_row
|
|
26
|
+
)
|
|
27
|
+
SHARED_CSS = get_css()
|
|
28
|
+
USE_HELPERS = True
|
|
29
|
+
except ImportError:
|
|
30
|
+
SHARED_CSS = None
|
|
31
|
+
USE_HELPERS = False
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def parse_args():
|
|
35
|
+
parser = argparse.ArgumentParser(description='API Coverage Matrix Generator')
|
|
36
|
+
parser.add_argument('--controller-dir', required=True, help='Path to Java controller directory')
|
|
37
|
+
parser.add_argument('--postman-dir', required=True, help='Path to Postman directory')
|
|
38
|
+
parser.add_argument('--html-output', required=True, help='Path for HTML output file')
|
|
39
|
+
return parser.parse_args()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
args = parse_args()
|
|
43
|
+
CONTROLLER_DIR = args.controller_dir
|
|
44
|
+
POSTMAN_DIR = args.postman_dir
|
|
45
|
+
HTML_OUTPUT_FILE = args.html_output
|
|
46
|
+
|
|
47
|
+
endpoints = []
|
|
48
|
+
tested = []
|
|
49
|
+
errors = []
|
|
50
|
+
|
|
51
|
+
print(f"Scanning {CONTROLLER_DIR}", flush=True)
|
|
52
|
+
print(f"Scanning {POSTMAN_DIR}", flush=True)
|
|
53
|
+
|
|
54
|
+
def safe_read_file(filepath, max_size_mb=50):
|
|
55
|
+
"""Safely read a file with size limits."""
|
|
56
|
+
try:
|
|
57
|
+
file_size = os.path.getsize(filepath)
|
|
58
|
+
if file_size > max_size_mb * 1024 * 1024:
|
|
59
|
+
errors.append(f"Skipped {filepath}: file too large")
|
|
60
|
+
return None
|
|
61
|
+
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
|
|
62
|
+
return f.read()
|
|
63
|
+
except Exception as e:
|
|
64
|
+
errors.append(f"Error reading {filepath}: {str(e)}")
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
def join_concatenated_strings(text):
|
|
68
|
+
"""Join Java string concatenations like '"part1" + "part2"' into single strings."""
|
|
69
|
+
result = text
|
|
70
|
+
result = re.sub(r'"\s*\n\s*\+\s*"', '', result) # NOSONAR - input is trusted local Java source files
|
|
71
|
+
result = re.sub(r'"\s*\+\s*\n\s*"', '', result) # NOSONAR
|
|
72
|
+
result = re.sub(r'"\s*\+\s*"', '', result)
|
|
73
|
+
return result
|
|
74
|
+
|
|
75
|
+
def resolve_constants(content, controller_dir):
|
|
76
|
+
"""Resolve Java constant references (e.g., PathConstants.V1) to their string values."""
|
|
77
|
+
constants = {}
|
|
78
|
+
constants_dir = os.path.dirname(controller_dir)
|
|
79
|
+
|
|
80
|
+
for root, dirs, files in os.walk(constants_dir):
|
|
81
|
+
dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['target', 'build']]
|
|
82
|
+
for f in files:
|
|
83
|
+
if f.endswith('.java'):
|
|
84
|
+
fpath = os.path.join(root, f)
|
|
85
|
+
try:
|
|
86
|
+
fcontent = safe_read_file(fpath)
|
|
87
|
+
if fcontent is None:
|
|
88
|
+
continue
|
|
89
|
+
class_match = re.search(r'(?:public\s+)?class\s+(\w+)', fcontent)
|
|
90
|
+
if not class_match:
|
|
91
|
+
continue
|
|
92
|
+
class_name = class_match.group(1)
|
|
93
|
+
for m in re.finditer(
|
|
94
|
+
r'(?:public\s+)?static\s+final\s+String\s+(\w+)\s*=\s*"([^"]*)"',
|
|
95
|
+
fcontent
|
|
96
|
+
):
|
|
97
|
+
constants[f"{class_name}.{m.group(1)}"] = m.group(2)
|
|
98
|
+
except Exception:
|
|
99
|
+
continue
|
|
100
|
+
|
|
101
|
+
if not constants:
|
|
102
|
+
return content
|
|
103
|
+
|
|
104
|
+
def replace_constant(match):
|
|
105
|
+
ref = match.group(0)
|
|
106
|
+
return f'"{constants[ref]}"' if ref in constants else match.group(0)
|
|
107
|
+
|
|
108
|
+
pattern = '|'.join(re.escape(k) for k in sorted(constants.keys(), key=len, reverse=True))
|
|
109
|
+
resolved = re.sub(pattern, replace_constant, content)
|
|
110
|
+
return resolved
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def extract_endpoints_from_file(filepath):
|
|
114
|
+
"""Extract API endpoints from a Java controller file."""
|
|
115
|
+
content = safe_read_file(filepath)
|
|
116
|
+
if content is None:
|
|
117
|
+
return
|
|
118
|
+
|
|
119
|
+
content = resolve_constants(content, CONTROLLER_DIR)
|
|
120
|
+
content = join_concatenated_strings(content)
|
|
121
|
+
controller_name = os.path.basename(filepath).replace('.java', '')
|
|
122
|
+
|
|
123
|
+
class_path_match = re.search(r'@RequestMapping\s*\(\s*(?:path\s*=\s*)?["\']([^"\']+)["\']', content)
|
|
124
|
+
class_path = class_path_match.group(1) if class_path_match else ''
|
|
125
|
+
|
|
126
|
+
method_blocks = re.split(r'(?=@(?:Get|Post|Put|Delete|Patch)Mapping)', content)
|
|
127
|
+
|
|
128
|
+
for block in method_blocks:
|
|
129
|
+
if len(endpoints) >= 1000:
|
|
130
|
+
return
|
|
131
|
+
|
|
132
|
+
if not re.match(r'@(?:Get|Post|Put|Delete|Patch)Mapping', block.strip()):
|
|
133
|
+
continue
|
|
134
|
+
|
|
135
|
+
mapping_patterns = [
|
|
136
|
+
r'@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*value\s*=\s*\{?\s*["\']([^"\']+)["\']',
|
|
137
|
+
r'@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*\{?\s*["\']([^"\']+)["\']',
|
|
138
|
+
r'@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*["\']([^"\']+)["\']',
|
|
139
|
+
r'@(Get|Post|Put|Delete|Patch)Mapping\(["\']([^"\']+)["\']',
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
http_method = None
|
|
143
|
+
path = None
|
|
144
|
+
|
|
145
|
+
for pattern in mapping_patterns:
|
|
146
|
+
match = re.search(pattern, block)
|
|
147
|
+
if match:
|
|
148
|
+
http_method = match.group(1).upper()
|
|
149
|
+
path = match.group(2)
|
|
150
|
+
break
|
|
151
|
+
|
|
152
|
+
if not http_method:
|
|
153
|
+
first_line = block.strip().split('\n')[0].strip()
|
|
154
|
+
no_path_match = re.match( # NOSONAR - input is trusted local Java source files
|
|
155
|
+
r'@(Get|Post|Put|Delete|Patch)Mapping\s*(?:\([^"\']*\))?\s*$',
|
|
156
|
+
first_line
|
|
157
|
+
)
|
|
158
|
+
if not no_path_match:
|
|
159
|
+
no_path_match = re.match(
|
|
160
|
+
r'@(Get|Post|Put|Delete|Patch)Mapping\s*$',
|
|
161
|
+
first_line
|
|
162
|
+
)
|
|
163
|
+
if no_path_match:
|
|
164
|
+
http_method = no_path_match.group(1).upper()
|
|
165
|
+
path = ''
|
|
166
|
+
|
|
167
|
+
if not http_method:
|
|
168
|
+
continue
|
|
169
|
+
|
|
170
|
+
if class_path and path:
|
|
171
|
+
full_path = f"{class_path.rstrip('/')}/{path.lstrip('/')}".replace('//', '/')
|
|
172
|
+
elif class_path:
|
|
173
|
+
full_path = class_path
|
|
174
|
+
else:
|
|
175
|
+
full_path = path
|
|
176
|
+
|
|
177
|
+
full_path = re.sub(r'\.json$', '', full_path)
|
|
178
|
+
full_path = full_path.rstrip('/')
|
|
179
|
+
if not full_path.startswith('/'):
|
|
180
|
+
full_path = '/' + full_path
|
|
181
|
+
|
|
182
|
+
param_block = block
|
|
183
|
+
defined_params = set()
|
|
184
|
+
include_params = set()
|
|
185
|
+
|
|
186
|
+
named_param_pattern = r'@RequestParam\s*\(\s*(?:(?:name|value)\s*=\s*)?["\']([^"\']+)["\'][^)]*\)' # NOSONAR
|
|
187
|
+
unnamed_param_pattern = r'@RequestParam\s*(?:\([^"\']*\))?\s*\w+\s+(\w+)' # NOSONAR
|
|
188
|
+
|
|
189
|
+
named_params = set()
|
|
190
|
+
for pm in re.finditer(named_param_pattern, param_block):
|
|
191
|
+
param_name = pm.group(1)
|
|
192
|
+
if param_name:
|
|
193
|
+
named_params.add(param_name)
|
|
194
|
+
defined_params.add(param_name.lower().replace('_', ''))
|
|
195
|
+
if 'include' in param_name.lower():
|
|
196
|
+
include_params.add(param_name)
|
|
197
|
+
|
|
198
|
+
for pm in re.finditer(unnamed_param_pattern, param_block):
|
|
199
|
+
var_name = pm.group(1)
|
|
200
|
+
if var_name:
|
|
201
|
+
match_start = pm.start()
|
|
202
|
+
preceding = param_block[:match_start]
|
|
203
|
+
last_annotation = preceding.rfind('@RequestParam')
|
|
204
|
+
if last_annotation >= 0:
|
|
205
|
+
annotation_text = param_block[last_annotation:match_start]
|
|
206
|
+
if 'name' not in annotation_text and '"\'' not in annotation_text[:50]:
|
|
207
|
+
defined_params.add(var_name.lower())
|
|
208
|
+
if 'include' in var_name.lower():
|
|
209
|
+
include_params.add(var_name)
|
|
210
|
+
|
|
211
|
+
endpoint = {
|
|
212
|
+
'method': http_method,
|
|
213
|
+
'path': full_path,
|
|
214
|
+
'controller': controller_name,
|
|
215
|
+
'defined_params': defined_params,
|
|
216
|
+
'has_include_param': len(include_params) > 0
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
is_duplicate = False
|
|
220
|
+
for existing in endpoints:
|
|
221
|
+
if existing['method'] == endpoint['method'] and existing['path'] == endpoint['path']:
|
|
222
|
+
existing['defined_params'].update(endpoint['defined_params'])
|
|
223
|
+
if endpoint['has_include_param']:
|
|
224
|
+
existing['has_include_param'] = True
|
|
225
|
+
is_duplicate = True
|
|
226
|
+
break
|
|
227
|
+
|
|
228
|
+
if not is_duplicate:
|
|
229
|
+
endpoints.append(endpoint)
|
|
230
|
+
|
|
231
|
+
# Process controller files
|
|
232
|
+
try:
|
|
233
|
+
for root, dirs, files in os.walk(CONTROLLER_DIR):
|
|
234
|
+
dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['target', 'build', 'node_modules']]
|
|
235
|
+
|
|
236
|
+
for file in files:
|
|
237
|
+
if file.endswith('.java'):
|
|
238
|
+
filepath = os.path.join(root, file)
|
|
239
|
+
try:
|
|
240
|
+
extract_endpoints_from_file(filepath)
|
|
241
|
+
except Exception as e:
|
|
242
|
+
errors.append(f"Error processing {file}: {str(e)}")
|
|
243
|
+
except Exception as e:
|
|
244
|
+
print(f"ERROR: Failed to scan controller directory: {e}", flush=True)
|
|
245
|
+
sys.exit(1)
|
|
246
|
+
|
|
247
|
+
def extract_tests_from_item(item, collection_name, depth=0):
|
|
248
|
+
"""Extract test information from Postman collection items."""
|
|
249
|
+
if depth > 20:
|
|
250
|
+
return []
|
|
251
|
+
|
|
252
|
+
results = []
|
|
253
|
+
|
|
254
|
+
if isinstance(item, list):
|
|
255
|
+
for i in item:
|
|
256
|
+
results.extend(extract_tests_from_item(i, collection_name, depth + 1))
|
|
257
|
+
elif isinstance(item, dict):
|
|
258
|
+
if 'request' in item:
|
|
259
|
+
try:
|
|
260
|
+
request = item['request']
|
|
261
|
+
if not isinstance(request, dict):
|
|
262
|
+
return results
|
|
263
|
+
|
|
264
|
+
method = request.get('method', 'GET')
|
|
265
|
+
|
|
266
|
+
url = request.get('url', {})
|
|
267
|
+
query_params = set()
|
|
268
|
+
query_param_values = defaultdict(set)
|
|
269
|
+
|
|
270
|
+
if isinstance(url, str):
|
|
271
|
+
url_path = url
|
|
272
|
+
if '?' in url_path:
|
|
273
|
+
url_path, query_string = url_path.split('?', 1)
|
|
274
|
+
for param in query_string.split('&'):
|
|
275
|
+
if '=' in param:
|
|
276
|
+
key, val = param.split('=', 1)
|
|
277
|
+
query_params.add(key)
|
|
278
|
+
query_param_values[key].add(val)
|
|
279
|
+
elif isinstance(url, dict):
|
|
280
|
+
path_parts = url.get('path', [])
|
|
281
|
+
if isinstance(path_parts, list):
|
|
282
|
+
url_path = '/' + '/'.join(str(p) for p in path_parts) if path_parts else ''
|
|
283
|
+
else:
|
|
284
|
+
url_path = str(path_parts)
|
|
285
|
+
|
|
286
|
+
for qp in url.get('query', []) or []:
|
|
287
|
+
if isinstance(qp, dict):
|
|
288
|
+
key = qp.get('key', '')
|
|
289
|
+
val = qp.get('value', '')
|
|
290
|
+
if key:
|
|
291
|
+
query_params.add(key)
|
|
292
|
+
if val:
|
|
293
|
+
query_param_values[key].add(val)
|
|
294
|
+
else:
|
|
295
|
+
url_path = ''
|
|
296
|
+
|
|
297
|
+
if '?' in url_path:
|
|
298
|
+
url_path, extra_qs = url_path.split('?', 1)
|
|
299
|
+
for param in extra_qs.split('&'):
|
|
300
|
+
if '=' in param:
|
|
301
|
+
key, val = param.split('=', 1)
|
|
302
|
+
query_params.add(key)
|
|
303
|
+
query_param_values[key].add(val)
|
|
304
|
+
|
|
305
|
+
url_path = re.sub(r'\.json$', '', url_path)
|
|
306
|
+
url_path = re.sub(r'\{\{[^}]+\}\}', '{VAR}', url_path)
|
|
307
|
+
url_path = re.sub(r':[^/]+', '{VAR}', url_path)
|
|
308
|
+
|
|
309
|
+
status_codes = set()
|
|
310
|
+
body_assertions = 0
|
|
311
|
+
has_oneof = False
|
|
312
|
+
has_skip = False
|
|
313
|
+
skip_reason = None
|
|
314
|
+
skip_env = None
|
|
315
|
+
|
|
316
|
+
# Check pre-request scripts for skips
|
|
317
|
+
for event in item.get('event', []) or []:
|
|
318
|
+
if isinstance(event, dict) and event.get('listen') == 'prerequest':
|
|
319
|
+
script = event.get('script', {})
|
|
320
|
+
if isinstance(script, dict):
|
|
321
|
+
exec_lines = script.get('exec', [])
|
|
322
|
+
if isinstance(exec_lines, list):
|
|
323
|
+
prereq_text = '\n'.join(str(line) for line in exec_lines)
|
|
324
|
+
else:
|
|
325
|
+
prereq_text = str(exec_lines)
|
|
326
|
+
|
|
327
|
+
# Detect pm.execution.skipRequest()
|
|
328
|
+
if 'pm.execution.skipRequest()' in prereq_text:
|
|
329
|
+
has_skip = True
|
|
330
|
+
|
|
331
|
+
# Extract skip reason - multiple formats supported:
|
|
332
|
+
# Format 1: console.log('[SKIP] Reason: ... | Env: ...')
|
|
333
|
+
# Format 2: console.log('[SKIP] ...') (reason only)
|
|
334
|
+
# Format 3: // SKIP: reason
|
|
335
|
+
|
|
336
|
+
# Try format with Reason and Env (flexible spacing/separators)
|
|
337
|
+
skip_match = re.search(
|
|
338
|
+
r"\[SKIP\].*?Reason:\s*([^|,;]+)(?:[|,;]\s*Env:\s*(\S+))?",
|
|
339
|
+
prereq_text, re.IGNORECASE
|
|
340
|
+
)
|
|
341
|
+
if skip_match:
|
|
342
|
+
skip_reason = skip_match.group(1).strip()
|
|
343
|
+
if skip_match.group(2):
|
|
344
|
+
skip_env = skip_match.group(2).strip().lower()
|
|
345
|
+
|
|
346
|
+
# Try simple [SKIP] message format
|
|
347
|
+
if not skip_reason:
|
|
348
|
+
simple_match = re.search(r"\[SKIP\]\s*(.+?)(?:['\"]|$)", prereq_text)
|
|
349
|
+
if simple_match and 'Reason:' not in simple_match.group(1):
|
|
350
|
+
skip_reason = simple_match.group(1).strip()
|
|
351
|
+
|
|
352
|
+
# Try comment-based skip reason
|
|
353
|
+
if not skip_reason:
|
|
354
|
+
comment_match = re.search(r"//\s*SKIP[:\s]+(.+)", prereq_text)
|
|
355
|
+
if comment_match:
|
|
356
|
+
skip_reason = comment_match.group(1).strip()
|
|
357
|
+
|
|
358
|
+
# Auto-detect environment from the if condition if not specified
|
|
359
|
+
if not skip_env:
|
|
360
|
+
# Check for environment condition patterns
|
|
361
|
+
env_patterns = [
|
|
362
|
+
# pm.environment.get('environment') === 'local'
|
|
363
|
+
(r"pm\.environment\.get\(['\"]environment['\"]\)\s*===\s*['\"](\w+)['\"]", lambda m: m.group(1)),
|
|
364
|
+
# pm.environment.get('environment') !== 'local' -> skip in non-local
|
|
365
|
+
(r"pm\.environment\.get\(['\"]environment['\"]\)\s*!==\s*['\"]local['\"]", lambda m: 'staging'),
|
|
366
|
+
(r"pm\.environment\.get\(['\"]environment['\"]\)\s*!==\s*['\"]staging['\"]", lambda m: 'local'),
|
|
367
|
+
# pm.variables.get('env') patterns
|
|
368
|
+
(r"pm\.variables\.get\(['\"]env['\"]\)\s*===\s*['\"](\w+)['\"]", lambda m: m.group(1)),
|
|
369
|
+
]
|
|
370
|
+
for pattern, extractor in env_patterns:
|
|
371
|
+
env_match = re.search(pattern, prereq_text)
|
|
372
|
+
if env_match:
|
|
373
|
+
skip_env = extractor(env_match).lower()
|
|
374
|
+
break
|
|
375
|
+
|
|
376
|
+
for event in item.get('event', []) or []:
|
|
377
|
+
if isinstance(event, dict) and event.get('listen') == 'test':
|
|
378
|
+
script = event.get('script', {})
|
|
379
|
+
if isinstance(script, dict):
|
|
380
|
+
exec_lines = script.get('exec', [])
|
|
381
|
+
if isinstance(exec_lines, list):
|
|
382
|
+
script_text = '\n'.join(str(line) for line in exec_lines)
|
|
383
|
+
else:
|
|
384
|
+
script_text = str(exec_lines)
|
|
385
|
+
|
|
386
|
+
matches = re.findall(r'to\.have\.status\((\d+)\)', script_text)
|
|
387
|
+
for m in matches:
|
|
388
|
+
status_codes.add(str(m))
|
|
389
|
+
|
|
390
|
+
matches_equal = re.findall(r'pm\.response\.code\)\.to\.equal\((\d+)\)', script_text)
|
|
391
|
+
for m in matches_equal:
|
|
392
|
+
status_codes.add(str(m))
|
|
393
|
+
|
|
394
|
+
matches_eql = re.findall(r'pm\.response\.code\)\.to\.eql\((\d+)\)', script_text)
|
|
395
|
+
for m in matches_eql:
|
|
396
|
+
status_codes.add(str(m))
|
|
397
|
+
|
|
398
|
+
# Detect weak assertions: oneOf, to.include with array for status codes
|
|
399
|
+
if 'oneOf' in script_text:
|
|
400
|
+
has_oneof = True
|
|
401
|
+
# Detect pm.expect([...]).to.include(pm.response.code) pattern - weak status code assertion
|
|
402
|
+
if re.search(r'pm\.expect\s*\(\s*\[.*?\]\s*\)\.to\.include\s*\(\s*pm\.response\.code', script_text):
|
|
403
|
+
has_oneof = True
|
|
404
|
+
# Detect pm.expect(pm.response.code).to.be.oneOf([...]) pattern
|
|
405
|
+
if re.search(r'pm\.response\.code\s*\)\s*\.to\.be\.oneOf', script_text):
|
|
406
|
+
has_oneof = True
|
|
407
|
+
|
|
408
|
+
# Extract status codes from oneOf/include arrays (weak but still counts for coverage)
|
|
409
|
+
for oneof_match in re.finditer(r'\.to\.be\.oneOf\s*\(\s*\[([^\]]+)\]', script_text):
|
|
410
|
+
for code in re.findall(r'\d{3}', oneof_match.group(1)):
|
|
411
|
+
status_codes.add(str(code))
|
|
412
|
+
for include_match in re.finditer(r'pm\.expect\s*\(\s*\[([\d,\s]+)\]\s*\)\.to\.include', script_text):
|
|
413
|
+
for code in re.findall(r'\d{3}', include_match.group(1)):
|
|
414
|
+
status_codes.add(str(code))
|
|
415
|
+
|
|
416
|
+
# Check for 5xx status code assertions (bad practice)
|
|
417
|
+
has_5xx = any(str(code).startswith('5') for code in status_codes)
|
|
418
|
+
|
|
419
|
+
body_patterns = [
|
|
420
|
+
r'pm\.expect\s*\(\s*jsonData\.',
|
|
421
|
+
r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.be\.an\s*\(',
|
|
422
|
+
r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.have\.property\s*\(',
|
|
423
|
+
r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.not\.be\.null',
|
|
424
|
+
r'pm\.expect\s*\(\s*response\.',
|
|
425
|
+
r'\.to\.have\.property\(',
|
|
426
|
+
r'\.to\.include\(',
|
|
427
|
+
r'\.to\.eql\(',
|
|
428
|
+
]
|
|
429
|
+
for p in body_patterns:
|
|
430
|
+
body_assertions += len(re.findall(p, script_text))
|
|
431
|
+
|
|
432
|
+
has_request_body = False
|
|
433
|
+
request_body_hash = None
|
|
434
|
+
request_body_fields = set()
|
|
435
|
+
body = request.get('body', {})
|
|
436
|
+
if isinstance(body, dict):
|
|
437
|
+
raw_body = body.get('raw', '')
|
|
438
|
+
if raw_body and isinstance(raw_body, str) and raw_body.strip():
|
|
439
|
+
has_request_body = True
|
|
440
|
+
request_body_hash = hash(raw_body.strip())
|
|
441
|
+
# Try JSON parsing first, fallback to regex for Postman template variables
|
|
442
|
+
try:
|
|
443
|
+
parsed = json.loads(raw_body)
|
|
444
|
+
if isinstance(parsed, dict):
|
|
445
|
+
request_body_fields = set(parsed.keys())
|
|
446
|
+
except (json.JSONDecodeError, ValueError):
|
|
447
|
+
# Use regex to extract field names when JSON parsing fails
|
|
448
|
+
# This handles Postman template variables like {{variable}}
|
|
449
|
+
field_pattern = r'"([^"]+)"\s*:'
|
|
450
|
+
request_body_fields = set(re.findall(field_pattern, raw_body))
|
|
451
|
+
|
|
452
|
+
if url_path:
|
|
453
|
+
results.append({
|
|
454
|
+
'method': method,
|
|
455
|
+
'path': url_path,
|
|
456
|
+
'name': item.get('name', ''),
|
|
457
|
+
'status_codes': list(status_codes) if status_codes else [],
|
|
458
|
+
'collection': collection_name,
|
|
459
|
+
'body_assertions': body_assertions,
|
|
460
|
+
'has_request_body': has_request_body,
|
|
461
|
+
'request_body_hash': request_body_hash,
|
|
462
|
+
'request_body_fields': request_body_fields,
|
|
463
|
+
'query_params': query_params,
|
|
464
|
+
'query_param_values': dict(query_param_values),
|
|
465
|
+
'has_oneof': has_oneof,
|
|
466
|
+
'has_5xx': any(str(code).startswith('5') for code in status_codes),
|
|
467
|
+
'has_skip': has_skip,
|
|
468
|
+
'skip_reason': skip_reason,
|
|
469
|
+
'skip_env': skip_env
|
|
470
|
+
})
|
|
471
|
+
except Exception as e:
|
|
472
|
+
errors.append(f"Error parsing request in {collection_name}: {str(e)}")
|
|
473
|
+
|
|
474
|
+
if 'item' in item:
|
|
475
|
+
results.extend(extract_tests_from_item(item['item'], collection_name, depth + 1))
|
|
476
|
+
|
|
477
|
+
return results
|
|
478
|
+
|
|
479
|
+
def find_collection_files(base_dir):
|
|
480
|
+
"""Find all Postman collection JSON files in the given directory."""
|
|
481
|
+
collection_files = []
|
|
482
|
+
seen_files = set() # Track seen file paths to avoid duplicates
|
|
483
|
+
|
|
484
|
+
search_dirs = [base_dir]
|
|
485
|
+
|
|
486
|
+
for subdir in ['collections', 'api-tests', 'tests']:
|
|
487
|
+
subpath = os.path.join(base_dir, subdir)
|
|
488
|
+
if os.path.isdir(subpath):
|
|
489
|
+
search_dirs.append(subpath)
|
|
490
|
+
|
|
491
|
+
seen_dirs = set()
|
|
492
|
+
|
|
493
|
+
for search_dir in search_dirs:
|
|
494
|
+
if search_dir in seen_dirs:
|
|
495
|
+
continue
|
|
496
|
+
seen_dirs.add(search_dir)
|
|
497
|
+
|
|
498
|
+
try:
|
|
499
|
+
for root, dirs, files in os.walk(search_dir):
|
|
500
|
+
# Skip directories that definitely don't contain Postman collections
|
|
501
|
+
dirs[:] = [d for d in dirs if d not in [
|
|
502
|
+
'node_modules', 'environments', 'reports', 'config',
|
|
503
|
+
'scripts', '.git', 'data', 'schemas'
|
|
504
|
+
]]
|
|
505
|
+
|
|
506
|
+
for file in files:
|
|
507
|
+
if file.endswith('.json'):
|
|
508
|
+
# Only skip files that are DEFINITELY not Postman collections
|
|
509
|
+
# Use exact matches or specific patterns to avoid false positives
|
|
510
|
+
# The Postman structure validation later will handle edge cases
|
|
511
|
+
skip_exact = [
|
|
512
|
+
'package.json', 'package-lock.json',
|
|
513
|
+
'tsconfig.json', 'jsconfig.json',
|
|
514
|
+
'.eslintrc.json', '.prettierrc.json',
|
|
515
|
+
'newman-reporter-config.json'
|
|
516
|
+
]
|
|
517
|
+
|
|
518
|
+
# Skip Postman environment files
|
|
519
|
+
# Common patterns: *.environment.json, *_environment.json,
|
|
520
|
+
# *.postman_environment.json (Postman export format)
|
|
521
|
+
is_environment_file = (
|
|
522
|
+
file.lower().endswith('.environment.json') or
|
|
523
|
+
file.lower().endswith('_environment.json') or
|
|
524
|
+
file.lower().endswith('.postman_environment.json') or
|
|
525
|
+
file.lower() == 'environment.json'
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
if file.lower() in skip_exact:
|
|
529
|
+
continue
|
|
530
|
+
if is_environment_file:
|
|
531
|
+
continue
|
|
532
|
+
|
|
533
|
+
# Deduplicate by absolute path
|
|
534
|
+
full_path = os.path.abspath(os.path.join(root, file))
|
|
535
|
+
if full_path not in seen_files:
|
|
536
|
+
seen_files.add(full_path)
|
|
537
|
+
collection_files.append(full_path)
|
|
538
|
+
except Exception as e:
|
|
539
|
+
errors.append(f"Error scanning {search_dir}: {str(e)}")
|
|
540
|
+
|
|
541
|
+
return collection_files
|
|
542
|
+
|
|
543
|
+
try:
|
|
544
|
+
collection_files = find_collection_files(POSTMAN_DIR)
|
|
545
|
+
print(f"Found {len(collection_files)} collection file(s)", flush=True)
|
|
546
|
+
|
|
547
|
+
for filepath in collection_files:
|
|
548
|
+
file = os.path.basename(filepath)
|
|
549
|
+
|
|
550
|
+
file_size = os.path.getsize(filepath)
|
|
551
|
+
if file_size > 50 * 1024 * 1024:
|
|
552
|
+
errors.append(f"Skipped {file}: collection too large")
|
|
553
|
+
continue
|
|
554
|
+
|
|
555
|
+
try:
|
|
556
|
+
content = safe_read_file(filepath)
|
|
557
|
+
if content:
|
|
558
|
+
collection = json.loads(content)
|
|
559
|
+
if not isinstance(collection, dict) or ('item' not in collection and 'info' not in collection):
|
|
560
|
+
continue
|
|
561
|
+
collection_name = file.replace('.json', '')
|
|
562
|
+
tests = extract_tests_from_item(collection.get('item', []), collection_name)
|
|
563
|
+
tested.extend(tests)
|
|
564
|
+
except json.JSONDecodeError as e:
|
|
565
|
+
errors.append(f"Invalid JSON in {file}: {str(e)}")
|
|
566
|
+
except Exception as e:
|
|
567
|
+
errors.append(f"Error processing {file}: {str(e)}")
|
|
568
|
+
except Exception as e:
|
|
569
|
+
print(f"ERROR: Failed to scan Postman directory: {e}", flush=True)
|
|
570
|
+
sys.exit(1)
|
|
571
|
+
|
|
572
|
+
# Sort and deduplicate endpoints
|
|
573
|
+
seen = set()
|
|
574
|
+
unique_endpoints = []
|
|
575
|
+
for ep in endpoints:
|
|
576
|
+
key = (ep['method'], ep['path'])
|
|
577
|
+
if key not in seen:
|
|
578
|
+
seen.add(key)
|
|
579
|
+
unique_endpoints.append(ep)
|
|
580
|
+
endpoints = sorted(unique_endpoints, key=lambda x: (x['path'], x['method']))
|
|
581
|
+
|
|
582
|
+
# Aggregate stats per endpoint
|
|
583
|
+
endpoint_stats = defaultdict(lambda: {
|
|
584
|
+
'test_count': 0,
|
|
585
|
+
'status_codes': set(),
|
|
586
|
+
'body_assertions': 0,
|
|
587
|
+
'request_body_variations': set(),
|
|
588
|
+
'has_body_test': False,
|
|
589
|
+
'query_params': set(),
|
|
590
|
+
'query_param_values': defaultdict(set),
|
|
591
|
+
'request_body_fields': set(),
|
|
592
|
+
'has_oneof': False,
|
|
593
|
+
'oneof_count': 0,
|
|
594
|
+
'has_5xx': False,
|
|
595
|
+
'count_5xx': 0,
|
|
596
|
+
'skipped_tests': []
|
|
597
|
+
})
|
|
598
|
+
|
|
599
|
+
# Track all skipped tests separately for reporting
|
|
600
|
+
all_skipped_tests = []
|
|
601
|
+
|
|
602
|
+
def normalize_path(path):
|
|
603
|
+
"""Normalize path for comparison."""
|
|
604
|
+
if not path:
|
|
605
|
+
return ''
|
|
606
|
+
normalized = re.sub(r'^/?(api/)?v\d+/', '/', path)
|
|
607
|
+
normalized = re.sub(r'\{\{[^}]+\}\}', '{VAR}', normalized)
|
|
608
|
+
normalized = re.sub(r'\{[^}]+\}', '{VAR}', normalized)
|
|
609
|
+
normalized = re.sub(r':[^/]+', '{VAR}', normalized)
|
|
610
|
+
normalized = re.sub(r'/(\d+)(?=/|$)', '/{VAR}', normalized)
|
|
611
|
+
normalized = re.sub(r'\.json$', '', normalized)
|
|
612
|
+
return normalized.lower().replace('{var}', '{VAR}')
|
|
613
|
+
|
|
614
|
+
def paths_match(controller_path, test_path):
|
|
615
|
+
"""Check if a test path matches a controller path."""
|
|
616
|
+
norm_controller = normalize_path(controller_path)
|
|
617
|
+
norm_test = normalize_path(test_path)
|
|
618
|
+
|
|
619
|
+
if norm_controller == norm_test:
|
|
620
|
+
return True
|
|
621
|
+
|
|
622
|
+
try:
|
|
623
|
+
pattern = re.escape(norm_controller).replace(r'\{VAR\}', '[^/]+')
|
|
624
|
+
if re.fullmatch(pattern, norm_test):
|
|
625
|
+
return True
|
|
626
|
+
except re.error:
|
|
627
|
+
pass
|
|
628
|
+
|
|
629
|
+
return False
|
|
630
|
+
|
|
631
|
+
for test in tested:
|
|
632
|
+
test_path = test['path']
|
|
633
|
+
test_method = test['method']
|
|
634
|
+
|
|
635
|
+
# Track skipped tests globally
|
|
636
|
+
if test.get('has_skip', False):
|
|
637
|
+
skip_info = {
|
|
638
|
+
'name': test.get('name', 'Unknown'),
|
|
639
|
+
'collection': test.get('collection', ''),
|
|
640
|
+
'method': test_method,
|
|
641
|
+
'path': test_path,
|
|
642
|
+
'reason': test.get('skip_reason'),
|
|
643
|
+
'env': test.get('skip_env')
|
|
644
|
+
}
|
|
645
|
+
all_skipped_tests.append(skip_info)
|
|
646
|
+
|
|
647
|
+
matched = False
|
|
648
|
+
for ep in endpoints:
|
|
649
|
+
if test_method == ep['method'] and paths_match(ep['path'], test_path):
|
|
650
|
+
key = f"{ep['method']} {ep['path']}"
|
|
651
|
+
endpoint_stats[key]['test_count'] += 1
|
|
652
|
+
endpoint_stats[key]['status_codes'].update(test['status_codes'])
|
|
653
|
+
endpoint_stats[key]['body_assertions'] += test['body_assertions']
|
|
654
|
+
if test['body_assertions'] > 0:
|
|
655
|
+
endpoint_stats[key]['has_body_test'] = True
|
|
656
|
+
if test['request_body_hash']:
|
|
657
|
+
endpoint_stats[key]['request_body_variations'].add(test['request_body_hash'])
|
|
658
|
+
endpoint_stats[key]['query_params'].update(test.get('query_params', set()))
|
|
659
|
+
for param, values in test.get('query_param_values', {}).items():
|
|
660
|
+
endpoint_stats[key]['query_param_values'][param].update(values)
|
|
661
|
+
endpoint_stats[key]['request_body_fields'].update(test.get('request_body_fields', set()))
|
|
662
|
+
if test.get('has_oneof', False):
|
|
663
|
+
endpoint_stats[key]['has_oneof'] = True
|
|
664
|
+
endpoint_stats[key]['oneof_count'] += 1
|
|
665
|
+
if test.get('has_5xx', False):
|
|
666
|
+
endpoint_stats[key]['has_5xx'] = True
|
|
667
|
+
endpoint_stats[key]['count_5xx'] += 1
|
|
668
|
+
if test.get('has_skip', False):
|
|
669
|
+
endpoint_stats[key]['skipped_tests'].append({
|
|
670
|
+
'name': test.get('name', ''),
|
|
671
|
+
'reason': test.get('skip_reason'),
|
|
672
|
+
'env': test.get('skip_env')
|
|
673
|
+
})
|
|
674
|
+
matched = True
|
|
675
|
+
break
|
|
676
|
+
|
|
677
|
+
if not matched:
|
|
678
|
+
key = f"{test_method} {test_path}"
|
|
679
|
+
endpoint_stats[key]['test_count'] += 1
|
|
680
|
+
endpoint_stats[key]['status_codes'].update(test['status_codes'])
|
|
681
|
+
endpoint_stats[key]['body_assertions'] += test['body_assertions']
|
|
682
|
+
if test['body_assertions'] > 0:
|
|
683
|
+
endpoint_stats[key]['has_body_test'] = True
|
|
684
|
+
if test['request_body_hash']:
|
|
685
|
+
endpoint_stats[key]['request_body_variations'].add(test['request_body_hash'])
|
|
686
|
+
endpoint_stats[key]['query_params'].update(test.get('query_params', set()))
|
|
687
|
+
for param, values in test.get('query_param_values', {}).items():
|
|
688
|
+
endpoint_stats[key]['query_param_values'][param].update(values)
|
|
689
|
+
endpoint_stats[key]['request_body_fields'].update(test.get('request_body_fields', set()))
|
|
690
|
+
if test.get('has_oneof', False):
|
|
691
|
+
endpoint_stats[key]['has_oneof'] = True
|
|
692
|
+
endpoint_stats[key]['oneof_count'] += 1
|
|
693
|
+
if test.get('has_5xx', False):
|
|
694
|
+
endpoint_stats[key]['has_5xx'] = True
|
|
695
|
+
endpoint_stats[key]['count_5xx'] += 1
|
|
696
|
+
if test.get('has_skip', False):
|
|
697
|
+
endpoint_stats[key]['skipped_tests'].append({
|
|
698
|
+
'name': test.get('name', ''),
|
|
699
|
+
'reason': test.get('skip_reason'),
|
|
700
|
+
'env': test.get('skip_env')
|
|
701
|
+
})
|
|
702
|
+
|
|
703
|
+
# Calculate metrics
|
|
704
|
+
total_tests = sum(s['test_count'] for s in endpoint_stats.values())
|
|
705
|
+
total_body = sum(s['body_assertions'] for s in endpoint_stats.values())
|
|
706
|
+
tested_ep = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('test_count', 0) > 0)
|
|
707
|
+
well_tested = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('test_count', 0) >= 3)
|
|
708
|
+
with_body = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_body_test', False))
|
|
709
|
+
not_tested = len(endpoints) - tested_ep
|
|
710
|
+
|
|
711
|
+
# Success path tested (has at least one 2xx status code)
|
|
712
|
+
def has_success_code(status_codes):
|
|
713
|
+
"""Check if any status code is a 2xx success code."""
|
|
714
|
+
return any(str(code).startswith('2') for code in status_codes)
|
|
715
|
+
|
|
716
|
+
def has_401_code(status_codes):
|
|
717
|
+
"""Check if any status code is 401 (unauthorized)."""
|
|
718
|
+
return '401' in [str(code) for code in status_codes]
|
|
719
|
+
|
|
720
|
+
success_tested = sum(1 for ep in endpoints if has_success_code(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('status_codes', set())))
|
|
721
|
+
error_only = tested_ep - success_tested
|
|
722
|
+
|
|
723
|
+
# 401 coverage (endpoints with auth tests)
|
|
724
|
+
auth_tested = sum(1 for ep in endpoints if has_401_code(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('status_codes', set())))
|
|
725
|
+
auth_pct = (auth_tested / len(endpoints) * 100) if endpoints else 0
|
|
726
|
+
|
|
727
|
+
# oneOf (weak assertion) metrics
|
|
728
|
+
oneof_endpoints = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_oneof', False))
|
|
729
|
+
total_oneof_tests = sum(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('oneof_count', 0) for ep in endpoints)
|
|
730
|
+
|
|
731
|
+
# 5xx assertion metrics (bad practice)
|
|
732
|
+
endpoints_with_5xx = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_5xx', False))
|
|
733
|
+
total_5xx_tests = sum(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('count_5xx', 0) for ep in endpoints)
|
|
734
|
+
|
|
735
|
+
# Skip metrics
|
|
736
|
+
total_skipped_tests = len(all_skipped_tests)
|
|
737
|
+
skips_with_reason = sum(1 for s in all_skipped_tests if s.get('reason'))
|
|
738
|
+
skips_without_reason = total_skipped_tests - skips_with_reason
|
|
739
|
+
skips_by_env = defaultdict(list)
|
|
740
|
+
for skip in all_skipped_tests:
|
|
741
|
+
env = skip.get('env', 'unknown')
|
|
742
|
+
skips_by_env[env].append(skip)
|
|
743
|
+
|
|
744
|
+
# Query params metrics
|
|
745
|
+
total_defined_params = 0
|
|
746
|
+
total_tested_params = 0
|
|
747
|
+
endpoints_with_params = []
|
|
748
|
+
for ep in endpoints:
|
|
749
|
+
defined_params = ep.get('defined_params', set())
|
|
750
|
+
if defined_params:
|
|
751
|
+
total_defined_params += len(defined_params)
|
|
752
|
+
key = f"{ep['method']} {ep['path']}"
|
|
753
|
+
stats = endpoint_stats.get(key, {})
|
|
754
|
+
tested_params = stats.get('query_params', set())
|
|
755
|
+
tested_normalized = set(p.lower().replace('_', '') for p in tested_params)
|
|
756
|
+
covered = len(defined_params.intersection(tested_normalized))
|
|
757
|
+
total_tested_params += covered
|
|
758
|
+
|
|
759
|
+
param_values = stats.get('query_param_values', {})
|
|
760
|
+
if tested_params:
|
|
761
|
+
endpoints_with_params.append({
|
|
762
|
+
'method': ep['method'],
|
|
763
|
+
'path': ep['path'],
|
|
764
|
+
'params': tested_params,
|
|
765
|
+
'param_values': param_values,
|
|
766
|
+
'total_values': sum(len(v) for v in param_values.values())
|
|
767
|
+
})
|
|
768
|
+
|
|
769
|
+
# Include params metrics
|
|
770
|
+
# Define expected include values per endpoint type
|
|
771
|
+
# - List endpoints: configs, oauth_configs_list, secure_iparams
|
|
772
|
+
# - Configuration endpoints: secure_iparams, oauth_iparams, extension_type
|
|
773
|
+
INCLUDE_VALUES_LIST = {'configs', 'oauth_configs_list', 'secure_iparams'}
|
|
774
|
+
INCLUDE_VALUES_CONFIG = {'secure_iparams', 'oauth_iparams', 'extension_type'}
|
|
775
|
+
|
|
776
|
+
endpoints_with_include = []
|
|
777
|
+
total_expected_include_values = 0
|
|
778
|
+
total_tested_include_values = 0
|
|
779
|
+
|
|
780
|
+
for ep in endpoints:
|
|
781
|
+
if ep.get('has_include_param', False):
|
|
782
|
+
key = f"{ep['method']} {ep['path']}"
|
|
783
|
+
stats = endpoint_stats.get(key, {})
|
|
784
|
+
tested_params = stats.get('query_params', set())
|
|
785
|
+
param_values = stats.get('query_param_values', {})
|
|
786
|
+
|
|
787
|
+
include_tested = any('include' in p.lower() for p in tested_params)
|
|
788
|
+
include_values = set()
|
|
789
|
+
include_variations = 0
|
|
790
|
+
|
|
791
|
+
for param, values in param_values.items():
|
|
792
|
+
if 'include' in param.lower():
|
|
793
|
+
for v in values:
|
|
794
|
+
for single_val in v.split(','):
|
|
795
|
+
include_values.add(single_val.strip())
|
|
796
|
+
include_variations = len(values)
|
|
797
|
+
|
|
798
|
+
# Determine expected values based on endpoint type
|
|
799
|
+
if 'configurations' in ep['path'].lower():
|
|
800
|
+
expected_values = INCLUDE_VALUES_CONFIG
|
|
801
|
+
else:
|
|
802
|
+
expected_values = INCLUDE_VALUES_LIST
|
|
803
|
+
|
|
804
|
+
tested_expected = include_values & expected_values
|
|
805
|
+
missing_values = expected_values - include_values
|
|
806
|
+
|
|
807
|
+
total_expected_include_values += len(expected_values)
|
|
808
|
+
total_tested_include_values += len(tested_expected)
|
|
809
|
+
|
|
810
|
+
endpoints_with_include.append({
|
|
811
|
+
'method': ep['method'],
|
|
812
|
+
'path': ep['path'],
|
|
813
|
+
'tested': include_tested,
|
|
814
|
+
'values': include_values,
|
|
815
|
+
'expected_values': expected_values,
|
|
816
|
+
'tested_expected': tested_expected,
|
|
817
|
+
'missing_values': missing_values,
|
|
818
|
+
'variations': include_variations
|
|
819
|
+
})
|
|
820
|
+
|
|
821
|
+
# Request body metrics
|
|
822
|
+
endpoints_with_body = []
|
|
823
|
+
for ep in endpoints:
|
|
824
|
+
if ep['method'] in ['POST', 'PUT', 'PATCH']:
|
|
825
|
+
key = f"{ep['method']} {ep['path']}"
|
|
826
|
+
stats = endpoint_stats.get(key, {})
|
|
827
|
+
fields = stats.get('request_body_fields', set())
|
|
828
|
+
variations = len(stats.get('request_body_variations', set()))
|
|
829
|
+
|
|
830
|
+
if fields or variations > 0:
|
|
831
|
+
endpoints_with_body.append({
|
|
832
|
+
'method': ep['method'],
|
|
833
|
+
'path': ep['path'],
|
|
834
|
+
'fields': fields,
|
|
835
|
+
'variations': variations
|
|
836
|
+
})
|
|
837
|
+
|
|
838
|
+
# Calculate percentages
|
|
839
|
+
tested_pct = (tested_ep / len(endpoints) * 100) if endpoints else 0
|
|
840
|
+
well_tested_pct = (well_tested / len(endpoints) * 100) if endpoints else 0
|
|
841
|
+
body_pct = (with_body / len(endpoints) * 100) if endpoints else 0
|
|
842
|
+
qp_pct = (total_tested_params / total_defined_params * 100) if total_defined_params > 0 else 0
|
|
843
|
+
success_pct = (success_tested / len(endpoints) * 100) if endpoints else 0
|
|
844
|
+
|
|
845
|
+
# Include param coverage - now based on individual values tested vs expected
|
|
846
|
+
include_tested_count = sum(1 for ep in endpoints_with_include if ep['tested'])
|
|
847
|
+
include_total_count = len(endpoints_with_include)
|
|
848
|
+
include_pct = (total_tested_include_values / total_expected_include_values * 100) if total_expected_include_values > 0 else 100
|
|
849
|
+
include_endpoints_pct = (include_tested_count / include_total_count * 100) if include_total_count > 0 else 100
|
|
850
|
+
|
|
851
|
+
# Categorize endpoints for gaps analysis
|
|
852
|
+
gaps_no_tests = []
|
|
853
|
+
gaps_no_2xx = []
|
|
854
|
+
gaps_weak_assertions = []
|
|
855
|
+
gaps_5xx_assertions = []
|
|
856
|
+
|
|
857
|
+
for ep in endpoints:
|
|
858
|
+
key = f"{ep['method']} {ep['path']}"
|
|
859
|
+
stats = endpoint_stats.get(key, {'test_count': 0, 'status_codes': set()})
|
|
860
|
+
|
|
861
|
+
if stats['test_count'] == 0:
|
|
862
|
+
gaps_no_tests.append(ep)
|
|
863
|
+
elif not has_success_code(stats.get('status_codes', set())):
|
|
864
|
+
gaps_no_2xx.append(ep)
|
|
865
|
+
|
|
866
|
+
if stats.get('has_oneof', False):
|
|
867
|
+
gaps_weak_assertions.append({**ep, 'oneof_count': stats.get('oneof_count', 0)})
|
|
868
|
+
|
|
869
|
+
if stats.get('has_5xx', False):
|
|
870
|
+
gaps_5xx_assertions.append({**ep, 'count_5xx': stats.get('count_5xx', 0)})
|
|
871
|
+
|
|
872
|
+
def get_status_class(pct):
|
|
873
|
+
if pct >= 80: return 'success'
|
|
874
|
+
if pct >= 50: return 'warning'
|
|
875
|
+
return 'error'
|
|
876
|
+
|
|
877
|
+
def get_health_status(success_pct, well_tested_pct, error_only, oneof_count):
|
|
878
|
+
"""Determine overall API test health."""
|
|
879
|
+
if success_pct >= 100 and well_tested_pct >= 80 and error_only == 0 and oneof_count == 0:
|
|
880
|
+
return ('excellent', '🟢', 'Excellent', 'All endpoints have 2xx tests with strong assertions')
|
|
881
|
+
elif success_pct >= 90 and well_tested_pct >= 60 and error_only <= 2:
|
|
882
|
+
return ('good', '🟡', 'Good', 'Most endpoints well tested, minor gaps exist')
|
|
883
|
+
elif success_pct >= 70 and well_tested_pct >= 40:
|
|
884
|
+
return ('fair', '🟠', 'Fair', 'Moderate coverage, needs improvement')
|
|
885
|
+
else:
|
|
886
|
+
return ('poor', '🔴', 'Needs Work', 'Significant testing gaps exist')
|
|
887
|
+
|
|
888
|
+
health_class, health_icon, health_label, health_desc = get_health_status(success_pct, well_tested_pct, error_only, total_oneof_tests)
|
|
889
|
+
|
|
890
|
+
# Build HTML content sections
|
|
891
|
+
def card(val, lbl, sub=None, status=None):
|
|
892
|
+
"""Build a summary card."""
|
|
893
|
+
cls = f'summary-card{" " + esc(str(status)) if status else ""}'
|
|
894
|
+
sub_html = f'<div class="sub-value">{esc(str(sub))}</div>' if sub else ''
|
|
895
|
+
return f'<div class="{cls}"><div class="value">{esc(str(val))}</div><div class="label">{esc(str(lbl))}</div>{sub_html}</div>'
|
|
896
|
+
|
|
897
|
+
def coverage_item(name, pct, details):
|
|
898
|
+
"""Build a coverage metric item."""
|
|
899
|
+
status = 'success' if pct >= 80 else 'warning' if pct >= 50 else 'error'
|
|
900
|
+
safe_pct = min(float(pct), 100)
|
|
901
|
+
return f'''<div class="coverage-item">
|
|
902
|
+
<div class="header"><span class="metric-name">{esc(str(name))}</span><span class="metric-value" style="color: var(--color-{status});">{safe_pct:.1f}%</span></div>
|
|
903
|
+
<div class="progress-bar"><div class="progress-fill {status}" style="width: {safe_pct:.1f}%;"></div></div>
|
|
904
|
+
<div class="details">{esc(str(details))}</div>
|
|
905
|
+
</div>'''
|
|
906
|
+
|
|
907
|
+
def gaps_section(title, items, warning=False, count_key=None):
|
|
908
|
+
"""Build a gaps section."""
|
|
909
|
+
if not items: return ''
|
|
910
|
+
cls = 'gaps-section warning' if warning else 'gaps-section'
|
|
911
|
+
def gap_line(ep):
|
|
912
|
+
method = esc(str(ep["method"]))
|
|
913
|
+
path = esc(str(ep["path"]))
|
|
914
|
+
count_suffix = f' ({esc(str(ep.get(count_key, 0)))} tests)' if count_key else ''
|
|
915
|
+
return f'<div class="gap-item"><span class="method method-{method}">{method}</span> {path}{count_suffix}</div>'
|
|
916
|
+
items_html = ''.join(gap_line(ep) for ep in items)
|
|
917
|
+
return f'<div class="{cls}"><div class="title">{esc(str(title))}</div><div class="gap-list">{items_html}</div></div>'
|
|
918
|
+
|
|
919
|
+
def collapsible(title, count, headers, rows):
|
|
920
|
+
"""Build a collapsible table section."""
|
|
921
|
+
if not rows: return ''
|
|
922
|
+
h = ''.join(f'<th style="{hdr.get("style","")}">{hdr["label"]}</th>' for hdr in headers)
|
|
923
|
+
r = ''.join(rows)
|
|
924
|
+
return f'''<details><summary>{title}<span class="count" style="background: var(--color-border); padding: 0.125rem 0.5rem; border-radius: 10px; font-size: 0.75rem; font-weight: 500;">{count}</span></summary>
|
|
925
|
+
<div class="content"><table><thead><tr>{h}</tr></thead><tbody>{r}</tbody></table></div></details>'''
|
|
926
|
+
|
|
927
|
+
# Compact summary - single row with key metrics
|
|
928
|
+
def metric(label, value, detail=None, status=None):
|
|
929
|
+
"""Inline metric span."""
|
|
930
|
+
color = f'var(--color-{status})' if status else 'var(--color-primary)'
|
|
931
|
+
det = f' <span style="color:var(--color-muted);font-size:0.75rem;">({detail})</span>' if detail else ''
|
|
932
|
+
return f'<span style="margin-right:1.5rem;"><strong style="color:{color};">{value}</strong> {label}{det}</span>'
|
|
933
|
+
|
|
934
|
+
issues = []
|
|
935
|
+
if not_tested > 0: issues.append(f'{not_tested} untested')
|
|
936
|
+
if total_5xx_tests > 0: issues.append(f'{total_5xx_tests} 5xx')
|
|
937
|
+
if total_oneof_tests > 0: issues.append(f'{total_oneof_tests} weak')
|
|
938
|
+
if skips_without_reason > 0: issues.append(f'{skips_without_reason} undoc')
|
|
939
|
+
issues_str = ', '.join(issues) if issues else None
|
|
940
|
+
|
|
941
|
+
summary_html = f'''<div class="section">
|
|
942
|
+
<div style="display:flex;align-items:center;justify-content:space-between;flex-wrap:wrap;gap:1rem;padding:1rem 1.25rem;background:var(--color-card);border:1px solid var(--color-border);border-radius:var(--radius-sm);box-shadow:var(--shadow-sm);">
|
|
943
|
+
<div style="display:flex;align-items:center;gap:0.75rem;">
|
|
944
|
+
<span style="font-size:1.5rem;">{health_icon}</span>
|
|
945
|
+
<span style="font-weight:600;color:var(--color-heading);">{health_label}</span>
|
|
946
|
+
</div>
|
|
947
|
+
<div style="display:flex;flex-wrap:wrap;align-items:center;">
|
|
948
|
+
{metric('endpoints', len(endpoints), f'{total_tests} tests')}
|
|
949
|
+
{metric('2xx', f'{success_pct:.0f}%', f'{success_tested}/{len(endpoints)}', 'success' if success_pct >= 100 else 'warning' if success_pct >= 80 else 'error')}
|
|
950
|
+
{metric('well tested', f'{well_tested_pct:.0f}%', f'{well_tested}/{len(endpoints)}', 'success' if well_tested_pct >= 100 else 'warning' if well_tested_pct >= 80 else 'error')}
|
|
951
|
+
{metric('body validated', f'{body_pct:.0f}%', f'{with_body}/{len(endpoints)}', 'success' if body_pct >= 100 else 'warning' if body_pct >= 80 else 'error')}
|
|
952
|
+
{metric('query params', f'{qp_pct:.0f}%', f'{total_tested_params}/{total_defined_params}', 'success' if qp_pct >= 100 else 'warning' if qp_pct >= 80 else 'error')}
|
|
953
|
+
{f'<span style="color:var(--color-error);font-weight:500;">⚠ {issues_str}</span>' if issues_str else ''}
|
|
954
|
+
</div>
|
|
955
|
+
</div>
|
|
956
|
+
</div>'''
|
|
957
|
+
|
|
958
|
+
# No separate coverage breakdown needed - all in summary
|
|
959
|
+
coverage_html = ''
|
|
960
|
+
|
|
961
|
+
# Gaps analysis
|
|
962
|
+
gaps_html = ''
|
|
963
|
+
if gaps_no_tests or gaps_no_2xx or gaps_weak_assertions or gaps_5xx_assertions:
|
|
964
|
+
gaps_html = '<div class="section"><div class="section-title">⚠️ Gaps Analysis</div>'
|
|
965
|
+
gaps_html += gaps_section(f'❌ No Tests ({len(gaps_no_tests)} endpoints)', gaps_no_tests)
|
|
966
|
+
gaps_html += gaps_section(f'❌ No 2xx Tests ({len(gaps_no_2xx)} endpoints)', gaps_no_2xx)
|
|
967
|
+
gaps_html += gaps_section(f'⚠️ Weak Assertions (oneOf) - {len(gaps_weak_assertions)} endpoints', gaps_weak_assertions, warning=True, count_key='oneof_count')
|
|
968
|
+
gaps_html += gaps_section(f'🔴 5xx Assertions (bad practice) - {len(gaps_5xx_assertions)} endpoints', gaps_5xx_assertions, count_key='count_5xx')
|
|
969
|
+
gaps_html += '</div>'
|
|
970
|
+
|
|
971
|
+
# Endpoint details table
|
|
972
|
+
def endpoint_row(ep, stats):
|
|
973
|
+
tc = stats.get('test_count', 0)
|
|
974
|
+
codes = stats.get('status_codes', set())
|
|
975
|
+
codes_str = ', '.join(sorted(codes, key=lambda x: int(x) if x.isdigit() else 0)) if codes else '—'
|
|
976
|
+
body = stats.get('body_assertions', 0) or '—'
|
|
977
|
+
has_2xx = any(str(c).startswith('2') for c in codes)
|
|
978
|
+
success_icon = '✅' if has_2xx else ('❌' if tc > 0 else '—')
|
|
979
|
+
defined = ep.get('defined_params', set())
|
|
980
|
+
tested = stats.get('query_params', set())
|
|
981
|
+
qp_str = '—' if not defined else f'{len(defined.intersection(set(p.lower().replace("_","") for p in tested)))}/{len(defined)}' if tested else f'0/{len(defined)}'
|
|
982
|
+
oneof = stats.get('oneof_count', 0)
|
|
983
|
+
weak = f'⚠️ {oneof}' if oneof > 0 else '—'
|
|
984
|
+
icon, style = ('❌', 'background:#fef2f2;') if tc == 0 else ('⚠️', '') if tc < 3 else ('✅', '')
|
|
985
|
+
return f'<tr style="{style}"><td class="status-icon">{icon}</td><td><span class="method method-{ep["method"]}">{ep["method"]}</span></td><td class="left"><span class="path">{ep["path"]}</span></td><td>{tc}</td><td class="status-icon">{success_icon}</td><td>{codes_str}</td><td>{body}</td><td>{qp_str}</td><td>{weak}</td></tr>'
|
|
986
|
+
|
|
987
|
+
endpoint_rows = [endpoint_row(ep, endpoint_stats.get(f"{ep['method']} {ep['path']}", {})) for ep in endpoints]
|
|
988
|
+
details_html = f'''<div class="section"><div class="section-title">Endpoint Details<span class="count">{len(endpoints)} endpoints</span></div>
|
|
989
|
+
<table><thead><tr><th style="width:40px;">Status</th><th style="width:70px;">Method</th><th style="text-align:left;">Endpoint</th><th style="width:50px;">Tests</th><th style="width:50px;">2xx</th><th style="width:120px;">Status Codes Asserted</th><th style="width:80px;">Response Assertions</th><th style="width:80px;">Query Params Coverage</th><th style="width:80px;">Weak Assertions</th></tr></thead>
|
|
990
|
+
<tbody>{''.join(endpoint_rows)}</tbody></table></div>'''
|
|
991
|
+
|
|
992
|
+
# Collapsible detail sections
|
|
993
|
+
qp_rows = [f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td><code>{", ".join(sorted(ep["params"]))}</code></td><td>{ep["total_values"]}</td></tr>' for ep in endpoints_with_params]
|
|
994
|
+
qp_html = collapsible('Query Parameter Details', f'{len(endpoints_with_params)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Parameters Tested'},{'label':'Value Variations'}], qp_rows)
|
|
995
|
+
|
|
996
|
+
body_rows = [f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td>{len(ep["fields"])} fields</td><td>{ep["variations"]}</td></tr>' for ep in endpoints_with_body]
|
|
997
|
+
body_html = collapsible('Request Body Coverage', f'{len(endpoints_with_body)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Fields Tested'},{'label':'Variations'}], body_rows)
|
|
998
|
+
|
|
999
|
+
def include_row(ep):
|
|
1000
|
+
tc, ec = len(ep['tested_expected']), len(ep['expected_values'])
|
|
1001
|
+
pct = (tc / ec * 100) if ec > 0 else 0
|
|
1002
|
+
col = 'var(--color-success)' if pct >= 100 else 'var(--color-warning)' if pct >= 50 else 'var(--color-error)'
|
|
1003
|
+
exp = ', '.join(sorted(ep['expected_values'])) or 'None'
|
|
1004
|
+
vals = ', '.join(sorted(ep['values'])) or 'None'
|
|
1005
|
+
miss = ', '.join(sorted(ep['missing_values'])) or '—'
|
|
1006
|
+
miss_col = 'var(--color-error)' if ep['missing_values'] else 'var(--color-success)'
|
|
1007
|
+
return f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td style="color:{col};">{tc}/{ec} ({pct:.0f}%)</td><td><code>{exp}</code></td><td><code>{vals}</code></td><td style="color:{miss_col};"><code>{miss}</code></td></tr>'
|
|
1008
|
+
|
|
1009
|
+
include_rows = [include_row(ep) for ep in endpoints_with_include]
|
|
1010
|
+
include_html = collapsible('Include Parameter Details', f'{len(endpoints_with_include)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Coverage'},{'label':'Expected Values'},{'label':'Tested Values'},{'label':'Missing'}], include_rows)
|
|
1011
|
+
|
|
1012
|
+
skip_rows = [f'<tr><td class="left"><span class="path">{s["collection"]}: {s["name"]}</span></td><td>{s.get("env","unknown").upper()}</td><td class="left" style="color:{"var(--color-error)" if not s.get("reason") else "inherit"};">{s.get("reason") or "Missing reason"}</td></tr>' for s in all_skipped_tests]
|
|
1013
|
+
skip_html = collapsible('Environment-Conditional Skips', f'{total_skipped_tests} tests', [{'label':'Test','style':'text-align:left;'},{'label':'Skipped In'},{'label':'Reason','style':'text-align:left;'}], skip_rows) if total_skipped_tests > 0 else ''
|
|
1014
|
+
|
|
1015
|
+
# Assemble full HTML
|
|
1016
|
+
content = summary_html + coverage_html + gaps_html + details_html + qp_html + body_html + include_html + skip_html
|
|
1017
|
+
|
|
1018
|
+
if USE_HELPERS:
|
|
1019
|
+
html = get_html_template(
|
|
1020
|
+
title='API Test Coverage Report',
|
|
1021
|
+
logo='DP Apps',
|
|
1022
|
+
content=content,
|
|
1023
|
+
health_badge={'class': health_class, 'icon': health_icon, 'label': health_label},
|
|
1024
|
+
footer_text=f'API Test Coverage Report • Generated by API Coverage Matrix Generator • {total_tests} test cases analyzed'
|
|
1025
|
+
)
|
|
1026
|
+
else:
|
|
1027
|
+
# Fallback: manual HTML construction
|
|
1028
|
+
html = f'''<!DOCTYPE html>
|
|
1029
|
+
<html lang="en">
|
|
1030
|
+
<head>
|
|
1031
|
+
<meta charset="UTF-8">
|
|
1032
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
1033
|
+
<title>API Coverage Report | DP Apps</title>
|
|
1034
|
+
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
|
|
1035
|
+
<style>{SHARED_CSS if SHARED_CSS else "/* Error: shared CSS library not loaded */"}</style>
|
|
1036
|
+
</head>
|
|
1037
|
+
<body>
|
|
1038
|
+
<div class="container">
|
|
1039
|
+
<div class="header">
|
|
1040
|
+
<div class="header-left">
|
|
1041
|
+
<div class="logo">DP Apps</div>
|
|
1042
|
+
<h1>API Test Coverage Report</h1>
|
|
1043
|
+
<p class="timestamp">Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}</p>
|
|
1044
|
+
</div>
|
|
1045
|
+
<div class="health-badge {health_class}">{health_icon} {health_label}</div>
|
|
1046
|
+
</div>
|
|
1047
|
+
{content}
|
|
1048
|
+
<footer>API Test Coverage Report • Generated by API Coverage Matrix Generator • {total_tests} test cases analyzed</footer>
|
|
1049
|
+
</div>
|
|
1050
|
+
</body>
|
|
1051
|
+
</html>'''
|
|
1052
|
+
|
|
1053
|
+
# Write HTML
|
|
1054
|
+
with open(HTML_OUTPUT_FILE, 'w', encoding='utf-8') as f:
|
|
1055
|
+
f.write(html)
|
|
1056
|
+
|
|
1057
|
+
# Also generate JSON summary for consolidated reports
|
|
1058
|
+
json_output_file = HTML_OUTPUT_FILE.replace('.html', '.json')
|
|
1059
|
+
|
|
1060
|
+
# Helper to check if endpoint has 2xx coverage
|
|
1061
|
+
def ep_has_2xx(ep):
|
|
1062
|
+
key = f"{ep['method']} {ep['path']}"
|
|
1063
|
+
stats = endpoint_stats.get(key, {})
|
|
1064
|
+
return has_success_code(stats.get('status_codes', set()))
|
|
1065
|
+
|
|
1066
|
+
# Calculate by-method stats
|
|
1067
|
+
method_stats = {}
|
|
1068
|
+
gaps = []
|
|
1069
|
+
for ep in endpoints:
|
|
1070
|
+
m = ep['method']
|
|
1071
|
+
if m not in method_stats:
|
|
1072
|
+
method_stats[m] = {'tested': 0, 'total': 0}
|
|
1073
|
+
method_stats[m]['total'] += 1
|
|
1074
|
+
if ep_has_2xx(ep):
|
|
1075
|
+
method_stats[m]['tested'] += 1
|
|
1076
|
+
else:
|
|
1077
|
+
gaps.append({
|
|
1078
|
+
'method': ep['method'],
|
|
1079
|
+
'path': ep['path'],
|
|
1080
|
+
'controller': ep['controller']
|
|
1081
|
+
})
|
|
1082
|
+
|
|
1083
|
+
coverage_summary = {
|
|
1084
|
+
'coverage_percent': round(success_pct, 1),
|
|
1085
|
+
'tested': success_tested,
|
|
1086
|
+
'total': len(endpoints),
|
|
1087
|
+
'well_tested_percent': round(well_tested_pct, 1),
|
|
1088
|
+
'query_params_percent': round(qp_pct, 1),
|
|
1089
|
+
'gaps': gaps[:20],
|
|
1090
|
+
'by_method': method_stats,
|
|
1091
|
+
'full_report': os.path.basename(HTML_OUTPUT_FILE)
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
with open(json_output_file, 'w', encoding='utf-8') as f:
|
|
1095
|
+
json.dump(coverage_summary, f, indent=2)
|
|
1096
|
+
|
|
1097
|
+
print(f"\nEndpoints in Code: {len(endpoints)}", flush=True)
|
|
1098
|
+
print(f"Total Test Cases: {total_tests}", flush=True)
|
|
1099
|
+
print(f"Success Path (2xx) Tested: {success_pct:.1f}% ({success_tested}/{len(endpoints)})", flush=True)
|
|
1100
|
+
print(f"Well Tested (≥3 tests): {well_tested_pct:.1f}% ({well_tested}/{len(endpoints)})", flush=True)
|
|
1101
|
+
print(f"Response Body Validated: {body_pct:.1f}% ({with_body}/{len(endpoints)})", flush=True)
|
|
1102
|
+
print(f"Query Params Tested: {qp_pct:.1f}% ({total_tested_params}/{total_defined_params})", flush=True)
|
|
1103
|
+
if include_total_count > 0:
|
|
1104
|
+
print(f"Include Params Tested: {include_pct:.1f}% ({total_tested_include_values}/{total_expected_include_values} values across {include_total_count} endpoints)", flush=True)
|
|
1105
|
+
if error_only > 0:
|
|
1106
|
+
print(f"⚠️ Error-Only Tests (no 2xx): {error_only} endpoints", flush=True)
|
|
1107
|
+
if total_oneof_tests > 0:
|
|
1108
|
+
print(f"⚠️ oneOf Tests (weak assertions): {total_oneof_tests} tests in {oneof_endpoints} endpoints", flush=True)
|
|
1109
|
+
|
|
1110
|
+
if total_skipped_tests > 0:
|
|
1111
|
+
print(f"⏭️ Environment-conditional skips: {total_skipped_tests} tests", flush=True)
|
|
1112
|
+
for env, skips in sorted(skips_by_env.items()):
|
|
1113
|
+
print(f" - {env}: {len(skips)} tests", flush=True)
|
|
1114
|
+
if skips_without_reason > 0:
|
|
1115
|
+
print(f"❌ Undocumented skips: {skips_without_reason} tests (add [SKIP] Reason: ... | Env: ...)", flush=True)
|
|
1116
|
+
|
|
1117
|
+
if errors:
|
|
1118
|
+
print(f"\nWarnings ({len(errors)}):", flush=True)
|
|
1119
|
+
for err in errors[:5]:
|
|
1120
|
+
print(f" - {err}", flush=True)
|
|
1121
|
+
if len(errors) > 5:
|
|
1122
|
+
print(f" ... and {len(errors) - 5} more", flush=True)
|