@freshworks/shiftleft-tools 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +351 -0
- package/bin/shiftleft.js +95 -0
- package/package.json +57 -0
- package/src/commands/doctor.js +208 -0
- package/src/commands/init-postman.js +298 -0
- package/src/commands/init-rules.js +78 -0
- package/src/commands/link.js +172 -0
- package/src/commands/protect.js +61 -0
- package/src/commands/run-tests.js +182 -0
- package/src/commands/setup-pipeline.js +209 -0
- package/src/commands/update.js +203 -0
- package/src/index.js +4 -0
- package/src/utils/copy-tree.js +98 -0
- package/src/utils/gitignore.js +26 -0
- package/src/utils/logger.js +9 -0
- package/src/utils/manifest.js +145 -0
- package/src/utils/stack.js +80 -0
- package/src/utils/template.js +135 -0
- package/templates/AGENTS.md +109 -0
- package/templates/CLAUDE.md +3 -0
- package/templates/jenkins/Jenkinsfile-java.groovy +432 -0
- package/templates/jenkins/Jenkinsfile-node.groovy +450 -0
- package/templates/postman/.husky/pre-commit +19 -0
- package/templates/postman/.prettierrc.json +5 -0
- package/templates/postman/README.md.ejs +147 -0
- package/templates/postman/collections/01-core.json.ejs +91 -0
- package/templates/postman/config/local.json.ejs +12 -0
- package/templates/postman/config/staging.json.ejs +26 -0
- package/templates/postman/environments/local.postman_environment.json.ejs +31 -0
- package/templates/postman/environments/staging.postman_environment.json.ejs +31 -0
- package/templates/postman/gitignore +16 -0
- package/templates/postman/npmrc +31 -0
- package/templates/postman/package.json.ejs +66 -0
- package/templates/postman/run-all-shim.sh +16 -0
- package/templates/postman/scripts/auth/generate-jwt.sh +113 -0
- package/templates/postman/scripts/auth/get-issuer-secret.sh +140 -0
- package/templates/postman/scripts/infra/start-mocks.sh +138 -0
- package/templates/postman/scripts/infra/stop-mocks.sh +43 -0
- package/templates/postman/scripts/lib/api_coverage.py +1122 -0
- package/templates/postman/scripts/lib/cleanup-reports.sh +101 -0
- package/templates/postman/scripts/lib/cleanup-stryker.sh +44 -0
- package/templates/postman/scripts/lib/report_combined.py +527 -0
- package/templates/postman/scripts/lib/report_consolidated.py +363 -0
- package/templates/postman/scripts/lib/report_generator.py +121 -0
- package/templates/postman/scripts/lib/report_migration.py +156 -0
- package/templates/postman/scripts/lib/report_mutation.py +110 -0
- package/templates/postman/scripts/lib/report_unit.py +353 -0
- package/templates/postman/scripts/lib/report_utils.py +973 -0
- package/templates/postman/scripts/report-generators/generate-consolidated-report.sh +445 -0
- package/templates/postman/scripts/report-generators/java-api-coverage-matrix.sh +257 -0
- package/templates/postman/scripts/report-generators/mutation-report.sh +672 -0
- package/templates/postman/scripts/report-generators/node-api-coverage-matrix.sh +167 -0
- package/templates/postman/scripts/report-generators/stage-report-artifacts.sh +27 -0
- package/templates/postman/scripts/run-all.sh +452 -0
- package/templates/postman/scripts/runners/run-mutation-tests.sh +113 -0
- package/templates/postman/scripts/runners/run-tests-local.sh +936 -0
- package/templates/postman/scripts/runners/run-tests-staging.sh +741 -0
- package/templates/postman-node/README.md.ejs +26 -0
- package/templates/postman-node/collections/crud/01-bootstrap.json.ejs +34 -0
- package/templates/postman-node/config/local.json.ejs +46 -0
- package/templates/postman-node/config/staging.json.ejs +31 -0
- package/templates/postman-node/local.test.env.ejs +3 -0
- package/templates/postman-node/mocks/external.js +14 -0
- package/templates/postman-node/package.json.ejs +39 -0
- package/templates/postman-node/requirements.txt +1 -0
- package/templates/postman-node/scripts/database/cleanup-mysql.sh +12 -0
- package/templates/postman-node/scripts/database/run-migrations.js +29 -0
- package/templates/postman-node/scripts/database/start-mysql.sh +34 -0
- package/templates/postman-node/scripts/database/wait-for-mysql.sh +36 -0
- package/templates/postman-node/scripts/lib/api_coverage_node.py +1137 -0
- package/templates/postman-node/scripts/lib/fetch-jwt.sh +86 -0
- package/templates/postman-node/scripts/lib/run-newman.sh +104 -0
- package/templates/postman-node/scripts/lib/setup-database.sh +55 -0
- package/templates/postman-node/scripts/lib/start-app.sh +48 -0
- package/templates/postman-node/scripts/lib/utils.sh +114 -0
- package/templates/postman-node/scripts/report-generators/stage-report-artifacts.sh +26 -0
- package/templates/postman-node/scripts/run-all.sh +303 -0
- package/templates/postman-node/scripts/runners/run-tests.sh +123 -0
- package/templates/postman-node/scripts/setup-mocks.js.ejs +29 -0
- package/templates/postman-node/stryker.config.js.ejs +51 -0
- package/templates/rules/local-test-setup.mdc +420 -0
- package/templates/rules/testing-node.mdc +66 -0
- package/templates/rules/testing.mdc +248 -0
- package/templates/skills/_shared/postman-standards.md +380 -0
- package/templates/skills/enhance-test-pipeline/SKILL-java.md +483 -0
- package/templates/skills/enhance-test-pipeline/SKILL-node.md +431 -0
- package/templates/skills/enhance-test-pipeline/SKILL.md +9 -0
- package/templates/skills/review-test-suite/SKILL-java.md +137 -0
- package/templates/skills/review-test-suite/SKILL-node.md +78 -0
- package/templates/skills/review-test-suite/SKILL.md +9 -0
- package/templates/skills/run-test-suite/SKILL-java.md +186 -0
- package/templates/skills/run-test-suite/SKILL-node.md +191 -0
- package/templates/skills/run-test-suite/SKILL.md +9 -0
- package/templates/skills/setup-api-tests/SKILL-java.md +1094 -0
- package/templates/skills/setup-api-tests/SKILL-node.md +141 -0
- package/templates/skills/setup-api-tests/SKILL.md +9 -0
- package/templates/skills/setup-mutation-tests/SKILL-java.md +303 -0
- package/templates/skills/setup-mutation-tests/SKILL-node.md +408 -0
- package/templates/skills/setup-mutation-tests/SKILL.md +9 -0
- package/templates/skills/setup-test-pipeline/SKILL-java.md +454 -0
- package/templates/skills/setup-test-pipeline/SKILL-node.md +318 -0
- package/templates/skills/setup-test-pipeline/SKILL.md +9 -0
- package/templates/skills/write-api-tests/SKILL-java.md +115 -0
- package/templates/skills/write-api-tests/SKILL-node.md +83 -0
- package/templates/skills/write-api-tests/SKILL.md +9 -0
- package/templates/stryker.config.js +50 -0
|
@@ -0,0 +1,1137 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
API Coverage Matrix Generator for Node.js/Express services.
|
|
4
|
+
|
|
5
|
+
Usage:
|
|
6
|
+
python3 api_coverage.py --routes-dir DIR --postman-dir DIR --html-output FILE
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parse_args():
|
|
13
|
+
parser = argparse.ArgumentParser(description='API Coverage Matrix Generator')
|
|
14
|
+
parser.add_argument('--routes-dir', required=True, help='Path to Express routes/controllers directory')
|
|
15
|
+
parser.add_argument('--postman-dir', required=True, help='Path to Postman directory')
|
|
16
|
+
parser.add_argument('--html-output', required=True, help='Path for HTML output file')
|
|
17
|
+
return parser.parse_args()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import json
|
|
23
|
+
import sys
|
|
24
|
+
import html as _html
|
|
25
|
+
from collections import defaultdict
|
|
26
|
+
from datetime import datetime
|
|
27
|
+
|
|
28
|
+
esc = _html.escape
|
|
29
|
+
|
|
30
|
+
MAX_FILE_SIZE_MB = 50
|
|
31
|
+
|
|
32
|
+
_args = parse_args()
|
|
33
|
+
ROUTES_DIR = _args.routes_dir
|
|
34
|
+
POSTMAN_DIR = _args.postman_dir
|
|
35
|
+
HTML_OUTPUT_FILE = _args.html_output
|
|
36
|
+
|
|
37
|
+
endpoints = []
|
|
38
|
+
tested = []
|
|
39
|
+
errors = []
|
|
40
|
+
|
|
41
|
+
print(f"Scanning {ROUTES_DIR}", flush=True)
|
|
42
|
+
print(f"Scanning {POSTMAN_DIR}", flush=True)
|
|
43
|
+
|
|
44
|
+
def safe_read_file(filepath, max_size_mb=MAX_FILE_SIZE_MB):
|
|
45
|
+
"""Safely read a file with size limits."""
|
|
46
|
+
try:
|
|
47
|
+
file_size = os.path.getsize(filepath)
|
|
48
|
+
if file_size > max_size_mb * 1024 * 1024:
|
|
49
|
+
errors.append(f"Skipped {filepath}: file too large ({file_size // 1024 // 1024}MB)")
|
|
50
|
+
return None
|
|
51
|
+
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
|
|
52
|
+
return f.read()
|
|
53
|
+
except Exception as e:
|
|
54
|
+
errors.append(f"Error reading {filepath}: {str(e)}")
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
# =============================================================================
|
|
58
|
+
# ROUTER MOUNTING MAP BUILDER
|
|
59
|
+
# =============================================================================
|
|
60
|
+
|
|
61
|
+
router_mount_map = {}
|
|
62
|
+
router_files = {}
|
|
63
|
+
|
|
64
|
+
def build_router_mount_map():
|
|
65
|
+
"""
|
|
66
|
+
Build a map of router names to their mount paths using multiple passes.
|
|
67
|
+
|
|
68
|
+
This handles nested router mounting patterns like:
|
|
69
|
+
- rootRouter.use('/api/v2', v2Router)
|
|
70
|
+
- v2Router.use('/apps', appsRouter)
|
|
71
|
+
|
|
72
|
+
Result: appsRouter -> '/api/v2/apps'
|
|
73
|
+
"""
|
|
74
|
+
print("Building router mounting map...", flush=True)
|
|
75
|
+
|
|
76
|
+
for root, dirs, files in os.walk(ROUTES_DIR):
|
|
77
|
+
dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['node_modules', 'build', 'dist', 'coverage']]
|
|
78
|
+
|
|
79
|
+
for file in files:
|
|
80
|
+
if file.endswith(('.js', '.ts')) and not file.endswith('.test.js') and not file.endswith('.spec.js'):
|
|
81
|
+
filepath = os.path.join(root, file)
|
|
82
|
+
content = safe_read_file(filepath)
|
|
83
|
+
if content is None:
|
|
84
|
+
continue
|
|
85
|
+
|
|
86
|
+
router_decl_patterns = [
|
|
87
|
+
r'const\s+([a-zA-Z_][a-zA-Z0-9_]*)Router\s*=\s*Router\s*\(',
|
|
88
|
+
r'const\s+([a-zA-Z_][a-zA-Z0-9_]*)Router\s*=\s*new\s+Router\s*\(',
|
|
89
|
+
]
|
|
90
|
+
for pattern in router_decl_patterns:
|
|
91
|
+
for match in re.finditer(pattern, content):
|
|
92
|
+
router_name = match.group(1) + 'Router'
|
|
93
|
+
router_files[router_name] = filepath
|
|
94
|
+
|
|
95
|
+
mount_relationships = []
|
|
96
|
+
|
|
97
|
+
for root, dirs, files in os.walk(ROUTES_DIR):
|
|
98
|
+
dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['node_modules', 'build', 'dist', 'coverage']]
|
|
99
|
+
|
|
100
|
+
for file in files:
|
|
101
|
+
if file.endswith(('.js', '.ts')) and not file.endswith('.test.js') and not file.endswith('.spec.js'):
|
|
102
|
+
filepath = os.path.join(root, file)
|
|
103
|
+
content = safe_read_file(filepath)
|
|
104
|
+
if content is None:
|
|
105
|
+
continue
|
|
106
|
+
|
|
107
|
+
mount_pattern = r'([a-zA-Z_][a-zA-Z0-9_]*)Router\.use\(["\']([^"\']+)["\'],\s*(?:\[[^\]]*\],\s*)?([a-zA-Z_][a-zA-Z0-9_]*)Router'
|
|
108
|
+
for match in re.finditer(mount_pattern, content):
|
|
109
|
+
parent_router = match.group(1) + 'Router'
|
|
110
|
+
mount_path = match.group(2)
|
|
111
|
+
child_router = match.group(3) + 'Router'
|
|
112
|
+
mount_relationships.append((parent_router, mount_path, child_router))
|
|
113
|
+
|
|
114
|
+
mount_array_pattern = r'([a-zA-Z_][a-zA-Z0-9_]*)Router\.use\(["\']([^"\']+)["\'],\s*\[([^\]]+)\]'
|
|
115
|
+
for match in re.finditer(mount_array_pattern, content):
|
|
116
|
+
parent_router = match.group(1) + 'Router'
|
|
117
|
+
mount_path = match.group(2)
|
|
118
|
+
routers_str = match.group(3)
|
|
119
|
+
for router_match in re.finditer(r'([a-zA-Z_][a-zA-Z0-9_]*)Router', routers_str):
|
|
120
|
+
child_router = router_match.group(1) + 'Router'
|
|
121
|
+
mount_relationships.append((parent_router, mount_path, child_router))
|
|
122
|
+
|
|
123
|
+
mount_backtick_pattern = r'([a-zA-Z_][a-zA-Z0-9_]*)Router\.use\(`([^`]+)`,\s*(?:\[[^\]]*\],\s*)?([a-zA-Z_][a-zA-Z0-9_]*)Router'
|
|
124
|
+
for match in re.finditer(mount_backtick_pattern, content):
|
|
125
|
+
parent_router = match.group(1) + 'Router'
|
|
126
|
+
mount_path = match.group(2)
|
|
127
|
+
mount_path = re.sub(r'\(\$\{[^}]+\}\)', '', mount_path)
|
|
128
|
+
mount_path = re.sub(r'\$\{[^}]+\}', '', mount_path)
|
|
129
|
+
child_router = match.group(3) + 'Router'
|
|
130
|
+
mount_relationships.append((parent_router, mount_path, child_router))
|
|
131
|
+
|
|
132
|
+
root_mount_pattern = r'(?:rootRouter|app|router)\.use\(["\']([^"\']+)["\'],\s*([a-zA-Z_][a-zA-Z0-9_]*)Router'
|
|
133
|
+
for match in re.finditer(root_mount_pattern, content):
|
|
134
|
+
mount_path = match.group(1)
|
|
135
|
+
child_router = match.group(2) + 'Router'
|
|
136
|
+
mount_relationships.append(('rootRouter', mount_path, child_router))
|
|
137
|
+
|
|
138
|
+
router_mount_map['rootRouter'] = ''
|
|
139
|
+
for root_name in ['rootRouter', 'appRouter', 'mainRouter', 'serverRouter']:
|
|
140
|
+
if root_name not in router_mount_map:
|
|
141
|
+
router_mount_map[root_name] = ''
|
|
142
|
+
|
|
143
|
+
max_iterations = 20
|
|
144
|
+
for iteration in range(max_iterations):
|
|
145
|
+
changed = False
|
|
146
|
+
for parent, mount_path, child in mount_relationships:
|
|
147
|
+
if parent in router_mount_map and child not in router_mount_map:
|
|
148
|
+
parent_path = router_mount_map[parent]
|
|
149
|
+
full_path = parent_path + mount_path if parent_path else mount_path
|
|
150
|
+
full_path = re.sub(r'//+', '/', full_path)
|
|
151
|
+
router_mount_map[child] = full_path
|
|
152
|
+
changed = True
|
|
153
|
+
if not changed:
|
|
154
|
+
break
|
|
155
|
+
|
|
156
|
+
mounted_count = len([r for r in router_files if r in router_mount_map])
|
|
157
|
+
unmounted_count = len([r for r in router_files if r not in router_mount_map])
|
|
158
|
+
print(f"Found {len(router_files)} routers: {mounted_count} mounted, {unmounted_count} unmounted", flush=True)
|
|
159
|
+
|
|
160
|
+
def extract_endpoints_from_file(filepath):
|
|
161
|
+
"""Extract API endpoints from a Node.js route file."""
|
|
162
|
+
content = safe_read_file(filepath)
|
|
163
|
+
if content is None:
|
|
164
|
+
return
|
|
165
|
+
|
|
166
|
+
file_name = os.path.basename(filepath)
|
|
167
|
+
|
|
168
|
+
base_path = None
|
|
169
|
+
for router_name, router_file in router_files.items():
|
|
170
|
+
if router_file == filepath:
|
|
171
|
+
if router_name in router_mount_map:
|
|
172
|
+
base_path = router_mount_map[router_name]
|
|
173
|
+
break
|
|
174
|
+
|
|
175
|
+
if base_path is None:
|
|
176
|
+
content_check = content or ''
|
|
177
|
+
has_router = bool(re.search(r'const\s+[a-zA-Z_][a-zA-Z0-9_]*Router\s*=', content_check))
|
|
178
|
+
if has_router:
|
|
179
|
+
return
|
|
180
|
+
base_path = ''
|
|
181
|
+
|
|
182
|
+
if not base_path:
|
|
183
|
+
base_path_patterns = [
|
|
184
|
+
r"(?:router|app|server|rootRouter|v2Router)\.use\(['\"]([^'\"]+)['\"]",
|
|
185
|
+
r"[a-zA-Z_][a-zA-Z0-9_]*Router\.use\(['\"]([^'\"]+)['\"]",
|
|
186
|
+
]
|
|
187
|
+
for pattern in base_path_patterns:
|
|
188
|
+
matches = re.findall(pattern, content)
|
|
189
|
+
if matches:
|
|
190
|
+
for match in matches:
|
|
191
|
+
if isinstance(match, tuple):
|
|
192
|
+
match = match[0] if match[0] else ''
|
|
193
|
+
if match and len(match) > len(base_path):
|
|
194
|
+
base_path = match
|
|
195
|
+
|
|
196
|
+
all_query_params = set()
|
|
197
|
+
all_include_params = set()
|
|
198
|
+
|
|
199
|
+
query_param_pattern = r'req\.query\.([a-zA-Z_][a-zA-Z0-9_]*)'
|
|
200
|
+
for qp_match in re.finditer(query_param_pattern, content):
|
|
201
|
+
param_name = qp_match.group(1).lower()
|
|
202
|
+
all_query_params.add(param_name)
|
|
203
|
+
if 'include' in param_name:
|
|
204
|
+
all_include_params.add(param_name)
|
|
205
|
+
|
|
206
|
+
destructure_pattern = r'const\s*\{[^}]*([a-zA-Z_][a-zA-Z0-9_]*)[^}]*\}\s*=\s*req\.query'
|
|
207
|
+
for ds_match in re.finditer(destructure_pattern, content):
|
|
208
|
+
param_name = ds_match.group(1).lower()
|
|
209
|
+
all_query_params.add(param_name)
|
|
210
|
+
if 'include' in param_name:
|
|
211
|
+
all_include_params.add(param_name)
|
|
212
|
+
|
|
213
|
+
bracket_pattern = r'req\.query\[["\']([a-zA-Z_][a-zA-Z0-9_]*)["\']'
|
|
214
|
+
for br_match in re.finditer(bracket_pattern, content):
|
|
215
|
+
param_name = br_match.group(1).lower()
|
|
216
|
+
all_query_params.add(param_name)
|
|
217
|
+
if 'include' in param_name:
|
|
218
|
+
all_include_params.add(param_name)
|
|
219
|
+
|
|
220
|
+
route_patterns = [
|
|
221
|
+
r'(?:app|router|server)\.(get|post|put|patch|delete)\s*\(\s*["\']([^"\']+)["\']',
|
|
222
|
+
r'[a-zA-Z_][a-zA-Z0-9_]*Router\.(get|post|put|patch|delete)\s*\(\s*["\']([^"\']+)["\']',
|
|
223
|
+
r'(?:app|router|server)\.(get|post|put|patch|delete)\s*\(\s*`([^`]+)`',
|
|
224
|
+
r'[a-zA-Z_][a-zA-Z0-9_]*Router\.(get|post|put|patch|delete)\s*\(\s*`([^`]+)`',
|
|
225
|
+
r'\.route\s*\(\s*["\']([^"\']+)["\']\s*\)\s*\.(get|post|put|patch|delete)',
|
|
226
|
+
r'(?:app|router|server)\.(get|post|put|patch|delete)\s*\(\s*["\']([^"\']+)["\']\s*,\s*\{',
|
|
227
|
+
]
|
|
228
|
+
|
|
229
|
+
for pattern in route_patterns:
|
|
230
|
+
for match in re.finditer(pattern, content):
|
|
231
|
+
if 'route' in pattern:
|
|
232
|
+
path = match.group(1)
|
|
233
|
+
method = match.group(2).upper()
|
|
234
|
+
else:
|
|
235
|
+
method = match.group(1).upper()
|
|
236
|
+
path = match.group(2)
|
|
237
|
+
|
|
238
|
+
if base_path:
|
|
239
|
+
if path.startswith('/'):
|
|
240
|
+
full_path = base_path + path
|
|
241
|
+
else:
|
|
242
|
+
full_path = base_path + '/' + path
|
|
243
|
+
else:
|
|
244
|
+
full_path = path if path.startswith('/') else '/' + path
|
|
245
|
+
|
|
246
|
+
full_path = re.sub(r'\(\$\{[^}]+\}\)', '', full_path)
|
|
247
|
+
full_path = re.sub(r'\$\{[^}]+\}', '', full_path)
|
|
248
|
+
full_path = re.sub(r'\.json$', '', full_path)
|
|
249
|
+
full_path = re.sub(r'//+', '/', full_path)
|
|
250
|
+
full_path = full_path.rstrip('/')
|
|
251
|
+
if not full_path.startswith('/'):
|
|
252
|
+
full_path = '/' + full_path
|
|
253
|
+
if full_path == '':
|
|
254
|
+
full_path = '/'
|
|
255
|
+
|
|
256
|
+
defined_params = all_query_params.copy()
|
|
257
|
+
include_params = all_include_params.copy()
|
|
258
|
+
|
|
259
|
+
endpoint = {
|
|
260
|
+
'method': method,
|
|
261
|
+
'path': full_path,
|
|
262
|
+
'file': file_name,
|
|
263
|
+
'defined_params': defined_params,
|
|
264
|
+
'has_include_param': len(include_params) > 0
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
is_duplicate = False
|
|
268
|
+
for existing in endpoints:
|
|
269
|
+
if existing['method'] == endpoint['method'] and existing['path'] == endpoint['path']:
|
|
270
|
+
existing['defined_params'].update(endpoint['defined_params'])
|
|
271
|
+
if endpoint['has_include_param']:
|
|
272
|
+
existing['has_include_param'] = True
|
|
273
|
+
is_duplicate = True
|
|
274
|
+
break
|
|
275
|
+
|
|
276
|
+
if not is_duplicate:
|
|
277
|
+
endpoints.append(endpoint)
|
|
278
|
+
|
|
279
|
+
# Process route files
|
|
280
|
+
try:
|
|
281
|
+
build_router_mount_map()
|
|
282
|
+
|
|
283
|
+
for root, dirs, files in os.walk(ROUTES_DIR):
|
|
284
|
+
dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['node_modules', 'build', 'dist', 'coverage']]
|
|
285
|
+
|
|
286
|
+
for file in files:
|
|
287
|
+
if file.endswith(('.js', '.ts')) and not file.endswith('.test.js') and not file.endswith('.spec.js'):
|
|
288
|
+
filepath = os.path.join(root, file)
|
|
289
|
+
try:
|
|
290
|
+
extract_endpoints_from_file(filepath)
|
|
291
|
+
except Exception as e:
|
|
292
|
+
errors.append(f"Error processing {file}: {str(e)}")
|
|
293
|
+
except Exception as e:
|
|
294
|
+
print(f"ERROR: Failed to scan routes directory: {e}", flush=True)
|
|
295
|
+
sys.exit(1)
|
|
296
|
+
|
|
297
|
+
# =============================================================================
|
|
298
|
+
# POSTMAN COLLECTION PARSER (with quality tracking)
|
|
299
|
+
# =============================================================================
|
|
300
|
+
|
|
301
|
+
def extract_tests_from_item(item, collection_name, depth=0):
|
|
302
|
+
"""Extract test information from Postman collection items."""
|
|
303
|
+
if depth > 20:
|
|
304
|
+
return []
|
|
305
|
+
|
|
306
|
+
results = []
|
|
307
|
+
|
|
308
|
+
if isinstance(item, list):
|
|
309
|
+
for i in item:
|
|
310
|
+
results.extend(extract_tests_from_item(i, collection_name, depth + 1))
|
|
311
|
+
elif isinstance(item, dict):
|
|
312
|
+
if 'request' in item:
|
|
313
|
+
try:
|
|
314
|
+
request = item['request']
|
|
315
|
+
if not isinstance(request, dict):
|
|
316
|
+
return results
|
|
317
|
+
|
|
318
|
+
method = request.get('method', 'GET')
|
|
319
|
+
|
|
320
|
+
url = request.get('url', {})
|
|
321
|
+
query_params = set()
|
|
322
|
+
query_param_values = defaultdict(set)
|
|
323
|
+
|
|
324
|
+
if isinstance(url, str):
|
|
325
|
+
url_path = url
|
|
326
|
+
if '?' in url_path:
|
|
327
|
+
url_path, query_string = url_path.split('?', 1)
|
|
328
|
+
for param in query_string.split('&'):
|
|
329
|
+
if '=' in param:
|
|
330
|
+
key, val = param.split('=', 1)
|
|
331
|
+
query_params.add(key)
|
|
332
|
+
query_param_values[key].add(val)
|
|
333
|
+
elif isinstance(url, dict):
|
|
334
|
+
path_parts = url.get('path', [])
|
|
335
|
+
if isinstance(path_parts, list):
|
|
336
|
+
url_path = '/' + '/'.join(str(p) for p in path_parts) if path_parts else ''
|
|
337
|
+
else:
|
|
338
|
+
url_path = str(path_parts)
|
|
339
|
+
|
|
340
|
+
for qp in url.get('query', []) or []:
|
|
341
|
+
if isinstance(qp, dict):
|
|
342
|
+
key = qp.get('key', '')
|
|
343
|
+
val = qp.get('value', '')
|
|
344
|
+
if key:
|
|
345
|
+
query_params.add(key)
|
|
346
|
+
if val:
|
|
347
|
+
query_param_values[key].add(val)
|
|
348
|
+
else:
|
|
349
|
+
url_path = ''
|
|
350
|
+
|
|
351
|
+
if '?' in url_path:
|
|
352
|
+
url_path, extra_qs = url_path.split('?', 1)
|
|
353
|
+
for param in extra_qs.split('&'):
|
|
354
|
+
if '=' in param:
|
|
355
|
+
key, val = param.split('=', 1)
|
|
356
|
+
query_params.add(key)
|
|
357
|
+
query_param_values[key].add(val)
|
|
358
|
+
|
|
359
|
+
url_path = re.sub(r'\.json$', '', url_path)
|
|
360
|
+
url_path = re.sub(r'\{\{[^}]+\}\}', '{VAR}', url_path)
|
|
361
|
+
url_path = re.sub(r':[^/]+', '{VAR}', url_path)
|
|
362
|
+
|
|
363
|
+
status_codes = set()
|
|
364
|
+
body_assertions = 0
|
|
365
|
+
has_oneof = False
|
|
366
|
+
has_skip = False
|
|
367
|
+
skip_reason = None
|
|
368
|
+
skip_env = None
|
|
369
|
+
|
|
370
|
+
# Check pre-request scripts for skips
|
|
371
|
+
for event in item.get('event', []) or []:
|
|
372
|
+
if isinstance(event, dict) and event.get('listen') == 'prerequest':
|
|
373
|
+
script = event.get('script', {})
|
|
374
|
+
if isinstance(script, dict):
|
|
375
|
+
exec_lines = script.get('exec', [])
|
|
376
|
+
if isinstance(exec_lines, list):
|
|
377
|
+
prereq_text = '\n'.join(str(line) for line in exec_lines)
|
|
378
|
+
else:
|
|
379
|
+
prereq_text = str(exec_lines)
|
|
380
|
+
|
|
381
|
+
if 'pm.execution.skipRequest()' in prereq_text:
|
|
382
|
+
has_skip = True
|
|
383
|
+
|
|
384
|
+
# Format 1: [SKIP] Reason: ... | Env: ...
|
|
385
|
+
skip_match = re.search(
|
|
386
|
+
r"\[SKIP\].*?Reason:\s*([^|,;]+)(?:[|,;]\s*Env:\s*(\S+))?",
|
|
387
|
+
prereq_text, re.IGNORECASE
|
|
388
|
+
)
|
|
389
|
+
if skip_match:
|
|
390
|
+
skip_reason = skip_match.group(1).strip()
|
|
391
|
+
if skip_match.group(2):
|
|
392
|
+
skip_env = skip_match.group(2).strip().lower()
|
|
393
|
+
|
|
394
|
+
# Format 2: simple [SKIP] message
|
|
395
|
+
if not skip_reason:
|
|
396
|
+
simple_match = re.search(r"\[SKIP\]\s*(.+?)(?:['\"]|$)", prereq_text)
|
|
397
|
+
if simple_match and 'Reason:' not in simple_match.group(1):
|
|
398
|
+
skip_reason = simple_match.group(1).strip()
|
|
399
|
+
|
|
400
|
+
# Format 3: comment-based skip
|
|
401
|
+
if not skip_reason:
|
|
402
|
+
comment_match = re.search(r"//\s*SKIP[:\s]+(.+)", prereq_text)
|
|
403
|
+
if comment_match:
|
|
404
|
+
skip_reason = comment_match.group(1).strip()
|
|
405
|
+
|
|
406
|
+
# Format 4: console.log with reason (common Node pattern)
|
|
407
|
+
if not skip_reason:
|
|
408
|
+
log_match = re.search(r"console\.log\(['\"](?:\[SKIP\]\s*)?([^'\"]+)['\"]", prereq_text)
|
|
409
|
+
if log_match:
|
|
410
|
+
skip_reason = log_match.group(1).strip()
|
|
411
|
+
if skip_reason.startswith('Skipping:'):
|
|
412
|
+
skip_reason = skip_reason[len('Skipping:'):].strip()
|
|
413
|
+
|
|
414
|
+
# Auto-detect environment from the if condition
|
|
415
|
+
if not skip_env:
|
|
416
|
+
env_patterns = [
|
|
417
|
+
(r"pm\.environment\.get\(['\"]environment['\"]\)\s*===\s*['\"](\w+)['\"]", lambda m: m.group(1)),
|
|
418
|
+
(r"pm\.environment\.get\(['\"]environment['\"]\)\s*!==\s*['\"]local['\"]", lambda m: 'staging'),
|
|
419
|
+
(r"pm\.environment\.get\(['\"]environment['\"]\)\s*!==\s*['\"]staging['\"]", lambda m: 'local'),
|
|
420
|
+
(r"pm\.environment\.get\(['\"]env_name['\"]\)\s*===\s*['\"](\w+)['\"]", lambda m: m.group(1)),
|
|
421
|
+
(r"pm\.environment\.get\(['\"]env_name['\"]\)\s*!==\s*['\"]local['\"]", lambda m: 'staging'),
|
|
422
|
+
(r"pm\.environment\.get\(['\"]env_name['\"]\)\s*!==\s*['\"]staging['\"]", lambda m: 'local'),
|
|
423
|
+
(r"pm\.variables\.get\(['\"]env['\"]\)\s*===\s*['\"](\w+)['\"]", lambda m: m.group(1)),
|
|
424
|
+
]
|
|
425
|
+
for pattern, extractor in env_patterns:
|
|
426
|
+
env_match = re.search(pattern, prereq_text)
|
|
427
|
+
if env_match:
|
|
428
|
+
skip_env = extractor(env_match).lower()
|
|
429
|
+
break
|
|
430
|
+
|
|
431
|
+
# Parse test scripts for assertions
|
|
432
|
+
for event in item.get('event', []) or []:
|
|
433
|
+
if isinstance(event, dict) and event.get('listen') == 'test':
|
|
434
|
+
script = event.get('script', {})
|
|
435
|
+
if isinstance(script, dict):
|
|
436
|
+
exec_lines = script.get('exec', [])
|
|
437
|
+
if isinstance(exec_lines, list):
|
|
438
|
+
script_text = '\n'.join(str(line) for line in exec_lines)
|
|
439
|
+
else:
|
|
440
|
+
script_text = str(exec_lines)
|
|
441
|
+
|
|
442
|
+
# Status code patterns
|
|
443
|
+
for m in re.findall(r'to\.have\.status\((\d+)\)', script_text):
|
|
444
|
+
status_codes.add(str(m))
|
|
445
|
+
for m in re.findall(r'pm\.response\.code\)\.to\.equal\((\d+)\)', script_text):
|
|
446
|
+
status_codes.add(str(m))
|
|
447
|
+
for m in re.findall(r'pm\.response\.code\)\.to\.eql\((\d+)\)', script_text):
|
|
448
|
+
status_codes.add(str(m))
|
|
449
|
+
|
|
450
|
+
# Detect weak assertions: oneOf patterns
|
|
451
|
+
if 'oneOf' in script_text:
|
|
452
|
+
has_oneof = True
|
|
453
|
+
if re.search(r'pm\.expect\s*\(\s*\[.*?\]\s*\)\.to\.include\s*\(\s*pm\.response\.code', script_text):
|
|
454
|
+
has_oneof = True
|
|
455
|
+
if re.search(r'pm\.response\.code\s*\)\s*\.to\.be\.oneOf', script_text):
|
|
456
|
+
has_oneof = True
|
|
457
|
+
|
|
458
|
+
# Extract status codes from oneOf/include arrays
|
|
459
|
+
for oneof_match in re.finditer(r'\.to\.be\.oneOf\s*\(\s*\[([^\]]+)\]', script_text):
|
|
460
|
+
for code in re.findall(r'\d{3}', oneof_match.group(1)):
|
|
461
|
+
status_codes.add(str(code))
|
|
462
|
+
for include_match in re.finditer(r'pm\.expect\s*\(\s*\[([\d,\s]+)\]\s*\)\.to\.include', script_text):
|
|
463
|
+
for code in re.findall(r'\d{3}', include_match.group(1)):
|
|
464
|
+
status_codes.add(str(code))
|
|
465
|
+
|
|
466
|
+
# Body assertion patterns
|
|
467
|
+
body_patterns = [
|
|
468
|
+
r'pm\.expect\s*\(\s*jsonData\.',
|
|
469
|
+
r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.be\.an\s*\(',
|
|
470
|
+
r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.have\.property\s*\(',
|
|
471
|
+
r'pm\.expect\s*\(\s*jsonData\s*\)\.to\.not\.be\.null',
|
|
472
|
+
r'pm\.expect\s*\(\s*response\.',
|
|
473
|
+
r'\.to\.have\.property\(',
|
|
474
|
+
r'\.to\.include\(',
|
|
475
|
+
r'\.to\.eql\(',
|
|
476
|
+
]
|
|
477
|
+
for p in body_patterns:
|
|
478
|
+
body_assertions += len(re.findall(p, script_text))
|
|
479
|
+
|
|
480
|
+
# Extract request body
|
|
481
|
+
has_request_body = False
|
|
482
|
+
request_body_hash = None
|
|
483
|
+
request_body_fields = set()
|
|
484
|
+
body = request.get('body', {})
|
|
485
|
+
if isinstance(body, dict):
|
|
486
|
+
raw_body = body.get('raw', '')
|
|
487
|
+
if raw_body and isinstance(raw_body, str) and raw_body.strip():
|
|
488
|
+
has_request_body = True
|
|
489
|
+
request_body_hash = hash(raw_body.strip())
|
|
490
|
+
try:
|
|
491
|
+
parsed = json.loads(raw_body)
|
|
492
|
+
if isinstance(parsed, dict):
|
|
493
|
+
request_body_fields = set(parsed.keys())
|
|
494
|
+
except (json.JSONDecodeError, ValueError):
|
|
495
|
+
field_pattern = r'"([^"]+)"\s*:'
|
|
496
|
+
request_body_fields = set(re.findall(field_pattern, raw_body))
|
|
497
|
+
|
|
498
|
+
if url_path:
|
|
499
|
+
results.append({
|
|
500
|
+
'method': method,
|
|
501
|
+
'path': url_path,
|
|
502
|
+
'name': item.get('name', ''),
|
|
503
|
+
'status_codes': list(status_codes) if status_codes else [],
|
|
504
|
+
'collection': collection_name,
|
|
505
|
+
'body_assertions': body_assertions,
|
|
506
|
+
'has_request_body': has_request_body,
|
|
507
|
+
'request_body_hash': request_body_hash,
|
|
508
|
+
'request_body_fields': request_body_fields,
|
|
509
|
+
'query_params': query_params,
|
|
510
|
+
'query_param_values': dict(query_param_values),
|
|
511
|
+
'has_oneof': has_oneof,
|
|
512
|
+
'has_5xx': any(str(code).startswith('5') for code in status_codes),
|
|
513
|
+
'has_skip': has_skip,
|
|
514
|
+
'skip_reason': skip_reason,
|
|
515
|
+
'skip_env': skip_env
|
|
516
|
+
})
|
|
517
|
+
except Exception as e:
|
|
518
|
+
errors.append(f"Error parsing request in {collection_name}: {str(e)}")
|
|
519
|
+
|
|
520
|
+
if 'item' in item:
|
|
521
|
+
results.extend(extract_tests_from_item(item['item'], collection_name, depth + 1))
|
|
522
|
+
|
|
523
|
+
return results
|
|
524
|
+
|
|
525
|
+
def find_collection_files(base_dir):
|
|
526
|
+
"""Find all Postman collection JSON files in the given directory."""
|
|
527
|
+
collection_files = []
|
|
528
|
+
seen_files = set()
|
|
529
|
+
|
|
530
|
+
search_dirs = [base_dir]
|
|
531
|
+
for subdir in ['collections', 'api-tests', 'tests']:
|
|
532
|
+
subpath = os.path.join(base_dir, subdir)
|
|
533
|
+
if os.path.isdir(subpath):
|
|
534
|
+
search_dirs.append(subpath)
|
|
535
|
+
|
|
536
|
+
seen_dirs = set()
|
|
537
|
+
for search_dir in search_dirs:
|
|
538
|
+
if search_dir in seen_dirs:
|
|
539
|
+
continue
|
|
540
|
+
seen_dirs.add(search_dir)
|
|
541
|
+
|
|
542
|
+
try:
|
|
543
|
+
for root, dirs, files in os.walk(search_dir):
|
|
544
|
+
dirs[:] = [d for d in dirs if d not in [
|
|
545
|
+
'node_modules', 'environments', 'reports', 'config',
|
|
546
|
+
'scripts', '.git', 'data', 'schemas', 'seeders',
|
|
547
|
+
'mocks', 'test_data'
|
|
548
|
+
]]
|
|
549
|
+
|
|
550
|
+
for file in files:
|
|
551
|
+
if file.endswith('.json'):
|
|
552
|
+
skip_exact = [
|
|
553
|
+
'package.json', 'package-lock.json',
|
|
554
|
+
'tsconfig.json', 'jsconfig.json',
|
|
555
|
+
'.eslintrc.json', '.prettierrc.json',
|
|
556
|
+
'newman-reporter-config.json'
|
|
557
|
+
]
|
|
558
|
+
|
|
559
|
+
is_environment_file = (
|
|
560
|
+
file.lower().endswith('.environment.json') or
|
|
561
|
+
file.lower().endswith('_environment.json') or
|
|
562
|
+
file.lower().endswith('.postman_environment.json') or
|
|
563
|
+
file.lower() == 'environment.json'
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
if file.lower() in skip_exact:
|
|
567
|
+
continue
|
|
568
|
+
if is_environment_file:
|
|
569
|
+
continue
|
|
570
|
+
|
|
571
|
+
full_path = os.path.abspath(os.path.join(root, file))
|
|
572
|
+
if full_path not in seen_files:
|
|
573
|
+
seen_files.add(full_path)
|
|
574
|
+
collection_files.append(full_path)
|
|
575
|
+
except Exception as e:
|
|
576
|
+
errors.append(f"Error scanning {search_dir}: {str(e)}")
|
|
577
|
+
|
|
578
|
+
return collection_files
|
|
579
|
+
|
|
580
|
+
try:
|
|
581
|
+
collection_files = find_collection_files(POSTMAN_DIR)
|
|
582
|
+
print(f"Found {len(collection_files)} collection file(s)", flush=True)
|
|
583
|
+
|
|
584
|
+
for filepath in collection_files:
|
|
585
|
+
file = os.path.basename(filepath)
|
|
586
|
+
file_size = os.path.getsize(filepath)
|
|
587
|
+
if file_size > MAX_FILE_SIZE_MB * 1024 * 1024:
|
|
588
|
+
errors.append(f"Skipped {file}: collection too large")
|
|
589
|
+
continue
|
|
590
|
+
|
|
591
|
+
try:
|
|
592
|
+
content = safe_read_file(filepath)
|
|
593
|
+
if content:
|
|
594
|
+
collection = json.loads(content)
|
|
595
|
+
if not isinstance(collection, dict) or ('item' not in collection and 'info' not in collection):
|
|
596
|
+
continue
|
|
597
|
+
collection_name = file.replace('.json', '')
|
|
598
|
+
tests = extract_tests_from_item(collection.get('item', []), collection_name)
|
|
599
|
+
tested.extend(tests)
|
|
600
|
+
except json.JSONDecodeError as e:
|
|
601
|
+
errors.append(f"Invalid JSON in {file}: {str(e)}")
|
|
602
|
+
except Exception as e:
|
|
603
|
+
errors.append(f"Error processing {file}: {str(e)}")
|
|
604
|
+
except Exception as e:
|
|
605
|
+
print(f"ERROR: Failed to scan Postman directory: {e}", flush=True)
|
|
606
|
+
sys.exit(1)
|
|
607
|
+
|
|
608
|
+
# =============================================================================
|
|
609
|
+
# MATCHING AND AGGREGATION
|
|
610
|
+
# =============================================================================
|
|
611
|
+
|
|
612
|
+
seen = set()
|
|
613
|
+
unique_endpoints = []
|
|
614
|
+
for ep in endpoints:
|
|
615
|
+
key = (ep['method'], ep['path'])
|
|
616
|
+
if key not in seen:
|
|
617
|
+
seen.add(key)
|
|
618
|
+
unique_endpoints.append(ep)
|
|
619
|
+
endpoints = sorted(unique_endpoints, key=lambda x: (x['path'], x['method']))
|
|
620
|
+
|
|
621
|
+
endpoint_stats = defaultdict(lambda: {
|
|
622
|
+
'test_count': 0,
|
|
623
|
+
'status_codes': set(),
|
|
624
|
+
'body_assertions': 0,
|
|
625
|
+
'request_body_variations': set(),
|
|
626
|
+
'has_body_test': False,
|
|
627
|
+
'query_params': set(),
|
|
628
|
+
'query_param_values': defaultdict(set),
|
|
629
|
+
'request_body_fields': set(),
|
|
630
|
+
'has_oneof': False,
|
|
631
|
+
'oneof_count': 0,
|
|
632
|
+
'has_5xx': False,
|
|
633
|
+
'count_5xx': 0,
|
|
634
|
+
'skipped_tests': []
|
|
635
|
+
})
|
|
636
|
+
|
|
637
|
+
all_skipped_tests = []
|
|
638
|
+
|
|
639
|
+
def normalize_path(path):
|
|
640
|
+
"""Normalize path for comparison."""
|
|
641
|
+
if not path:
|
|
642
|
+
return ''
|
|
643
|
+
normalized = re.sub(r'^/?(api/)?v\d+/', '/', path)
|
|
644
|
+
normalized = re.sub(r'\{\{[^}]+\}\}', '{VAR}', normalized)
|
|
645
|
+
normalized = re.sub(r'\{[^}]+\}', '{VAR}', normalized)
|
|
646
|
+
normalized = re.sub(r':[^/]+', '{VAR}', normalized)
|
|
647
|
+
normalized = re.sub(r'/(\d+)(?=/|$)', '/{VAR}', normalized)
|
|
648
|
+
normalized = re.sub(r'\.json$', '', normalized)
|
|
649
|
+
return normalized.lower().replace('{var}', '{VAR}')
|
|
650
|
+
|
|
651
|
+
def paths_match(controller_path, test_path):
|
|
652
|
+
"""Check if a test path matches a controller path."""
|
|
653
|
+
norm_controller = normalize_path(controller_path)
|
|
654
|
+
norm_test = normalize_path(test_path)
|
|
655
|
+
|
|
656
|
+
if norm_controller == norm_test:
|
|
657
|
+
return True
|
|
658
|
+
|
|
659
|
+
try:
|
|
660
|
+
pattern = re.escape(norm_controller).replace(r'\{VAR\}', '[^/]+')
|
|
661
|
+
if re.fullmatch(pattern, norm_test):
|
|
662
|
+
return True
|
|
663
|
+
except re.error:
|
|
664
|
+
pass
|
|
665
|
+
|
|
666
|
+
controller_parts = norm_controller.rstrip('/').split('/')
|
|
667
|
+
test_parts = norm_test.rstrip('/').split('/')
|
|
668
|
+
|
|
669
|
+
if len(controller_parts) >= 2 and len(test_parts) >= 2:
|
|
670
|
+
if controller_parts[-2:] == test_parts[-2:]:
|
|
671
|
+
return True
|
|
672
|
+
c_last = '/'.join(controller_parts[-2:])
|
|
673
|
+
t_last = '/'.join(test_parts[-2:])
|
|
674
|
+
pattern = re.sub(r'\{VAR\}', '[^/]+', re.escape(c_last)).replace(r'\{VAR\}', '[^/]+')
|
|
675
|
+
try:
|
|
676
|
+
if re.fullmatch(pattern, t_last):
|
|
677
|
+
return True
|
|
678
|
+
except re.error:
|
|
679
|
+
pass
|
|
680
|
+
|
|
681
|
+
return False
|
|
682
|
+
|
|
683
|
+
def update_stats(key, test):
|
|
684
|
+
"""Update endpoint stats with test data."""
|
|
685
|
+
endpoint_stats[key]['test_count'] += 1
|
|
686
|
+
endpoint_stats[key]['status_codes'].update(test['status_codes'])
|
|
687
|
+
endpoint_stats[key]['body_assertions'] += test['body_assertions']
|
|
688
|
+
if test['body_assertions'] > 0:
|
|
689
|
+
endpoint_stats[key]['has_body_test'] = True
|
|
690
|
+
if test['request_body_hash']:
|
|
691
|
+
endpoint_stats[key]['request_body_variations'].add(test['request_body_hash'])
|
|
692
|
+
endpoint_stats[key]['query_params'].update(test.get('query_params', set()))
|
|
693
|
+
for param, values in test.get('query_param_values', {}).items():
|
|
694
|
+
endpoint_stats[key]['query_param_values'][param].update(values)
|
|
695
|
+
endpoint_stats[key]['request_body_fields'].update(test.get('request_body_fields', set()))
|
|
696
|
+
if test.get('has_oneof', False):
|
|
697
|
+
endpoint_stats[key]['has_oneof'] = True
|
|
698
|
+
endpoint_stats[key]['oneof_count'] += 1
|
|
699
|
+
if test.get('has_5xx', False):
|
|
700
|
+
endpoint_stats[key]['has_5xx'] = True
|
|
701
|
+
endpoint_stats[key]['count_5xx'] += 1
|
|
702
|
+
if test.get('has_skip', False):
|
|
703
|
+
endpoint_stats[key]['skipped_tests'].append({
|
|
704
|
+
'name': test.get('name', ''),
|
|
705
|
+
'reason': test.get('skip_reason'),
|
|
706
|
+
'env': test.get('skip_env')
|
|
707
|
+
})
|
|
708
|
+
|
|
709
|
+
for test in tested:
|
|
710
|
+
test_path = test['path']
|
|
711
|
+
test_method = test['method']
|
|
712
|
+
|
|
713
|
+
if test.get('has_skip', False):
|
|
714
|
+
all_skipped_tests.append({
|
|
715
|
+
'name': test.get('name', 'Unknown'),
|
|
716
|
+
'collection': test.get('collection', ''),
|
|
717
|
+
'method': test_method,
|
|
718
|
+
'path': test_path,
|
|
719
|
+
'reason': test.get('skip_reason'),
|
|
720
|
+
'env': test.get('skip_env')
|
|
721
|
+
})
|
|
722
|
+
|
|
723
|
+
matched = False
|
|
724
|
+
for ep in endpoints:
|
|
725
|
+
if test_method == ep['method'] and paths_match(ep['path'], test_path):
|
|
726
|
+
key = f"{ep['method']} {ep['path']}"
|
|
727
|
+
update_stats(key, test)
|
|
728
|
+
matched = True
|
|
729
|
+
break
|
|
730
|
+
|
|
731
|
+
if not matched:
|
|
732
|
+
key = f"{test_method} {test_path}"
|
|
733
|
+
update_stats(key, test)
|
|
734
|
+
|
|
735
|
+
# =============================================================================
|
|
736
|
+
# METRICS CALCULATION
|
|
737
|
+
# =============================================================================
|
|
738
|
+
|
|
739
|
+
total_tests = sum(s['test_count'] for s in endpoint_stats.values())
|
|
740
|
+
total_body = sum(s['body_assertions'] for s in endpoint_stats.values())
|
|
741
|
+
tested_ep = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('test_count', 0) > 0)
|
|
742
|
+
well_tested = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('test_count', 0) >= 3)
|
|
743
|
+
with_body = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_body_test', False))
|
|
744
|
+
not_tested = len(endpoints) - tested_ep
|
|
745
|
+
|
|
746
|
+
def has_success_code(status_codes):
|
|
747
|
+
return any(str(code).startswith('2') for code in status_codes)
|
|
748
|
+
|
|
749
|
+
def has_401_code(status_codes):
|
|
750
|
+
return '401' in [str(code) for code in status_codes]
|
|
751
|
+
|
|
752
|
+
success_tested = sum(1 for ep in endpoints if has_success_code(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('status_codes', set())))
|
|
753
|
+
error_only = tested_ep - success_tested
|
|
754
|
+
|
|
755
|
+
auth_tested = sum(1 for ep in endpoints if has_401_code(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('status_codes', set())))
|
|
756
|
+
auth_pct = (auth_tested / len(endpoints) * 100) if endpoints else 0
|
|
757
|
+
|
|
758
|
+
oneof_endpoints = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_oneof', False))
|
|
759
|
+
total_oneof_tests = sum(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('oneof_count', 0) for ep in endpoints)
|
|
760
|
+
|
|
761
|
+
endpoints_with_5xx = sum(1 for ep in endpoints if endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('has_5xx', False))
|
|
762
|
+
total_5xx_tests = sum(endpoint_stats.get(f"{ep['method']} {ep['path']}", {}).get('count_5xx', 0) for ep in endpoints)
|
|
763
|
+
|
|
764
|
+
total_skipped_tests = len(all_skipped_tests)
|
|
765
|
+
skips_with_reason = sum(1 for s in all_skipped_tests if s.get('reason'))
|
|
766
|
+
skips_without_reason = total_skipped_tests - skips_with_reason
|
|
767
|
+
skips_by_env = defaultdict(list)
|
|
768
|
+
for skip in all_skipped_tests:
|
|
769
|
+
env = skip.get('env', 'unknown')
|
|
770
|
+
skips_by_env[env].append(skip)
|
|
771
|
+
|
|
772
|
+
# Query params metrics
|
|
773
|
+
total_defined_params = 0
|
|
774
|
+
total_tested_params = 0
|
|
775
|
+
endpoints_with_params = []
|
|
776
|
+
for ep in endpoints:
|
|
777
|
+
defined_params = ep.get('defined_params', set())
|
|
778
|
+
if defined_params:
|
|
779
|
+
total_defined_params += len(defined_params)
|
|
780
|
+
key = f"{ep['method']} {ep['path']}"
|
|
781
|
+
stats = endpoint_stats.get(key, {})
|
|
782
|
+
tested_params = stats.get('query_params', set())
|
|
783
|
+
tested_normalized = set(p.lower().replace('_', '') for p in tested_params)
|
|
784
|
+
covered = len(defined_params.intersection(tested_normalized))
|
|
785
|
+
total_tested_params += covered
|
|
786
|
+
|
|
787
|
+
param_values = stats.get('query_param_values', {})
|
|
788
|
+
if tested_params:
|
|
789
|
+
endpoints_with_params.append({
|
|
790
|
+
'method': ep['method'],
|
|
791
|
+
'path': ep['path'],
|
|
792
|
+
'params': tested_params,
|
|
793
|
+
'param_values': param_values,
|
|
794
|
+
'total_values': sum(len(v) for v in param_values.values())
|
|
795
|
+
})
|
|
796
|
+
|
|
797
|
+
# Include params metrics
|
|
798
|
+
endpoints_with_include = []
|
|
799
|
+
total_expected_include_values = 0
|
|
800
|
+
total_tested_include_values = 0
|
|
801
|
+
|
|
802
|
+
for ep in endpoints:
|
|
803
|
+
if ep.get('has_include_param', False):
|
|
804
|
+
key = f"{ep['method']} {ep['path']}"
|
|
805
|
+
stats = endpoint_stats.get(key, {})
|
|
806
|
+
tested_params = stats.get('query_params', set())
|
|
807
|
+
param_values = stats.get('query_param_values', {})
|
|
808
|
+
|
|
809
|
+
include_tested = any('include' in p.lower() for p in tested_params)
|
|
810
|
+
include_values = set()
|
|
811
|
+
include_variations = 0
|
|
812
|
+
|
|
813
|
+
for param, values in param_values.items():
|
|
814
|
+
if 'include' in param.lower():
|
|
815
|
+
for v in values:
|
|
816
|
+
for single_val in v.split(','):
|
|
817
|
+
include_values.add(single_val.strip())
|
|
818
|
+
include_variations = len(values)
|
|
819
|
+
|
|
820
|
+
endpoints_with_include.append({
|
|
821
|
+
'method': ep['method'],
|
|
822
|
+
'path': ep['path'],
|
|
823
|
+
'tested': include_tested,
|
|
824
|
+
'values': include_values,
|
|
825
|
+
'variations': include_variations
|
|
826
|
+
})
|
|
827
|
+
|
|
828
|
+
# Request body metrics
|
|
829
|
+
endpoints_with_body = []
|
|
830
|
+
for ep in endpoints:
|
|
831
|
+
if ep['method'] in ['POST', 'PUT', 'PATCH']:
|
|
832
|
+
key = f"{ep['method']} {ep['path']}"
|
|
833
|
+
stats = endpoint_stats.get(key, {})
|
|
834
|
+
fields = stats.get('request_body_fields', set())
|
|
835
|
+
variations = len(stats.get('request_body_variations', set()))
|
|
836
|
+
|
|
837
|
+
if fields or variations > 0:
|
|
838
|
+
endpoints_with_body.append({
|
|
839
|
+
'method': ep['method'],
|
|
840
|
+
'path': ep['path'],
|
|
841
|
+
'fields': fields,
|
|
842
|
+
'variations': variations
|
|
843
|
+
})
|
|
844
|
+
|
|
845
|
+
# Percentages
|
|
846
|
+
tested_pct = (tested_ep / len(endpoints) * 100) if endpoints else 0
|
|
847
|
+
well_tested_pct = (well_tested / len(endpoints) * 100) if endpoints else 0
|
|
848
|
+
body_pct = (with_body / len(endpoints) * 100) if endpoints else 0
|
|
849
|
+
qp_pct = (total_tested_params / total_defined_params * 100) if total_defined_params > 0 else 0
|
|
850
|
+
success_pct = (success_tested / len(endpoints) * 100) if endpoints else 0
|
|
851
|
+
|
|
852
|
+
include_tested_count = sum(1 for ep in endpoints_with_include if ep['tested'])
|
|
853
|
+
include_total_count = len(endpoints_with_include)
|
|
854
|
+
include_endpoints_pct = (include_tested_count / include_total_count * 100) if include_total_count > 0 else 100
|
|
855
|
+
|
|
856
|
+
# Gaps analysis
|
|
857
|
+
gaps_no_tests = []
|
|
858
|
+
gaps_no_2xx = []
|
|
859
|
+
gaps_weak_assertions = []
|
|
860
|
+
gaps_5xx_assertions = []
|
|
861
|
+
|
|
862
|
+
for ep in endpoints:
|
|
863
|
+
key = f"{ep['method']} {ep['path']}"
|
|
864
|
+
stats = endpoint_stats.get(key, {'test_count': 0, 'status_codes': set()})
|
|
865
|
+
|
|
866
|
+
if stats['test_count'] == 0:
|
|
867
|
+
gaps_no_tests.append(ep)
|
|
868
|
+
elif not has_success_code(stats.get('status_codes', set())):
|
|
869
|
+
gaps_no_2xx.append(ep)
|
|
870
|
+
|
|
871
|
+
if stats.get('has_oneof', False):
|
|
872
|
+
gaps_weak_assertions.append({**ep, 'oneof_count': stats.get('oneof_count', 0)})
|
|
873
|
+
|
|
874
|
+
if stats.get('has_5xx', False):
|
|
875
|
+
gaps_5xx_assertions.append({**ep, 'count_5xx': stats.get('count_5xx', 0)})
|
|
876
|
+
|
|
877
|
+
# Health status
|
|
878
|
+
def get_health_status(success_pct, well_tested_pct, error_only, oneof_count):
|
|
879
|
+
if success_pct >= 100 and well_tested_pct >= 80 and error_only == 0 and oneof_count == 0:
|
|
880
|
+
return ('excellent', '🟢', 'Excellent', 'All endpoints have 2xx tests with strong assertions')
|
|
881
|
+
elif success_pct >= 90 and well_tested_pct >= 60 and error_only <= 2:
|
|
882
|
+
return ('good', '🟡', 'Good', 'Most endpoints well tested, minor gaps exist')
|
|
883
|
+
elif success_pct >= 70 and well_tested_pct >= 40:
|
|
884
|
+
return ('fair', '🟠', 'Fair', 'Moderate coverage, needs improvement')
|
|
885
|
+
else:
|
|
886
|
+
return ('poor', '🔴', 'Needs Work', 'Significant testing gaps exist')
|
|
887
|
+
|
|
888
|
+
health_class, health_icon, health_label, health_desc = get_health_status(success_pct, well_tested_pct, error_only, total_oneof_tests)
|
|
889
|
+
|
|
890
|
+
# =============================================================================
|
|
891
|
+
# HTML GENERATION (single-pass, direct)
|
|
892
|
+
# =============================================================================
|
|
893
|
+
|
|
894
|
+
CSS = """
|
|
895
|
+
:root {
|
|
896
|
+
--color-primary: #3b82f6; --color-success: #10b981; --color-warning: #f59e0b;
|
|
897
|
+
--color-error: #ef4444; --color-heading: #0f172a; --color-text: #1e293b;
|
|
898
|
+
--color-muted: #64748b; --color-card: #ffffff; --color-bg: #f8fafc;
|
|
899
|
+
--color-border: #e2e8f0; --radius-sm: 8px; --radius-md: 12px;
|
|
900
|
+
--shadow-sm: 0 1px 2px rgba(0,0,0,0.05); --shadow-md: 0 4px 6px rgba(0,0,0,0.07);
|
|
901
|
+
}
|
|
902
|
+
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
903
|
+
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: var(--color-bg); color: var(--color-text); line-height: 1.6; padding: 2rem; }
|
|
904
|
+
.container { max-width: 1200px; margin: 0 auto; }
|
|
905
|
+
.header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 2rem; padding-bottom: 1rem; border-bottom: 2px solid var(--color-border); }
|
|
906
|
+
.header-left h1 { font-size: 1.5rem; font-weight: 700; color: var(--color-heading); }
|
|
907
|
+
.header-left .logo { font-size: 0.75rem; font-weight: 600; color: var(--color-primary); text-transform: uppercase; letter-spacing: 0.05em; }
|
|
908
|
+
.timestamp { color: var(--color-muted); font-size: 0.8rem; }
|
|
909
|
+
.health-badge { padding: 0.5rem 1rem; border-radius: var(--radius-sm); font-weight: 600; font-size: 0.9rem; }
|
|
910
|
+
.health-badge.excellent { background: #ecfdf5; color: #047857; border: 1px solid #a7f3d0; }
|
|
911
|
+
.health-badge.good { background: #fefce8; color: #a16207; border: 1px solid #fde68a; }
|
|
912
|
+
.health-badge.fair { background: #fff7ed; color: #c2410c; border: 1px solid #fed7aa; }
|
|
913
|
+
.health-badge.poor { background: #fef2f2; color: #b91c1c; border: 1px solid #fecaca; }
|
|
914
|
+
.section { margin-bottom: 1.5rem; }
|
|
915
|
+
.section-title { font-size: 1.1rem; font-weight: 600; color: var(--color-heading); margin-bottom: 0.75rem; display: flex; align-items: center; gap: 0.5rem; }
|
|
916
|
+
table { width: 100%; border-collapse: collapse; background: var(--color-card); border: 1px solid var(--color-border); border-radius: var(--radius-sm); overflow: hidden; margin-bottom: 1rem; font-size: 0.8rem; }
|
|
917
|
+
th, td { padding: 0.5rem; text-align: center; border-bottom: 1px solid var(--color-border); vertical-align: middle; }
|
|
918
|
+
th { background: #f1f5f9; font-weight: 600; font-size: 0.65rem; text-transform: uppercase; color: var(--color-muted); }
|
|
919
|
+
td.left, th.left { text-align: left; }
|
|
920
|
+
tr:hover { background: #f8fafc; }
|
|
921
|
+
.method { font-weight: 600; padding: 0.15rem 0.4rem; border-radius: 4px; font-size: 0.7rem; display: inline-block; }
|
|
922
|
+
.method-GET { background: #dbeafe; color: #1d4ed8; }
|
|
923
|
+
.method-POST { background: #d1fae5; color: #047857; }
|
|
924
|
+
.method-PUT { background: #fef3c7; color: #b45309; }
|
|
925
|
+
.method-PATCH { background: #ede9fe; color: #6d28d9; }
|
|
926
|
+
.method-DELETE { background: #fee2e2; color: #b91c1c; }
|
|
927
|
+
.path { font-family: 'SF Mono', Consolas, monospace; font-size: 0.78rem; color: var(--color-heading); word-break: break-all; }
|
|
928
|
+
.status-icon { font-size: 1rem; }
|
|
929
|
+
.count { background: var(--color-border); padding: 0.125rem 0.5rem; border-radius: 10px; font-size: 0.75rem; font-weight: 500; margin-left: 0.5rem; }
|
|
930
|
+
.gaps-section { margin-bottom: 1rem; padding: 0.75rem 1rem; border-radius: var(--radius-sm); background: #fef2f2; border: 1px solid #fecaca; }
|
|
931
|
+
.gaps-section.warning { background: #fffbeb; border-color: #fde68a; }
|
|
932
|
+
.gaps-section .title { font-weight: 600; font-size: 0.85rem; margin-bottom: 0.5rem; }
|
|
933
|
+
.gap-item { font-size: 0.8rem; padding: 0.2rem 0; color: var(--color-text); }
|
|
934
|
+
details { margin-bottom: 1rem; background: var(--color-card); border: 1px solid var(--color-border); border-radius: var(--radius-sm); overflow: hidden; }
|
|
935
|
+
details summary { padding: 0.75rem 1rem; cursor: pointer; font-weight: 600; font-size: 0.9rem; background: #f8fafc; display: flex; align-items: center; gap: 0.5rem; }
|
|
936
|
+
details summary:hover { background: #f1f5f9; }
|
|
937
|
+
details .content { padding: 0; }
|
|
938
|
+
footer { margin-top: 2rem; text-align: center; color: #94a3b8; font-size: 0.75rem; padding-top: 1rem; border-top: 1px solid var(--color-border); }
|
|
939
|
+
"""
|
|
940
|
+
|
|
941
|
+
def metric(label, value, detail=None, status=None):
|
|
942
|
+
color = f'var(--color-{status})' if status else 'var(--color-primary)'
|
|
943
|
+
det = f' <span style="color:var(--color-muted);font-size:0.75rem;">({esc(str(detail))})</span>' if detail else ''
|
|
944
|
+
return f'<span style="margin-right:1.5rem;"><strong style="color:{color};">{esc(str(value))}</strong> {esc(str(label))}{det}</span>'
|
|
945
|
+
|
|
946
|
+
def gaps_section(title, items, warning=False, count_key=None):
|
|
947
|
+
if not items:
|
|
948
|
+
return ''
|
|
949
|
+
cls = 'gaps-section warning' if warning else 'gaps-section'
|
|
950
|
+
items_html = ''.join(
|
|
951
|
+
f'<div class="gap-item"><span class="method method-{esc(ep["method"])}">{esc(ep["method"])}</span> {esc(ep["path"])}'
|
|
952
|
+
f'{f" ({ep.get(count_key, 0)} tests)" if count_key else ""}</div>'
|
|
953
|
+
for ep in items
|
|
954
|
+
)
|
|
955
|
+
return f'<div class="{cls}"><div class="title">{esc(title)}</div>{items_html}</div>'
|
|
956
|
+
|
|
957
|
+
def collapsible(title, count, headers, rows):
|
|
958
|
+
if not rows:
|
|
959
|
+
return ''
|
|
960
|
+
h = ''.join(f'<th style="{esc(hdr.get("style",""))}">{esc(hdr["label"])}</th>' for hdr in headers)
|
|
961
|
+
r = ''.join(rows)
|
|
962
|
+
return f'''<details><summary>{esc(title)}<span class="count">{esc(str(count))}</span></summary>
|
|
963
|
+
<div class="content"><table><thead><tr>{h}</tr></thead><tbody>{r}</tbody></table></div></details>'''
|
|
964
|
+
|
|
965
|
+
# Issues summary
|
|
966
|
+
issues = []
|
|
967
|
+
if not_tested > 0:
|
|
968
|
+
issues.append(f'{not_tested} untested')
|
|
969
|
+
if total_5xx_tests > 0:
|
|
970
|
+
issues.append(f'{total_5xx_tests} 5xx')
|
|
971
|
+
if total_oneof_tests > 0:
|
|
972
|
+
issues.append(f'{total_oneof_tests} weak')
|
|
973
|
+
if skips_without_reason > 0:
|
|
974
|
+
issues.append(f'{skips_without_reason} undoc skips')
|
|
975
|
+
issues_str = ', '.join(issues) if issues else None
|
|
976
|
+
|
|
977
|
+
# Build summary
|
|
978
|
+
summary_html = f'''<div class="section">
|
|
979
|
+
<div style="display:flex;align-items:center;justify-content:space-between;flex-wrap:wrap;gap:1rem;padding:1rem 1.25rem;background:var(--color-card);border:1px solid var(--color-border);border-radius:var(--radius-sm);box-shadow:var(--shadow-sm);">
|
|
980
|
+
<div style="display:flex;align-items:center;gap:0.75rem;">
|
|
981
|
+
<span style="font-size:1.5rem;">{health_icon}</span>
|
|
982
|
+
<span style="font-weight:600;color:var(--color-heading);">{health_label}</span>
|
|
983
|
+
</div>
|
|
984
|
+
<div style="display:flex;flex-wrap:wrap;align-items:center;">
|
|
985
|
+
{metric('endpoints', len(endpoints), f'{total_tests} tests')}
|
|
986
|
+
{metric('2xx', f'{success_pct:.0f}%', f'{success_tested}/{len(endpoints)}', 'success' if success_pct >= 100 else 'warning' if success_pct >= 80 else 'error')}
|
|
987
|
+
{metric('well tested', f'{well_tested_pct:.0f}%', f'{well_tested}/{len(endpoints)}', 'success' if well_tested_pct >= 100 else 'warning' if well_tested_pct >= 80 else 'error')}
|
|
988
|
+
{metric('body validated', f'{body_pct:.0f}%', f'{with_body}/{len(endpoints)}', 'success' if body_pct >= 100 else 'warning' if body_pct >= 80 else 'error')}
|
|
989
|
+
{metric('query params', f'{qp_pct:.0f}%', f'{total_tested_params}/{total_defined_params}', 'success' if qp_pct >= 100 else 'warning' if qp_pct >= 80 else 'error')}
|
|
990
|
+
{f'<span style="color:var(--color-error);font-weight:500;">⚠ {issues_str}</span>' if issues_str else ''}
|
|
991
|
+
</div>
|
|
992
|
+
</div>
|
|
993
|
+
</div>'''
|
|
994
|
+
|
|
995
|
+
# Gaps analysis
|
|
996
|
+
gaps_html = ''
|
|
997
|
+
if gaps_no_tests or gaps_no_2xx or gaps_weak_assertions or gaps_5xx_assertions:
|
|
998
|
+
gaps_html = '<div class="section"><div class="section-title">⚠️ Gaps Analysis</div>'
|
|
999
|
+
gaps_html += gaps_section(f'❌ No Tests ({len(gaps_no_tests)} endpoints)', gaps_no_tests)
|
|
1000
|
+
gaps_html += gaps_section(f'❌ No 2xx Tests ({len(gaps_no_2xx)} endpoints)', gaps_no_2xx)
|
|
1001
|
+
gaps_html += gaps_section(f'⚠️ Weak Assertions (oneOf) - {len(gaps_weak_assertions)} endpoints', gaps_weak_assertions, warning=True, count_key='oneof_count')
|
|
1002
|
+
gaps_html += gaps_section(f'🔴 5xx Assertions (bad practice) - {len(gaps_5xx_assertions)} endpoints', gaps_5xx_assertions, count_key='count_5xx')
|
|
1003
|
+
gaps_html += '</div>'
|
|
1004
|
+
|
|
1005
|
+
# Endpoint details table
|
|
1006
|
+
def endpoint_row(ep, stats):
|
|
1007
|
+
tc = stats.get('test_count', 0)
|
|
1008
|
+
codes = stats.get('status_codes', set())
|
|
1009
|
+
codes_str = ', '.join(sorted(codes, key=lambda x: int(x) if x.isdigit() else 0)) if codes else '—'
|
|
1010
|
+
body = stats.get('body_assertions', 0) or '—'
|
|
1011
|
+
has_2xx = any(str(c).startswith('2') for c in codes)
|
|
1012
|
+
success_icon = '✅' if has_2xx else ('❌' if tc > 0 else '—')
|
|
1013
|
+
defined = ep.get('defined_params', set())
|
|
1014
|
+
tested_p = stats.get('query_params', set())
|
|
1015
|
+
qp_str = '—' if not defined else f'{len(defined.intersection(set(p.lower().replace("_","") for p in tested_p)))}/{len(defined)}' if tested_p else f'0/{len(defined)}'
|
|
1016
|
+
oneof = stats.get('oneof_count', 0)
|
|
1017
|
+
weak = f'⚠️ {oneof}' if oneof > 0 else '—'
|
|
1018
|
+
fivexx = stats.get('count_5xx', 0)
|
|
1019
|
+
fivexx_str = f'🔴 {fivexx}' if fivexx > 0 else '—'
|
|
1020
|
+
icon = '❌' if tc == 0 else '⚠️' if tc < 3 else '✅'
|
|
1021
|
+
style = 'background:#fef2f2;' if tc == 0 else ''
|
|
1022
|
+
return f'<tr style="{style}"><td class="status-icon">{icon}</td><td><span class="method method-{ep["method"]}">{ep["method"]}</span></td><td class="left"><span class="path">{ep["path"]}</span></td><td>{tc}</td><td class="status-icon">{success_icon}</td><td>{codes_str}</td><td>{body}</td><td>{qp_str}</td><td>{weak}</td><td>{fivexx_str}</td></tr>'
|
|
1023
|
+
|
|
1024
|
+
endpoint_rows = [endpoint_row(ep, endpoint_stats.get(f"{ep['method']} {ep['path']}", {})) for ep in endpoints]
|
|
1025
|
+
details_html = f'''<div class="section"><div class="section-title">Endpoint Details<span class="count">{len(endpoints)} endpoints</span></div>
|
|
1026
|
+
<table><thead><tr><th style="width:40px;">Status</th><th style="width:70px;">Method</th><th class="left">Endpoint</th><th style="width:50px;">Tests</th><th style="width:50px;">2xx</th><th style="width:120px;">Status Codes</th><th style="width:70px;">Body</th><th style="width:70px;">Query</th><th style="width:70px;">Weak</th><th style="width:70px;">5xx</th></tr></thead>
|
|
1027
|
+
<tbody>{''.join(endpoint_rows)}</tbody></table></div>'''
|
|
1028
|
+
|
|
1029
|
+
# Collapsible sections
|
|
1030
|
+
qp_rows = [f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td><code>{", ".join(sorted(ep["params"]))}</code></td><td>{ep["total_values"]}</td></tr>' for ep in endpoints_with_params]
|
|
1031
|
+
qp_html = collapsible('Query Parameter Details', f'{len(endpoints_with_params)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Parameters Tested','style':''},{'label':'Value Variations','style':''}], qp_rows)
|
|
1032
|
+
|
|
1033
|
+
body_rows = [f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td>{len(ep["fields"])} fields</td><td>{ep["variations"]}</td></tr>' for ep in endpoints_with_body]
|
|
1034
|
+
body_detail_html = collapsible('Request Body Coverage', f'{len(endpoints_with_body)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Fields Tested','style':''},{'label':'Variations','style':''}], body_rows)
|
|
1035
|
+
|
|
1036
|
+
def include_row(ep):
|
|
1037
|
+
vals = ', '.join(sorted(ep['values'])) or 'None'
|
|
1038
|
+
status = '✅' if ep['tested'] else '❌'
|
|
1039
|
+
return f'<tr><td class="left"><span class="method method-{ep["method"]}">{ep["method"]}</span> <span class="path">{ep["path"]}</span></td><td>{status}</td><td><code>{vals}</code></td><td>{ep["variations"]}</td></tr>'
|
|
1040
|
+
|
|
1041
|
+
include_rows = [include_row(ep) for ep in endpoints_with_include]
|
|
1042
|
+
include_html = collapsible('Include Parameter Details', f'{len(endpoints_with_include)} endpoints', [{'label':'Endpoint','style':'text-align:left;'},{'label':'Tested','style':''},{'label':'Values','style':''},{'label':'Variations','style':''}], include_rows) if endpoints_with_include else ''
|
|
1043
|
+
|
|
1044
|
+
skip_rows = [f'<tr><td class="left"><span class="path">{s["collection"]}: {s["name"]}</span></td><td>{(s.get("env") or "unknown").upper()}</td><td class="left" style="color:{"var(--color-error)" if not s.get("reason") else "inherit"};">{s.get("reason") or "Missing reason"}</td></tr>' for s in all_skipped_tests]
|
|
1045
|
+
skip_html = collapsible('Environment-Conditional Skips', f'{total_skipped_tests} tests', [{'label':'Test','style':'text-align:left;'},{'label':'Skipped In','style':''},{'label':'Reason','style':'text-align:left;'}], skip_rows) if total_skipped_tests > 0 else ''
|
|
1046
|
+
|
|
1047
|
+
# Assemble full HTML
|
|
1048
|
+
content = summary_html + gaps_html + details_html + qp_html + body_detail_html + include_html + skip_html
|
|
1049
|
+
|
|
1050
|
+
html = f'''<!DOCTYPE html>
|
|
1051
|
+
<html lang="en">
|
|
1052
|
+
<head>
|
|
1053
|
+
<meta charset="UTF-8">
|
|
1054
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
1055
|
+
<title>API Coverage Report | freshapps_api_node</title>
|
|
1056
|
+
<style>{CSS}</style>
|
|
1057
|
+
</head>
|
|
1058
|
+
<body>
|
|
1059
|
+
<div class="container">
|
|
1060
|
+
<div class="header">
|
|
1061
|
+
<div class="header-left">
|
|
1062
|
+
<div class="logo">freshapps_api_node</div>
|
|
1063
|
+
<h1>API Test Coverage Report</h1>
|
|
1064
|
+
<p class="timestamp">Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}</p>
|
|
1065
|
+
</div>
|
|
1066
|
+
<div class="health-badge {health_class}">{health_icon} {health_label}</div>
|
|
1067
|
+
</div>
|
|
1068
|
+
{content}
|
|
1069
|
+
<footer>API Test Coverage Report • Generated by Node.js API Coverage Matrix Generator • {total_tests} test cases analyzed</footer>
|
|
1070
|
+
</div>
|
|
1071
|
+
</body>
|
|
1072
|
+
</html>'''
|
|
1073
|
+
|
|
1074
|
+
# Write HTML
|
|
1075
|
+
with open(HTML_OUTPUT_FILE, 'w', encoding='utf-8') as f:
|
|
1076
|
+
f.write(html)
|
|
1077
|
+
|
|
1078
|
+
# Write JSON summary
|
|
1079
|
+
json_output_file = HTML_OUTPUT_FILE.replace('.html', '.json')
|
|
1080
|
+
|
|
1081
|
+
method_stats = {}
|
|
1082
|
+
gaps = []
|
|
1083
|
+
for ep in endpoints:
|
|
1084
|
+
m = ep['method']
|
|
1085
|
+
if m not in method_stats:
|
|
1086
|
+
method_stats[m] = {'tested': 0, 'total': 0}
|
|
1087
|
+
method_stats[m]['total'] += 1
|
|
1088
|
+
key = f"{ep['method']} {ep['path']}"
|
|
1089
|
+
if has_success_code(endpoint_stats.get(key, {}).get('status_codes', set())):
|
|
1090
|
+
method_stats[m]['tested'] += 1
|
|
1091
|
+
else:
|
|
1092
|
+
gaps.append({'method': ep['method'], 'path': ep['path'], 'file': ep.get('file', '')})
|
|
1093
|
+
|
|
1094
|
+
coverage_summary = {
|
|
1095
|
+
'coverage_percent': round(success_pct, 1),
|
|
1096
|
+
'tested': success_tested,
|
|
1097
|
+
'total': len(endpoints),
|
|
1098
|
+
'well_tested_percent': round(well_tested_pct, 1),
|
|
1099
|
+
'query_params_percent': round(qp_pct, 1),
|
|
1100
|
+
'oneof_tests': total_oneof_tests,
|
|
1101
|
+
'fivexx_tests': total_5xx_tests,
|
|
1102
|
+
'skipped_tests': total_skipped_tests,
|
|
1103
|
+
'undocumented_skips': skips_without_reason,
|
|
1104
|
+
'gaps': gaps[:20],
|
|
1105
|
+
'by_method': method_stats,
|
|
1106
|
+
'full_report': os.path.basename(HTML_OUTPUT_FILE)
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
with open(json_output_file, 'w', encoding='utf-8') as f:
|
|
1110
|
+
json.dump(coverage_summary, f, indent=2)
|
|
1111
|
+
|
|
1112
|
+
# Console output
|
|
1113
|
+
print(f"\nEndpoints in Code: {len(endpoints)}", flush=True)
|
|
1114
|
+
print(f"Total Test Cases: {total_tests}", flush=True)
|
|
1115
|
+
print(f"Success Path (2xx) Tested: {success_pct:.1f}% ({success_tested}/{len(endpoints)})", flush=True)
|
|
1116
|
+
print(f"Well Tested (>=3 tests): {well_tested_pct:.1f}% ({well_tested}/{len(endpoints)})", flush=True)
|
|
1117
|
+
print(f"Response Body Validated: {body_pct:.1f}% ({with_body}/{len(endpoints)})", flush=True)
|
|
1118
|
+
print(f"Query Params Tested: {qp_pct:.1f}% ({total_tested_params}/{total_defined_params})", flush=True)
|
|
1119
|
+
print(f"Auth (401) Coverage: {auth_pct:.1f}% ({auth_tested}/{len(endpoints)})", flush=True)
|
|
1120
|
+
if error_only > 0:
|
|
1121
|
+
print(f" Error-Only Tests (no 2xx): {error_only} endpoints", flush=True)
|
|
1122
|
+
if total_oneof_tests > 0:
|
|
1123
|
+
print(f" oneOf Tests (weak assertions): {total_oneof_tests} tests in {oneof_endpoints} endpoints", flush=True)
|
|
1124
|
+
if total_5xx_tests > 0:
|
|
1125
|
+
print(f" 5xx Assertions (bad practice): {total_5xx_tests} tests in {endpoints_with_5xx} endpoints", flush=True)
|
|
1126
|
+
if total_skipped_tests > 0:
|
|
1127
|
+
print(f" Environment-conditional skips: {total_skipped_tests} tests", flush=True)
|
|
1128
|
+
for env, skips in sorted(skips_by_env.items(), key=lambda x: x[0] or 'zzz'):
|
|
1129
|
+
print(f" - {env or 'unknown'}: {len(skips)} tests", flush=True)
|
|
1130
|
+
if skips_without_reason > 0:
|
|
1131
|
+
print(f" Undocumented skips: {skips_without_reason} tests (add [SKIP] Reason: ... | Env: ...)", flush=True)
|
|
1132
|
+
if errors:
|
|
1133
|
+
print(f"\nWarnings ({len(errors)}):", flush=True)
|
|
1134
|
+
for err in errors[:5]:
|
|
1135
|
+
print(f" - {err}", flush=True)
|
|
1136
|
+
if len(errors) > 5:
|
|
1137
|
+
print(f" ... and {len(errors) - 5} more", flush=True)
|