create-backlist 10.0.3 → 10.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/qa.js +138 -183
- package/package.json +6 -1
- package/src/qa/analyzers/accessibility.js +81 -0
- package/src/qa/analyzers/api.js +125 -0
- package/src/qa/analyzers/performance.js +137 -0
- package/src/qa/analyzers/security.js +207 -0
- package/src/qa/analyzers/seo.js +248 -0
- package/src/qa/browser/crawler.js +223 -0
- package/src/qa/browser/interactions.js +317 -0
- package/src/qa/browser/screenshot.js +34 -0
- package/src/qa/qa-engine.js +756 -2936
- package/src/qa/reporters/html.js +623 -0
- package/src/qa/reporters/json.js +49 -0
- package/src/qa/reporters/terminal.js +184 -0
- package/src/qa/utils/ai-classifier.js +98 -0
package/src/qa/qa-engine.js
CHANGED
|
@@ -1,3092 +1,912 @@
|
|
|
1
1
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
2
|
-
// Backlist QA
|
|
2
|
+
// Backlist Enterprise AI QA Platform — qa-engine.js v12.0
|
|
3
3
|
// Copyright (c) W.A.H.ISHAN — MIT License
|
|
4
4
|
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// ✦ Zero limits on bug reports
|
|
8
|
-
// ✦ Deep file system scanner
|
|
9
|
-
// ✦ Full security audit suite
|
|
10
|
-
// ✦ Complete API contract validator
|
|
11
|
-
// ✦ Database schema deep validator
|
|
12
|
-
// ✦ Auth flow complete validator
|
|
13
|
-
// ✦ Performance profiler (all routes)
|
|
14
|
-
// ✦ Docker / CI / CD validator
|
|
15
|
-
// ✦ Dependency vulnerability scanner
|
|
16
|
-
// ✦ Code quality scanner
|
|
17
|
-
// ✦ Environment config validator
|
|
18
|
-
// ✦ TypeScript strict checker
|
|
19
|
-
// ✦ Package.json deep validator
|
|
20
|
-
// ✦ Error handling validator
|
|
21
|
-
// ✦ Logging infrastructure checker
|
|
22
|
-
// ✦ CORS + Rate limit deep checker
|
|
23
|
-
// ✦ HTTP probe (all COMMON_ROUTES)
|
|
24
|
-
// ✦ SEO full suite per route
|
|
25
|
-
// ✦ A11y full suite
|
|
26
|
-
// ✦ Security headers complete scan
|
|
27
|
-
// ✦ Broken link detector
|
|
28
|
-
// ✦ Mobile/responsive checker
|
|
29
|
-
// ✦ JSON schema validator
|
|
30
|
-
// ✦ Unlimited parallel bug accumulation
|
|
5
|
+
// REAL RUNTIME TESTING — NO FAKE DATA
|
|
6
|
+
// Every result is collected from actual browser execution
|
|
31
7
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
32
8
|
|
|
33
|
-
import * as p
|
|
34
|
-
import chalk
|
|
35
|
-
import fs
|
|
36
|
-
import path
|
|
37
|
-
import os
|
|
38
|
-
import
|
|
39
|
-
import
|
|
40
|
-
import { URL } from 'node:url';
|
|
41
|
-
import { EventEmitter } from 'node:events';
|
|
42
|
-
import { performance } from 'node:perf_hooks';
|
|
43
|
-
|
|
44
|
-
// ── Constants ─────────────────────────────────────────────────────────────
|
|
45
|
-
const VERSION = '11.0.0';
|
|
46
|
-
const QA_DIR = path.join(process.cwd(), '.backlist', 'qa');
|
|
47
|
-
const HISTORY_FILE = path.join(QA_DIR, 'history.json');
|
|
48
|
-
const REPORT_DIR = path.join(QA_DIR, 'reports');
|
|
49
|
-
|
|
50
|
-
const SEVERITY_LEVELS = { P0: 'Critical', P1: 'High', P2: 'Medium', P3: 'Low' };
|
|
51
|
-
const TEST_TYPES = [
|
|
52
|
-
'happy-path','validation','auth','edge-case','performance',
|
|
53
|
-
'security','e2e','ui','seo','a11y','links','http',
|
|
54
|
-
'database','docker','ci','code-quality','typescript',
|
|
55
|
-
'environment','logging','error-handling','api-contract',
|
|
56
|
-
'dependency','cors','rate-limit','file-structure',
|
|
57
|
-
];
|
|
58
|
-
const DEFAULT_TIMEOUT_MS = 20_000;
|
|
59
|
-
const HTTP_TIMEOUT_MS = 10_000;
|
|
60
|
-
const FLAKY_RETRY_COUNT = 3;
|
|
61
|
-
const WATCH_INTERVAL_MS = 30_000;
|
|
62
|
-
|
|
63
|
-
// ── ALL routes to probe (expanded) ────────────────────────────────────────
|
|
64
|
-
const COMMON_ROUTES = [
|
|
65
|
-
'/', '/login', '/register', '/dashboard', '/dashboard/analytics',
|
|
66
|
-
'/dashboard/sales', '/dashboard/reports', '/dashboard/settings',
|
|
67
|
-
'/profile', '/settings', '/settings/account', '/settings/security',
|
|
68
|
-
'/admin', '/admin/users', '/admin/dashboard', '/admin/reports',
|
|
69
|
-
'/about', '/contact', '/pricing', '/faq', '/help', '/support',
|
|
70
|
-
'/api/health', '/api/status', '/api/v1/health', '/api/v1/status',
|
|
71
|
-
'/api/v1/users', '/api/v1/products', '/api/v1/orders',
|
|
72
|
-
'/api/v2/health', '/api/docs', '/api/swagger',
|
|
73
|
-
'/sitemap.xml', '/robots.txt', '/favicon.ico', '/manifest.json',
|
|
74
|
-
'/.well-known/security.txt',
|
|
75
|
-
];
|
|
76
|
-
|
|
77
|
-
// ── Security headers ───────────────────────────────────────────────────────
|
|
78
|
-
const SECURITY_HEADERS = [
|
|
79
|
-
{ header: 'content-security-policy', label: 'CSP', sev: 'P1' },
|
|
80
|
-
{ header: 'x-frame-options', label: 'X-Frame-Options', sev: 'P1' },
|
|
81
|
-
{ header: 'x-content-type-options', label: 'X-Content-Type', sev: 'P2' },
|
|
82
|
-
{ header: 'strict-transport-security', label: 'HSTS', sev: 'P1' },
|
|
83
|
-
{ header: 'referrer-policy', label: 'Referrer-Policy', sev: 'P2' },
|
|
84
|
-
{ header: 'permissions-policy', label: 'Permissions', sev: 'P3' },
|
|
85
|
-
{ header: 'access-control-allow-origin', label: 'CORS', sev: 'P2' },
|
|
86
|
-
{ header: 'x-xss-protection', label: 'XSS-Protection', sev: 'P2' },
|
|
87
|
-
{ header: 'cross-origin-embedder-policy', label: 'COEP', sev: 'P3' },
|
|
88
|
-
{ header: 'cross-origin-opener-policy', label: 'COOP', sev: 'P3' },
|
|
89
|
-
{ header: 'cross-origin-resource-policy', label: 'CORP', sev: 'P3' },
|
|
90
|
-
{ header: 'cache-control', label: 'Cache-Control', sev: 'P3' },
|
|
91
|
-
];
|
|
92
|
-
|
|
93
|
-
// ── Dangerous packages ─────────────────────────────────────────────────────
|
|
94
|
-
const DANGEROUS_PACKAGES = [
|
|
95
|
-
'eval','vm2','node-serialize','serialize-javascript',
|
|
96
|
-
'shelljs','child_process','exec','execSync',
|
|
97
|
-
];
|
|
98
|
-
|
|
99
|
-
// ── Known vulnerable package patterns ─────────────────────────────────────
|
|
100
|
-
const VULN_PATTERNS = [
|
|
101
|
-
{ name: 'lodash', below: '4.17.21', reason: 'Prototype pollution' },
|
|
102
|
-
{ name: 'moment', below: '2.29.4', reason: 'ReDoS vulnerability' },
|
|
103
|
-
{ name: 'axios', below: '1.6.0', reason: 'SSRF vulnerability' },
|
|
104
|
-
{ name: 'jsonwebtoken', below: '9.0.0', reason: 'Algorithm confusion' },
|
|
105
|
-
];
|
|
106
|
-
|
|
107
|
-
// ── ANSI helpers ──────────────────────────────────────────────────────────
|
|
108
|
-
const ESC = '\x1b[';
|
|
109
|
-
const CLEAR_LINE = ESC + '2K\r';
|
|
110
|
-
const CURSOR_UP = (n) => ESC + `${n}A`;
|
|
111
|
-
const CURSOR_HIDE = ESC + '?25l';
|
|
112
|
-
const CURSOR_SHOW = ESC + '?25h';
|
|
113
|
-
const BOLD = chalk.bold;
|
|
114
|
-
const DIM = chalk.dim;
|
|
115
|
-
|
|
116
|
-
// ── Utilities ─────────────────────────────────────────────────────────────
|
|
117
|
-
function timestamp() { return new Date().toISOString(); }
|
|
118
|
-
function shortId() { return Math.random().toString(36).slice(2, 9); }
|
|
119
|
-
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
120
|
-
function pluralize(n, w) { return `${n} ${n === 1 ? w : w + 's'}`; }
|
|
121
|
-
|
|
122
|
-
function colorSeverity(sev) {
|
|
123
|
-
return ({
|
|
124
|
-
P0: chalk.red.bold,
|
|
125
|
-
P1: chalk.yellow.bold,
|
|
126
|
-
P2: chalk.cyan,
|
|
127
|
-
P3: chalk.gray,
|
|
128
|
-
}[sev] ?? chalk.white)(sev);
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
function colorStatus(status) {
|
|
132
|
-
return ({
|
|
133
|
-
PASS : chalk.green('✓ PASS'),
|
|
134
|
-
FAIL : chalk.red('✗ FAIL'),
|
|
135
|
-
SKIP : chalk.gray('⊘ SKIP'),
|
|
136
|
-
FLAKY : chalk.yellow('⚠ FLAKY'),
|
|
137
|
-
RUN : chalk.cyan('⟳ RUN'),
|
|
138
|
-
})[status] ?? status;
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
function buildProgressBar(pct, width = 20) {
|
|
142
|
-
const filled = Math.min(Math.round((pct / 100) * width), width);
|
|
143
|
-
const empty = width - filled;
|
|
144
|
-
const color = pct >= 90 ? chalk.green : pct >= 70 ? chalk.yellow : chalk.red;
|
|
145
|
-
return color('█'.repeat(filled)) + chalk.gray('░'.repeat(empty));
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
function formatDuration(ms) {
|
|
149
|
-
if (ms < 1000) return `${ms}ms`;
|
|
150
|
-
return `${(ms / 1000).toFixed(2)}s`;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
function formatBytes(b) {
|
|
154
|
-
if (b < 1024) return `${b}B`;
|
|
155
|
-
if (b < 1024 * 1024) return `${(b / 1024).toFixed(1)}KB`;
|
|
156
|
-
return `${(b / 1024 / 1024).toFixed(1)}MB`;
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
function getSystemStats() {
|
|
160
|
-
const mem = process.memoryUsage();
|
|
161
|
-
const heapMB = (mem.heapUsed / 1024 / 1024).toFixed(1);
|
|
162
|
-
const rss = formatBytes(mem.rss);
|
|
163
|
-
const uptime = process.uptime().toFixed(1);
|
|
164
|
-
return { heapMB, rss, uptime };
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
// semver compare (simple)
|
|
168
|
-
function semverLt(a, b) {
|
|
169
|
-
if (!a || !b) return false;
|
|
170
|
-
const pa = a.replace(/[^0-9.]/g, '').split('.').map(Number);
|
|
171
|
-
const pb = b.replace(/[^0-9.]/g, '').split('.').map(Number);
|
|
172
|
-
for (let i = 0; i < 3; i++) {
|
|
173
|
-
if ((pa[i] || 0) < (pb[i] || 0)) return true;
|
|
174
|
-
if ((pa[i] || 0) > (pb[i] || 0)) return false;
|
|
175
|
-
}
|
|
176
|
-
return false;
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
180
|
-
// HTTP Probe Engine
|
|
181
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
182
|
-
export class HttpProbe {
|
|
183
|
-
#baseUrl;
|
|
184
|
-
|
|
185
|
-
constructor(baseUrl) {
|
|
186
|
-
this.#baseUrl = baseUrl.replace(/\/$/, '');
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
async fetch(route = '/', options = {}) {
|
|
190
|
-
const url = this.#baseUrl + route;
|
|
191
|
-
const t0 = performance.now();
|
|
192
|
-
try {
|
|
193
|
-
const controller = new AbortController();
|
|
194
|
-
const timer = setTimeout(() => controller.abort(), HTTP_TIMEOUT_MS);
|
|
195
|
-
const res = await fetch(url, {
|
|
196
|
-
signal : controller.signal,
|
|
197
|
-
headers: { 'User-Agent': 'Backlist-QA/11.0', ...options.headers },
|
|
198
|
-
method : options.method || 'GET',
|
|
199
|
-
redirect: 'follow',
|
|
200
|
-
...(options.body ? { body: options.body } : {}),
|
|
201
|
-
});
|
|
202
|
-
clearTimeout(timer);
|
|
203
|
-
const duration = Math.round(performance.now() - t0);
|
|
204
|
-
const headers = {};
|
|
205
|
-
res.headers.forEach((v, k) => { headers[k] = v; });
|
|
206
|
-
const text = options.readBody ? await res.text().catch(() => '') : '';
|
|
207
|
-
return { ok: true, status: res.status, headers, duration, url, text };
|
|
208
|
-
} catch (err) {
|
|
209
|
-
const duration = Math.round(performance.now() - t0);
|
|
210
|
-
return { ok: false, status: 0, headers: {}, duration, url, error: err.message };
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
async probeRoutes(routes = COMMON_ROUTES) {
|
|
215
|
-
const results = [];
|
|
216
|
-
for (const route of routes) {
|
|
217
|
-
const r = await this.fetch(route, { readBody: true });
|
|
218
|
-
results.push({ route, ...r });
|
|
219
|
-
}
|
|
220
|
-
return results;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
async checkSecurityHeaders(route = '/') {
|
|
224
|
-
const r = await this.fetch(route);
|
|
225
|
-
if (!r.ok && r.status === 0) return { ok: false, results: [], error: r.error };
|
|
226
|
-
const results = SECURITY_HEADERS.map(({ header, label, sev }) => ({
|
|
227
|
-
header, label, sev,
|
|
228
|
-
present: header in r.headers,
|
|
229
|
-
value : r.headers[header] || null,
|
|
230
|
-
}));
|
|
231
|
-
return { ok: true, results };
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
async benchmarkRoute(route = '/', samples = 8) {
|
|
235
|
-
const timings = [];
|
|
236
|
-
for (let i = 0; i < samples; i++) {
|
|
237
|
-
const r = await this.fetch(route);
|
|
238
|
-
if (r.ok || r.status > 0) timings.push(r.duration);
|
|
239
|
-
await sleep(80);
|
|
240
|
-
}
|
|
241
|
-
if (!timings.length) return { p50: 0, p95: 0, p99: 0, avg: 0, min: 0, max: 0, samples: 0 };
|
|
242
|
-
timings.sort((a, b) => a - b);
|
|
243
|
-
const p = (pct) => timings[Math.min(Math.floor(timings.length * pct / 100), timings.length - 1)];
|
|
244
|
-
return {
|
|
245
|
-
p50 : p(50),
|
|
246
|
-
p95 : p(95),
|
|
247
|
-
p99 : p(99),
|
|
248
|
-
avg : Math.round(timings.reduce((a, b) => a + b, 0) / timings.length),
|
|
249
|
-
min : timings[0],
|
|
250
|
-
max : timings[timings.length - 1],
|
|
251
|
-
samples: timings.length,
|
|
252
|
-
};
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
async checkSEO(route = '/') {
|
|
256
|
-
const r = await this.fetch(route, { readBody: true });
|
|
257
|
-
if (!r.ok && r.status === 0) return { ok: false, checks: [] };
|
|
258
|
-
const html = r.text || '';
|
|
259
|
-
const checks = [
|
|
260
|
-
{ name: 'Title tag', pass: /<title[^>]*>[^<]+<\/title>/i.test(html), sev: 'P1' },
|
|
261
|
-
{ name: 'Meta description', pass: /<meta[^>]+name=["']description["'][^>]*content/i.test(html), sev: 'P2' },
|
|
262
|
-
{ name: 'H1 tag', pass: /<h1[^>]*>[^<]+<\/h1>/i.test(html), sev: 'P1' },
|
|
263
|
-
{ name: 'Viewport meta', pass: /<meta[^>]+name=["']viewport["'][^>]*>/i.test(html), sev: 'P1' },
|
|
264
|
-
{ name: 'Lang attribute', pass: /<html[^>]+lang=["'][^"']+["']/i.test(html), sev: 'P2' },
|
|
265
|
-
{ name: 'Canonical link', pass: /<link[^>]+rel=["']canonical["'][^>]*>/i.test(html), sev: 'P2' },
|
|
266
|
-
{ name: 'OG title', pass: /<meta[^>]+property=["']og:title["'][^>]*>/i.test(html), sev: 'P3' },
|
|
267
|
-
{ name: 'OG description', pass: /<meta[^>]+property=["']og:description["'][^>]*>/i.test(html), sev: 'P3' },
|
|
268
|
-
{ name: 'OG image', pass: /<meta[^>]+property=["']og:image["'][^>]*>/i.test(html), sev: 'P3' },
|
|
269
|
-
{ name: 'Twitter card', pass: /<meta[^>]+name=["']twitter:card["'][^>]*>/i.test(html), sev: 'P3' },
|
|
270
|
-
{ name: 'Structured data', pass: /application\/ld\+json/i.test(html), sev: 'P3' },
|
|
271
|
-
{ name: 'Alt attributes', pass: !/<img(?![^>]*alt=)[^>]*>/i.test(html), sev: 'P2' },
|
|
272
|
-
{ name: 'No broken meta', pass: !/<meta[^>]*content=["']["']/i.test(html), sev: 'P2' },
|
|
273
|
-
];
|
|
274
|
-
return { ok: true, checks, statusCode: r.status };
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
async checkA11y(route = '/') {
|
|
278
|
-
const r = await this.fetch(route, { readBody: true });
|
|
279
|
-
if (!r.ok && r.status === 0) return { ok: false, checks: [] };
|
|
280
|
-
const html = r.text || '';
|
|
281
|
-
const checks = [
|
|
282
|
-
{ name: 'html[lang] set', pass: /<html[^>]+lang=["'][^"']+["']/i.test(html), sev: 'P1' },
|
|
283
|
-
{ name: 'Viewport meta', pass: /<meta[^>]+name=["']viewport["'][^>]*>/i.test(html), sev: 'P1' },
|
|
284
|
-
{ name: 'Skip nav link', pass: /skip.*nav|skip.*content|skip.*main/i.test(html), sev: 'P2' },
|
|
285
|
-
{ name: 'Main landmark', pass: /<main[^>]*>/i.test(html), sev: 'P2' },
|
|
286
|
-
{ name: 'No images without alt', pass: !/<img(?![^>]*alt=)[^>]*>/i.test(html), sev: 'P1' },
|
|
287
|
-
{ name: 'Form labels present', pass: !/<input(?![^>]*aria-label)(?![^>]*id=)[^>]*>/i.test(html), sev: 'P2' },
|
|
288
|
-
{ name: 'Heading hierarchy', pass: /<h1/i.test(html), sev: 'P2' },
|
|
289
|
-
{ name: 'No autofocus abuse', pass: (html.match(/autofocus/gi) || []).length <= 1, sev: 'P3' },
|
|
290
|
-
{ name: 'ARIA roles used', pass: /role=/i.test(html), sev: 'P3' },
|
|
291
|
-
{ name: 'Focus visible (class)', pass: /focus-visible|focus:ring|focus:outline/i.test(html), sev: 'P2' },
|
|
292
|
-
];
|
|
293
|
-
return { ok: true, checks };
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
get baseUrl() { return this.#baseUrl; }
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
300
|
-
// MAXIMUM URL Test Suite Builder (100+ HTTP tests)
|
|
301
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
302
|
-
function buildUrlTestSuite(probe, label = 'target') {
|
|
303
|
-
const tests = [];
|
|
304
|
-
const t = (name, type, sev, fn) =>
|
|
305
|
-
tests.push({ id: shortId(), name: `[${label}] ${name}`, type, sev, fn });
|
|
306
|
-
|
|
307
|
-
// ── Connectivity (10 tests) ────────────────────────────────────────────
|
|
308
|
-
t('Homepage reachable', 'http', 'P0', async () => {
|
|
309
|
-
const r = await probe.fetch('/');
|
|
310
|
-
if (!r.ok && r.status === 0) throw new Error(`Connection failed: ${r.error}`);
|
|
311
|
-
if (r.status >= 500) throw new Error(`Server error: HTTP ${r.status}`);
|
|
312
|
-
});
|
|
313
|
-
|
|
314
|
-
t('Homepage returns 2xx or 3xx', 'http', 'P0', async () => {
|
|
315
|
-
const r = await probe.fetch('/');
|
|
316
|
-
if (r.status >= 400) throw new Error(`Unexpected status: ${r.status}`);
|
|
317
|
-
});
|
|
318
|
-
|
|
319
|
-
t('API health endpoint', 'http', 'P0', async () => {
|
|
320
|
-
const candidates = ['/api/health', '/api/status', '/api/v1/health', '/health', '/ping'];
|
|
321
|
-
let found = false;
|
|
322
|
-
for (const c of candidates) {
|
|
323
|
-
const r = await probe.fetch(c);
|
|
324
|
-
if (r.status >= 200 && r.status < 400) { found = true; break; }
|
|
325
|
-
}
|
|
326
|
-
if (!found) throw new Error('No reachable API health endpoint found');
|
|
327
|
-
});
|
|
328
|
-
|
|
329
|
-
t('404 handler works', 'http', 'P1', async () => {
|
|
330
|
-
const r = await probe.fetch('/this-route-does-not-exist-' + shortId());
|
|
331
|
-
if (r.status !== 404) throw new Error(`Expected 404, got ${r.status}`);
|
|
332
|
-
});
|
|
333
|
-
|
|
334
|
-
t('405 Method Not Allowed', 'http', 'P2', async () => {
|
|
335
|
-
const r = await probe.fetch('/', { method: 'DELETE' });
|
|
336
|
-
if (r.status === 200) throw new Error('DELETE / should not return 200');
|
|
337
|
-
});
|
|
338
|
-
|
|
339
|
-
t('HEAD request supported', 'http', 'P3', async () => {
|
|
340
|
-
const r = await probe.fetch('/', { method: 'HEAD' });
|
|
341
|
-
if (r.status >= 500) throw new Error(`HEAD / returned ${r.status}`);
|
|
342
|
-
});
|
|
343
|
-
|
|
344
|
-
t('OPTIONS request supported', 'http', 'P3', async () => {
|
|
345
|
-
const r = await probe.fetch('/', { method: 'OPTIONS' });
|
|
346
|
-
if (r.status >= 500) throw new Error(`OPTIONS returned server error ${r.status}`);
|
|
347
|
-
});
|
|
348
|
-
|
|
349
|
-
t('No directory listing exposed', 'http', 'P0', async () => {
|
|
350
|
-
const r = await probe.fetch('/static', { readBody: true });
|
|
351
|
-
if (r.text && /index of\//i.test(r.text)) throw new Error('Directory listing exposed at /static');
|
|
352
|
-
});
|
|
353
|
-
|
|
354
|
-
t('No server version in header', 'http', 'P1', async () => {
|
|
355
|
-
const r = await probe.fetch('/');
|
|
356
|
-
const server = r.headers['server'] || '';
|
|
357
|
-
if (/apache\//i.test(server) || /nginx\//i.test(server) || /express/i.test(server)) {
|
|
358
|
-
throw new Error(`Server version exposed: ${server}`);
|
|
359
|
-
}
|
|
360
|
-
});
|
|
361
|
-
|
|
362
|
-
t('No X-Powered-By header', 'http', 'P1', async () => {
|
|
363
|
-
const r = await probe.fetch('/');
|
|
364
|
-
if (r.headers['x-powered-by']) throw new Error(`X-Powered-By exposed: ${r.headers['x-powered-by']}`);
|
|
365
|
-
});
|
|
366
|
-
|
|
367
|
-
// ── Security Headers (12 tests) ────────────────────────────────────────
|
|
368
|
-
t('CSP header present', 'security', 'P1', async () => {
|
|
369
|
-
const r = await probe.fetch('/');
|
|
370
|
-
if (!r.headers['content-security-policy']) throw new Error('Missing Content-Security-Policy header');
|
|
371
|
-
});
|
|
372
|
-
|
|
373
|
-
t('HSTS header present', 'security', 'P1', async () => {
|
|
374
|
-
const r = await probe.fetch('/');
|
|
375
|
-
if (!r.headers['strict-transport-security']) throw new Error('Missing Strict-Transport-Security header');
|
|
376
|
-
});
|
|
377
|
-
|
|
378
|
-
t('X-Frame-Options present', 'security', 'P1', async () => {
|
|
379
|
-
const r = await probe.fetch('/');
|
|
380
|
-
const xfo = r.headers['x-frame-options'];
|
|
381
|
-
if (!xfo) throw new Error('Missing X-Frame-Options header — clickjacking risk');
|
|
382
|
-
if (!['DENY','SAMEORIGIN'].includes(xfo.toUpperCase()))
|
|
383
|
-
throw new Error(`X-Frame-Options value invalid: ${xfo}`);
|
|
384
|
-
});
|
|
385
|
-
|
|
386
|
-
t('X-Content-Type-Options present', 'security', 'P2', async () => {
|
|
387
|
-
const r = await probe.fetch('/');
|
|
388
|
-
if (r.headers['x-content-type-options'] !== 'nosniff')
|
|
389
|
-
throw new Error('Missing or incorrect X-Content-Type-Options header');
|
|
390
|
-
});
|
|
391
|
-
|
|
392
|
-
t('Referrer-Policy present', 'security', 'P2', async () => {
|
|
393
|
-
const r = await probe.fetch('/');
|
|
394
|
-
if (!r.headers['referrer-policy']) throw new Error('Missing Referrer-Policy header');
|
|
395
|
-
});
|
|
396
|
-
|
|
397
|
-
t('All security headers scan', 'security', 'P1', async () => {
|
|
398
|
-
const scan = await probe.checkSecurityHeaders('/');
|
|
399
|
-
if (!scan.ok) throw new Error(`Could not reach server: ${scan.error}`);
|
|
400
|
-
const critical = scan.results.filter(r => !r.present && (r.sev === 'P0' || r.sev === 'P1'));
|
|
401
|
-
if (critical.length > 0)
|
|
402
|
-
throw new Error(`Missing critical headers: ${critical.map(r => r.label).join(', ')}`);
|
|
403
|
-
});
|
|
404
|
-
|
|
405
|
-
t('HTTPS redirect enforced', 'security', 'P1', async () => {
|
|
406
|
-
if (probe.baseUrl.startsWith('http://')) {
|
|
407
|
-
const r = await probe.fetch('/');
|
|
408
|
-
if (r.status === 200 && !r.url?.startsWith('https'))
|
|
409
|
-
throw new Error('HTTP serving without HTTPS redirect');
|
|
410
|
-
}
|
|
411
|
-
});
|
|
412
|
-
|
|
413
|
-
t('No sensitive data in headers', 'security', 'P0', async () => {
|
|
414
|
-
const r = await probe.fetch('/');
|
|
415
|
-
const dangerous = ['authorization','cookie','set-cookie'];
|
|
416
|
-
for (const h of dangerous) {
|
|
417
|
-
if (r.headers[h] && r.headers[h].includes('secret'))
|
|
418
|
-
throw new Error(`Sensitive data in ${h} header`);
|
|
419
|
-
}
|
|
420
|
-
});
|
|
421
|
-
|
|
422
|
-
t('CORS header not wildcard on auth routes', 'security', 'P1', async () => {
|
|
423
|
-
const r = await probe.fetch('/api/health');
|
|
424
|
-
const cors = r.headers['access-control-allow-origin'];
|
|
425
|
-
if (cors === '*' && r.headers['access-control-allow-credentials'] === 'true')
|
|
426
|
-
throw new Error('CORS wildcard + credentials — security vulnerability');
|
|
427
|
-
});
|
|
428
|
-
|
|
429
|
-
t('No cache on API responses', 'security', 'P2', async () => {
|
|
430
|
-
const r = await probe.fetch('/api/health');
|
|
431
|
-
const cc = r.headers['cache-control'] || '';
|
|
432
|
-
if (r.status === 200 && !cc.includes('no-store') && !cc.includes('no-cache') && !cc.includes('private'))
|
|
433
|
-
throw new Error(`API response may be cached: cache-control: ${cc || '(missing)'}`);
|
|
434
|
-
});
|
|
435
|
-
|
|
436
|
-
t('XSS Protection header', 'security', 'P2', async () => {
|
|
437
|
-
const r = await probe.fetch('/');
|
|
438
|
-
const xss = r.headers['x-xss-protection'];
|
|
439
|
-
if (!xss) throw new Error('Missing X-XSS-Protection header');
|
|
440
|
-
});
|
|
441
|
-
|
|
442
|
-
t('Permissions-Policy present', 'security', 'P3', async () => {
|
|
443
|
-
const r = await probe.fetch('/');
|
|
444
|
-
if (!r.headers['permissions-policy']) throw new Error('Missing Permissions-Policy header');
|
|
445
|
-
});
|
|
446
|
-
|
|
447
|
-
// ── Authentication (12 tests) ──────────────────────────────────────────
|
|
448
|
-
t('Login page accessible', 'auth', 'P1', async () => {
|
|
449
|
-
const candidates = ['/login','/auth/login','/signin','/auth','/user/login','/account/login'];
|
|
450
|
-
for (const c of candidates) {
|
|
451
|
-
const r = await probe.fetch(c);
|
|
452
|
-
if (r.status >= 200 && r.status < 400) return;
|
|
453
|
-
}
|
|
454
|
-
throw new Error('No login page found at common paths');
|
|
455
|
-
});
|
|
456
|
-
|
|
457
|
-
t('Register page accessible', 'auth', 'P2', async () => {
|
|
458
|
-
const candidates = ['/register','/signup','/auth/register','/create-account','/join'];
|
|
459
|
-
for (const c of candidates) {
|
|
460
|
-
const r = await probe.fetch(c);
|
|
461
|
-
if (r.status >= 200 && r.status < 400) return;
|
|
462
|
-
}
|
|
463
|
-
throw new Error('No registration page found');
|
|
464
|
-
});
|
|
465
|
-
|
|
466
|
-
t('Dashboard protected', 'auth', 'P0', async () => {
|
|
467
|
-
const r = await probe.fetch('/dashboard', { readBody: true });
|
|
468
|
-
if (r.status === 200) {
|
|
469
|
-
const html = r.text || '';
|
|
470
|
-
if (!/login|signin|unauthorized|forbidden|redirect/i.test(html))
|
|
471
|
-
throw new Error('/dashboard accessible without auth (no redirect detected)');
|
|
472
|
-
}
|
|
473
|
-
});
|
|
474
|
-
|
|
475
|
-
t('Admin panel protected', 'auth', 'P0', async () => {
|
|
476
|
-
const r = await probe.fetch('/admin', { readBody: true });
|
|
477
|
-
if (r.status === 200) {
|
|
478
|
-
const html = r.text || '';
|
|
479
|
-
if (!/login|signin|unauthorized|forbidden|redirect/i.test(html))
|
|
480
|
-
throw new Error('/admin accessible without auth');
|
|
481
|
-
}
|
|
482
|
-
});
|
|
483
|
-
|
|
484
|
-
t('Profile page protected', 'auth', 'P0', async () => {
|
|
485
|
-
const r = await probe.fetch('/profile');
|
|
486
|
-
if (r.status === 200) throw new Error('/profile accessible without auth');
|
|
487
|
-
});
|
|
488
|
-
|
|
489
|
-
t('Settings page protected', 'auth', 'P0', async () => {
|
|
490
|
-
const r = await probe.fetch('/settings');
|
|
491
|
-
if (r.status === 200) throw new Error('/settings accessible without auth');
|
|
492
|
-
});
|
|
493
|
-
|
|
494
|
-
t('API requires auth token', 'auth', 'P0', async () => {
|
|
495
|
-
const r = await probe.fetch('/api/v1/users');
|
|
496
|
-
if (r.status === 200) throw new Error('/api/v1/users returns data without auth token');
|
|
497
|
-
});
|
|
498
|
-
|
|
499
|
-
t('Invalid token rejected', 'auth', 'P0', async () => {
|
|
500
|
-
const r = await probe.fetch('/api/v1/users', {
|
|
501
|
-
headers: { Authorization: 'Bearer invalid-token-xyz-123' },
|
|
502
|
-
});
|
|
503
|
-
if (r.status === 200) throw new Error('Invalid token accepted by API');
|
|
504
|
-
});
|
|
505
|
-
|
|
506
|
-
t('No JWT in URL params', 'auth', 'P1', async () => {
|
|
507
|
-
const r = await probe.fetch('/api/health?token=eyJtest');
|
|
508
|
-
const body = r.text || '';
|
|
509
|
-
if (/token.*ey[A-Za-z0-9]/i.test(r.url || ''))
|
|
510
|
-
throw new Error('JWT token appears to be accepted in URL param');
|
|
511
|
-
});
|
|
512
|
-
|
|
513
|
-
t('Password reset page exists', 'auth', 'P2', async () => {
|
|
514
|
-
const candidates = ['/forgot-password','/reset-password','/auth/forgot','/password-reset'];
|
|
515
|
-
for (const c of candidates) {
|
|
516
|
-
const r = await probe.fetch(c);
|
|
517
|
-
if (r.status >= 200 && r.status < 400) return;
|
|
518
|
-
}
|
|
519
|
-
throw new Error('No password reset page found');
|
|
520
|
-
});
|
|
521
|
-
|
|
522
|
-
t('Logout endpoint exists', 'auth', 'P2', async () => {
|
|
523
|
-
const candidates = ['/logout','/signout','/auth/logout','/api/auth/logout'];
|
|
524
|
-
for (const c of candidates) {
|
|
525
|
-
const r = await probe.fetch(c);
|
|
526
|
-
if (r.status < 500) return;
|
|
527
|
-
}
|
|
528
|
-
throw new Error('No logout endpoint found');
|
|
529
|
-
});
|
|
530
|
-
|
|
531
|
-
t('MFA/2FA page exists', 'auth', 'P3', async () => {
|
|
532
|
-
const candidates = ['/2fa','/mfa','/auth/2fa','/verify','/two-factor'];
|
|
533
|
-
for (const c of candidates) {
|
|
534
|
-
const r = await probe.fetch(c);
|
|
535
|
-
if (r.status < 500) return;
|
|
536
|
-
}
|
|
537
|
-
throw new Error('No 2FA/MFA page found — consider implementing MFA');
|
|
538
|
-
});
|
|
539
|
-
|
|
540
|
-
// ── Performance (10 tests) ─────────────────────────────────────────────
|
|
541
|
-
t('Homepage avg response < 2s', 'performance', 'P1', async () => {
|
|
542
|
-
const b = await probe.benchmarkRoute('/', 5);
|
|
543
|
-
if (b.avg > 2000) throw new Error(`Avg ${b.avg}ms exceeds 2000ms threshold`);
|
|
544
|
-
});
|
|
545
|
-
|
|
546
|
-
t('Homepage p95 < 3s', 'performance', 'P1', async () => {
|
|
547
|
-
const b = await probe.benchmarkRoute('/', 5);
|
|
548
|
-
if (b.p95 > 3000) throw new Error(`p95 ${b.p95}ms exceeds 3000ms`);
|
|
549
|
-
});
|
|
550
|
-
|
|
551
|
-
t('API health p50 < 500ms', 'performance', 'P1', async () => {
|
|
552
|
-
const candidates = ['/api/health','/api/status','/health'];
|
|
553
|
-
for (const c of candidates) {
|
|
554
|
-
const r = await probe.fetch(c);
|
|
555
|
-
if (r.status > 0) {
|
|
556
|
-
const b = await probe.benchmarkRoute(c, 5);
|
|
557
|
-
if (b.p50 > 500) throw new Error(`API p50 ${b.p50}ms on ${c} exceeds 500ms`);
|
|
558
|
-
return;
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
});
|
|
562
|
-
|
|
563
|
-
t('API health p95 < 1s', 'performance', 'P2', async () => {
|
|
564
|
-
const candidates = ['/api/health','/api/status','/health'];
|
|
565
|
-
for (const c of candidates) {
|
|
566
|
-
const r = await probe.fetch(c);
|
|
567
|
-
if (r.status > 0) {
|
|
568
|
-
const b = await probe.benchmarkRoute(c, 5);
|
|
569
|
-
if (b.p95 > 1000) throw new Error(`API p95 ${b.p95}ms exceeds 1000ms`);
|
|
570
|
-
return;
|
|
571
|
-
}
|
|
572
|
-
}
|
|
573
|
-
});
|
|
574
|
-
|
|
575
|
-
t('No response > 10s', 'performance', 'P0', async () => {
|
|
576
|
-
const r = await probe.fetch('/');
|
|
577
|
-
if (r.duration > 10000) throw new Error(`Response took ${r.duration}ms — exceeds 10s`);
|
|
578
|
-
});
|
|
579
|
-
|
|
580
|
-
t('Login page loads < 3s', 'performance', 'P2', async () => {
|
|
581
|
-
const r = await probe.fetch('/login');
|
|
582
|
-
if (r.duration > 3000) throw new Error(`Login page took ${r.duration}ms`);
|
|
583
|
-
});
|
|
584
|
-
|
|
585
|
-
t('Static assets fast', 'performance', 'P2', async () => {
|
|
586
|
-
const r = await probe.fetch('/favicon.ico');
|
|
587
|
-
if (r.status === 200 && r.duration > 2000)
|
|
588
|
-
throw new Error(`favicon.ico took ${r.duration}ms`);
|
|
589
|
-
});
|
|
590
|
-
|
|
591
|
-
t('Consistent response times (no spike)', 'performance', 'P2', async () => {
|
|
592
|
-
const b = await probe.benchmarkRoute('/', 6);
|
|
593
|
-
if (b.max > b.avg * 5) throw new Error(`Response spike: max=${b.max}ms avg=${b.avg}ms`);
|
|
594
|
-
});
|
|
595
|
-
|
|
596
|
-
t('Gzip/Brotli compression', 'performance', 'P2', async () => {
|
|
597
|
-
const r = await probe.fetch('/', { headers: { 'Accept-Encoding': 'gzip, deflate, br' } });
|
|
598
|
-
const enc = r.headers['content-encoding'] || '';
|
|
599
|
-
if (r.status === 200 && !enc) throw new Error('No compression detected (gzip/br)');
|
|
600
|
-
});
|
|
601
|
-
|
|
602
|
-
t('ETag or Last-Modified caching', 'performance', 'P3', async () => {
|
|
603
|
-
const r = await probe.fetch('/');
|
|
604
|
-
if (!r.headers['etag'] && !r.headers['last-modified'])
|
|
605
|
-
throw new Error('No caching headers (ETag/Last-Modified)');
|
|
606
|
-
});
|
|
607
|
-
|
|
608
|
-
// ── SEO (13 tests) ─────────────────────────────────────────────────────
|
|
609
|
-
t('Homepage SEO P1 tags', 'seo', 'P1', async () => {
|
|
610
|
-
const seo = await probe.checkSEO('/');
|
|
611
|
-
if (!seo.ok) throw new Error('Could not fetch homepage');
|
|
612
|
-
const failing = seo.checks.filter(c => !c.pass && c.sev === 'P1');
|
|
613
|
-
if (failing.length > 0) throw new Error(`Missing P1 SEO: ${failing.map(c => c.name).join(', ')}`);
|
|
614
|
-
});
|
|
615
|
-
|
|
616
|
-
t('Homepage SEO P2 tags', 'seo', 'P2', async () => {
|
|
617
|
-
const seo = await probe.checkSEO('/');
|
|
618
|
-
if (!seo.ok) throw new Error('Could not fetch homepage');
|
|
619
|
-
const failing = seo.checks.filter(c => !c.pass && c.sev === 'P2');
|
|
620
|
-
if (failing.length > 2) throw new Error(`${failing.length} P2 SEO issues: ${failing.map(c => c.name).join(', ')}`);
|
|
621
|
-
});
|
|
622
|
-
|
|
623
|
-
t('robots.txt accessible', 'seo', 'P1', async () => {
|
|
624
|
-
const r = await probe.fetch('/robots.txt');
|
|
625
|
-
if (r.status !== 200) throw new Error(`robots.txt returned ${r.status}`);
|
|
626
|
-
});
|
|
627
|
-
|
|
628
|
-
t('robots.txt content valid', 'seo', 'P2', async () => {
|
|
629
|
-
const r = await probe.fetch('/robots.txt', { readBody: true });
|
|
630
|
-
if (r.status !== 200) return;
|
|
631
|
-
if (!r.text?.includes('User-agent')) throw new Error('robots.txt missing User-agent directive');
|
|
632
|
-
});
|
|
633
|
-
|
|
634
|
-
t('sitemap.xml accessible', 'seo', 'P1', async () => {
|
|
635
|
-
const r = await probe.fetch('/sitemap.xml');
|
|
636
|
-
if (r.status !== 200) throw new Error(`sitemap.xml returned ${r.status}`);
|
|
637
|
-
});
|
|
638
|
-
|
|
639
|
-
t('sitemap.xml is valid XML', 'seo', 'P2', async () => {
|
|
640
|
-
const r = await probe.fetch('/sitemap.xml', { readBody: true });
|
|
641
|
-
if (r.status !== 200) return;
|
|
642
|
-
if (!r.text?.includes('<urlset') && !r.text?.includes('<sitemapindex'))
|
|
643
|
-
throw new Error('sitemap.xml missing <urlset> or <sitemapindex>');
|
|
644
|
-
});
|
|
645
|
-
|
|
646
|
-
t('OG meta tags present', 'seo', 'P2', async () => {
|
|
647
|
-
const seo = await probe.checkSEO('/');
|
|
648
|
-
if (!seo.ok) return;
|
|
649
|
-
const og = seo.checks.filter(c => c.name.startsWith('OG') && !c.pass);
|
|
650
|
-
if (og.length > 1) throw new Error(`Missing OG tags: ${og.map(c => c.name).join(', ')}`);
|
|
651
|
-
});
|
|
652
|
-
|
|
653
|
-
t('Twitter card present', 'seo', 'P3', async () => {
|
|
654
|
-
const seo = await probe.checkSEO('/');
|
|
655
|
-
const tc = seo.checks?.find(c => c.name === 'Twitter card');
|
|
656
|
-
if (tc && !tc.pass) throw new Error('Missing Twitter card meta tag');
|
|
657
|
-
});
|
|
658
|
-
|
|
659
|
-
t('Structured data (JSON-LD)', 'seo', 'P3', async () => {
|
|
660
|
-
const seo = await probe.checkSEO('/');
|
|
661
|
-
const sd = seo.checks?.find(c => c.name === 'Structured data');
|
|
662
|
-
if (sd && !sd.pass) throw new Error('No JSON-LD structured data found');
|
|
663
|
-
});
|
|
664
|
-
|
|
665
|
-
t('Canonical link present', 'seo', 'P2', async () => {
|
|
666
|
-
const seo = await probe.checkSEO('/');
|
|
667
|
-
const cl = seo.checks?.find(c => c.name === 'Canonical link');
|
|
668
|
-
if (cl && !cl.pass) throw new Error('Missing canonical link tag');
|
|
669
|
-
});
|
|
670
|
-
|
|
671
|
-
t('Images have alt text', 'seo', 'P2', async () => {
|
|
672
|
-
const seo = await probe.checkSEO('/');
|
|
673
|
-
const ia = seo.checks?.find(c => c.name === 'Alt attributes');
|
|
674
|
-
if (ia && !ia.pass) throw new Error('Images without alt attributes detected');
|
|
675
|
-
});
|
|
676
|
-
|
|
677
|
-
t('manifest.json present', 'seo', 'P3', async () => {
|
|
678
|
-
const r = await probe.fetch('/manifest.json');
|
|
679
|
-
if (r.status !== 200) throw new Error(`manifest.json returned ${r.status}`);
|
|
680
|
-
});
|
|
681
|
-
|
|
682
|
-
t('favicon accessible', 'seo', 'P2', async () => {
|
|
683
|
-
const r = await probe.fetch('/favicon.ico');
|
|
684
|
-
if (r.status !== 200) throw new Error(`favicon.ico returned ${r.status}`);
|
|
685
|
-
});
|
|
686
|
-
|
|
687
|
-
// ── Accessibility (10 tests) ───────────────────────────────────────────
|
|
688
|
-
t('HTML lang attribute', 'a11y', 'P1', async () => {
|
|
689
|
-
const a = await probe.checkA11y('/');
|
|
690
|
-
if (!a.ok) throw new Error('Could not fetch page');
|
|
691
|
-
const c = a.checks.find(c => c.name === 'html[lang] set');
|
|
692
|
-
if (c && !c.pass) throw new Error('Missing lang attribute on <html>');
|
|
693
|
-
});
|
|
694
|
-
|
|
695
|
-
t('Viewport meta tag', 'a11y', 'P1', async () => {
|
|
696
|
-
const a = await probe.checkA11y('/');
|
|
697
|
-
if (!a.ok) throw new Error('Could not fetch page');
|
|
698
|
-
const c = a.checks.find(c => c.name === 'Viewport meta');
|
|
699
|
-
if (c && !c.pass) throw new Error('Missing viewport meta — mobile broken');
|
|
700
|
-
});
|
|
701
|
-
|
|
702
|
-
t('Main landmark present', 'a11y', 'P2', async () => {
|
|
703
|
-
const a = await probe.checkA11y('/');
|
|
704
|
-
if (!a.ok) return;
|
|
705
|
-
const c = a.checks.find(c => c.name === 'Main landmark');
|
|
706
|
-
if (c && !c.pass) throw new Error('No <main> landmark element found');
|
|
707
|
-
});
|
|
708
|
-
|
|
709
|
-
t('Images have alt text (a11y)', 'a11y', 'P1', async () => {
|
|
710
|
-
const a = await probe.checkA11y('/');
|
|
711
|
-
if (!a.ok) return;
|
|
712
|
-
const c = a.checks.find(c => c.name === 'No images without alt');
|
|
713
|
-
if (c && !c.pass) throw new Error('Images without alt attributes — screen reader issue');
|
|
714
|
-
});
|
|
715
|
-
|
|
716
|
-
t('Heading hierarchy', 'a11y', 'P2', async () => {
|
|
717
|
-
const a = await probe.checkA11y('/');
|
|
718
|
-
if (!a.ok) return;
|
|
719
|
-
const c = a.checks.find(c => c.name === 'Heading hierarchy');
|
|
720
|
-
if (c && !c.pass) throw new Error('No H1 tag found — heading hierarchy broken');
|
|
721
|
-
});
|
|
722
|
-
|
|
723
|
-
t('ARIA roles used', 'a11y', 'P2', async () => {
|
|
724
|
-
const a = await probe.checkA11y('/');
|
|
725
|
-
if (!a.ok) return;
|
|
726
|
-
const c = a.checks.find(c => c.name === 'ARIA roles used');
|
|
727
|
-
if (c && !c.pass) throw new Error('No ARIA roles found — accessibility reduced');
|
|
728
|
-
});
|
|
729
|
-
|
|
730
|
-
t('Skip nav link present', 'a11y', 'P2', async () => {
|
|
731
|
-
const a = await probe.checkA11y('/');
|
|
732
|
-
if (!a.ok) return;
|
|
733
|
-
const c = a.checks.find(c => c.name === 'Skip nav link');
|
|
734
|
-
if (c && !c.pass) throw new Error('No skip navigation link found');
|
|
735
|
-
});
|
|
736
|
-
|
|
737
|
-
t('Focus visible styles', 'a11y', 'P2', async () => {
|
|
738
|
-
const a = await probe.checkA11y('/');
|
|
739
|
-
if (!a.ok) return;
|
|
740
|
-
const c = a.checks.find(c => c.name === 'Focus visible (class)');
|
|
741
|
-
if (c && !c.pass) throw new Error('No focus-visible styles detected');
|
|
742
|
-
});
|
|
743
|
-
|
|
744
|
-
t('No autofocus abuse', 'a11y', 'P3', async () => {
|
|
745
|
-
const a = await probe.checkA11y('/');
|
|
746
|
-
if (!a.ok) return;
|
|
747
|
-
const c = a.checks.find(c => c.name === 'No autofocus abuse');
|
|
748
|
-
if (c && !c.pass) throw new Error('Multiple autofocus attributes detected');
|
|
749
|
-
});
|
|
750
|
-
|
|
751
|
-
t('All a11y checks pass', 'a11y', 'P2', async () => {
|
|
752
|
-
const a = await probe.checkA11y('/');
|
|
753
|
-
if (!a.ok) return;
|
|
754
|
-
const failing = a.checks.filter(c => !c.pass && (c.sev === 'P1' || c.sev === 'P2'));
|
|
755
|
-
if (failing.length > 2)
|
|
756
|
-
throw new Error(`${failing.length} a11y issues: ${failing.map(c => c.name).join(', ')}`);
|
|
757
|
-
});
|
|
758
|
-
|
|
759
|
-
// ── Content-Type & API Contract (8 tests) ─────────────────────────────
|
|
760
|
-
t('HTML content-type correct', 'http', 'P1', async () => {
|
|
761
|
-
const r = await probe.fetch('/');
|
|
762
|
-
if (!r.ok && r.status === 0) throw new Error('Connection failed');
|
|
763
|
-
const ct = r.headers['content-type'] || '';
|
|
764
|
-
if (!ct.includes('text/html')) throw new Error(`Expected text/html, got: ${ct}`);
|
|
765
|
-
});
|
|
766
|
-
|
|
767
|
-
t('API returns JSON', 'api-contract', 'P1', async () => {
|
|
768
|
-
const candidates = ['/api/health','/api/status','/api/v1/health'];
|
|
769
|
-
for (const c of candidates) {
|
|
770
|
-
const r = await probe.fetch(c);
|
|
771
|
-
if (r.status > 0 && r.status < 500) {
|
|
772
|
-
const ct = r.headers['content-type'] || '';
|
|
773
|
-
if (!ct.includes('json')) throw new Error(`${c} does not return JSON (${ct})`);
|
|
774
|
-
return;
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
});
|
|
778
|
-
|
|
779
|
-
t('API charset UTF-8', 'api-contract', 'P2', async () => {
|
|
780
|
-
const candidates = ['/api/health','/api/status'];
|
|
781
|
-
for (const c of candidates) {
|
|
782
|
-
const r = await probe.fetch(c);
|
|
783
|
-
if (r.status > 0) {
|
|
784
|
-
const ct = r.headers['content-type'] || '';
|
|
785
|
-
if (ct.includes('json') && !ct.includes('utf-8') && !ct.includes('UTF-8'))
|
|
786
|
-
throw new Error(`API missing UTF-8 charset in content-type: ${ct}`);
|
|
787
|
-
return;
|
|
788
|
-
}
|
|
789
|
-
}
|
|
790
|
-
});
|
|
791
|
-
|
|
792
|
-
t('Error response is JSON', 'api-contract', 'P2', async () => {
|
|
793
|
-
const r = await probe.fetch('/api/nonexistent-' + shortId());
|
|
794
|
-
const ct = r.headers['content-type'] || '';
|
|
795
|
-
if (r.status === 404 && !ct.includes('json'))
|
|
796
|
-
throw new Error('404 API response is not JSON');
|
|
797
|
-
});
|
|
798
|
-
|
|
799
|
-
t('API versioning present', 'api-contract', 'P2', async () => {
|
|
800
|
-
const r1 = await probe.fetch('/api/v1/health');
|
|
801
|
-
const r2 = await probe.fetch('/api/v2/health');
|
|
802
|
-
if (r1.status === 0 && r2.status === 0)
|
|
803
|
-
throw new Error('No versioned API endpoints found (/api/v1/ or /api/v2/)');
|
|
804
|
-
});
|
|
805
|
-
|
|
806
|
-
t('API docs accessible', 'api-contract', 'P2', async () => {
|
|
807
|
-
const candidates = ['/api/docs','/docs','/swagger','/api-docs','/openapi.json'];
|
|
808
|
-
for (const c of candidates) {
|
|
809
|
-
const r = await probe.fetch(c);
|
|
810
|
-
if (r.status >= 200 && r.status < 400) return;
|
|
811
|
-
}
|
|
812
|
-
throw new Error('No API documentation endpoint found');
|
|
813
|
-
});
|
|
814
|
-
|
|
815
|
-
t('Core routes return non-500', 'e2e', 'P1', async () => {
|
|
816
|
-
const routes = ['/', '/login', '/about', '/contact'];
|
|
817
|
-
const errors = [];
|
|
818
|
-
for (const route of routes) {
|
|
819
|
-
const r = await probe.fetch(route);
|
|
820
|
-
if (r.status >= 500) errors.push(`${route} → ${r.status}`);
|
|
821
|
-
}
|
|
822
|
-
if (errors.length > 0) throw new Error(`Server errors: ${errors.join(', ')}`);
|
|
823
|
-
});
|
|
824
|
-
|
|
825
|
-
t('Admin routes return non-500', 'e2e', 'P1', async () => {
|
|
826
|
-
const routes = ['/admin', '/admin/users', '/admin/dashboard'];
|
|
827
|
-
const errors = [];
|
|
828
|
-
for (const route of routes) {
|
|
829
|
-
const r = await probe.fetch(route);
|
|
830
|
-
if (r.status >= 500) errors.push(`${route} → ${r.status}`);
|
|
831
|
-
}
|
|
832
|
-
if (errors.length > 0) throw new Error(`Admin server errors: ${errors.join(', ')}`);
|
|
833
|
-
});
|
|
834
|
-
|
|
835
|
-
// ── Links & Redirects (5 tests) ────────────────────────────────────────
|
|
836
|
-
t('No redirect loops', 'links', 'P1', async () => {
|
|
837
|
-
const r = await probe.fetch('/');
|
|
838
|
-
if (r.error?.includes('redirect') || r.error?.includes('loop'))
|
|
839
|
-
throw new Error(`Redirect loop detected: ${r.error}`);
|
|
840
|
-
});
|
|
841
|
-
|
|
842
|
-
t('HTTPS upgrade redirect', 'links', 'P2', async () => {
|
|
843
|
-
if (!probe.baseUrl.startsWith('https')) return;
|
|
844
|
-
const httpUrl = probe.baseUrl.replace('https://', 'http://');
|
|
845
|
-
try {
|
|
846
|
-
const r = await (new HttpProbe(httpUrl)).fetch('/');
|
|
847
|
-
if (r.status >= 200 && r.status < 300 && !r.url?.startsWith('https'))
|
|
848
|
-
throw new Error('HTTP not redirecting to HTTPS');
|
|
849
|
-
} catch {}
|
|
850
|
-
});
|
|
851
|
-
|
|
852
|
-
t('www redirect consistent', 'links', 'P3', async () => {
|
|
853
|
-
// Just checks the base URL works
|
|
854
|
-
const r = await probe.fetch('/');
|
|
855
|
-
if (r.status === 0) throw new Error('Base URL not reachable');
|
|
856
|
-
});
|
|
857
|
-
|
|
858
|
-
t('No 5xx on common routes', 'links', 'P1', async () => {
|
|
859
|
-
const routes = COMMON_ROUTES.slice(0, 15);
|
|
860
|
-
const errors = [];
|
|
861
|
-
for (const route of routes) {
|
|
862
|
-
const r = await probe.fetch(route);
|
|
863
|
-
if (r.status >= 500) errors.push(`${route}(${r.status})`);
|
|
864
|
-
}
|
|
865
|
-
if (errors.length > 0) throw new Error(`Server errors: ${errors.join(', ')}`);
|
|
866
|
-
});
|
|
867
|
-
|
|
868
|
-
t('security.txt present', 'security', 'P3', async () => {
|
|
869
|
-
const r = await probe.fetch('/.well-known/security.txt');
|
|
870
|
-
if (r.status !== 200) throw new Error('Missing /.well-known/security.txt');
|
|
871
|
-
});
|
|
872
|
-
|
|
873
|
-
return tests;
|
|
874
|
-
}
|
|
875
|
-
|
|
876
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
877
|
-
// MAXIMUM File System Test Suite (120+ tests)
|
|
878
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
879
|
-
function buildFullSystemTests(projectDir = process.cwd()) {
|
|
880
|
-
const tests = [];
|
|
881
|
-
const t = (name, type, sev, fn) =>
|
|
882
|
-
tests.push({ id: shortId(), name, type, sev, fn });
|
|
883
|
-
|
|
884
|
-
// ── Project Structure (15 tests) ───────────────────────────────────────
|
|
885
|
-
t('Project directory exists', 'file-structure', 'P0', async () => {
|
|
886
|
-
if (!await fs.pathExists(projectDir)) throw new Error('Project directory not found');
|
|
887
|
-
});
|
|
888
|
-
|
|
889
|
-
t('package.json exists', 'file-structure', 'P0', async () => {
|
|
890
|
-
if (!await fs.pathExists(path.join(projectDir, 'package.json')))
|
|
891
|
-
throw new Error('package.json missing');
|
|
892
|
-
});
|
|
893
|
-
|
|
894
|
-
t('package.json valid JSON', 'validation', 'P0', async () => {
|
|
895
|
-
const p = path.join(projectDir, 'package.json');
|
|
896
|
-
if (!await fs.pathExists(p)) throw new Error('package.json missing');
|
|
897
|
-
const pkg = await fs.readJson(p).catch(e => { throw new Error(`Invalid JSON: ${e.message}`); });
|
|
898
|
-
if (!pkg.name) throw new Error('package.json missing "name"');
|
|
899
|
-
if (!pkg.version) throw new Error('package.json missing "version"');
|
|
900
|
-
});
|
|
901
|
-
|
|
902
|
-
t('package.json has scripts', 'validation', 'P1', async () => {
|
|
903
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
904
|
-
if (!pkg.scripts || Object.keys(pkg.scripts).length === 0)
|
|
905
|
-
throw new Error('No scripts defined in package.json');
|
|
906
|
-
});
|
|
907
|
-
|
|
908
|
-
t('npm start or dev script', 'validation', 'P1', async () => {
|
|
909
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
910
|
-
if (!pkg.scripts?.start && !pkg.scripts?.dev)
|
|
911
|
-
throw new Error('No start or dev script in package.json');
|
|
912
|
-
});
|
|
913
|
-
|
|
914
|
-
t('npm test script', 'validation', 'P2', async () => {
|
|
915
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
916
|
-
if (!pkg.scripts?.test) throw new Error('No test script in package.json');
|
|
917
|
-
});
|
|
918
|
-
|
|
919
|
-
t('npm build script', 'validation', 'P2', async () => {
|
|
920
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
921
|
-
if (!pkg.scripts?.build) throw new Error('No build script in package.json');
|
|
922
|
-
});
|
|
923
|
-
|
|
924
|
-
t('Dependencies declared', 'validation', 'P1', async () => {
|
|
925
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
926
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
927
|
-
if (Object.keys(deps).length === 0) throw new Error('No dependencies declared');
|
|
928
|
-
});
|
|
929
|
-
|
|
930
|
-
t('No duplicate deps in dev+prod', 'validation', 'P2', async () => {
|
|
931
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
932
|
-
const prod = Object.keys(pkg.dependencies || {});
|
|
933
|
-
const dev = Object.keys(pkg.devDependencies || {});
|
|
934
|
-
const dups = prod.filter(d => dev.includes(d));
|
|
935
|
-
if (dups.length > 0) throw new Error(`Deps in both dependencies+devDependencies: ${dups.join(', ')}`);
|
|
936
|
-
});
|
|
937
|
-
|
|
938
|
-
t('Entry point exists', 'file-structure', 'P0', async () => {
|
|
939
|
-
const candidates = [
|
|
940
|
-
'src/index.ts','src/index.js','index.ts','index.js',
|
|
941
|
-
'main.py','main.go','Program.cs','src/main.ts','src/main.js',
|
|
942
|
-
];
|
|
943
|
-
for (const c of candidates) {
|
|
944
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
945
|
-
}
|
|
946
|
-
throw new Error('No recognisable entry point found');
|
|
947
|
-
});
|
|
948
|
-
|
|
949
|
-
t('Routes directory exists', 'file-structure', 'P1', async () => {
|
|
950
|
-
const candidates = ['src/routes','routes','src/api','api','src/controllers','controllers'];
|
|
951
|
-
for (const c of candidates) {
|
|
952
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
953
|
-
}
|
|
954
|
-
throw new Error('No routes/api directory found');
|
|
955
|
-
});
|
|
956
|
-
|
|
957
|
-
t('Source directory exists', 'file-structure', 'P1', async () => {
|
|
958
|
-
if (!await fs.pathExists(path.join(projectDir, 'src')))
|
|
959
|
-
throw new Error('No src/ directory found');
|
|
960
|
-
});
|
|
961
|
-
|
|
962
|
-
t('README.md present', 'file-structure', 'P2', async () => {
|
|
963
|
-
if (!await fs.pathExists(path.join(projectDir, 'README.md')))
|
|
964
|
-
throw new Error('README.md missing');
|
|
965
|
-
});
|
|
966
|
-
|
|
967
|
-
t('README.md not empty', 'file-structure', 'P3', async () => {
|
|
968
|
-
const rp = path.join(projectDir, 'README.md');
|
|
969
|
-
if (!await fs.pathExists(rp)) return;
|
|
970
|
-
const content = await fs.readFile(rp, 'utf8').catch(() => '');
|
|
971
|
-
if (content.trim().length < 100) throw new Error('README.md is too short (< 100 chars)');
|
|
972
|
-
});
|
|
973
|
-
|
|
974
|
-
t('.gitignore present', 'file-structure', 'P2', async () => {
|
|
975
|
-
if (!await fs.pathExists(path.join(projectDir, '.gitignore')))
|
|
976
|
-
throw new Error('.gitignore missing');
|
|
977
|
-
});
|
|
978
|
-
|
|
979
|
-
// ── Environment Config (8 tests) ──────────────────────────────────────
|
|
980
|
-
t('Env config file present', 'environment', 'P1', async () => {
|
|
981
|
-
const candidates = ['.env','.env.example','.env.sample','.env.local'];
|
|
982
|
-
for (const c of candidates) {
|
|
983
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
984
|
-
}
|
|
985
|
-
throw new Error('No environment config file found');
|
|
986
|
-
});
|
|
987
|
-
|
|
988
|
-
t('.env not committed (has .env.example)', 'environment', 'P0', async () => {
|
|
989
|
-
const gitignorePath = path.join(projectDir, '.gitignore');
|
|
990
|
-
if (await fs.pathExists(gitignorePath)) {
|
|
991
|
-
const content = await fs.readFile(gitignorePath, 'utf8').catch(() => '');
|
|
992
|
-
if (!content.includes('.env'))
|
|
993
|
-
throw new Error('.env not in .gitignore — secrets may be committed');
|
|
994
|
-
}
|
|
995
|
-
});
|
|
996
|
-
|
|
997
|
-
t('PORT env variable referenced', 'environment', 'P2', async () => {
|
|
998
|
-
const candidates = ['src/index.ts','src/index.js','index.js','index.ts','src/main.ts'];
|
|
999
|
-
for (const c of candidates) {
|
|
1000
|
-
const fp = path.join(projectDir, c);
|
|
1001
|
-
if (await fs.pathExists(fp)) {
|
|
1002
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1003
|
-
if (content.includes('process.env.PORT') || content.includes('PORT')) return;
|
|
1004
|
-
}
|
|
1005
|
-
}
|
|
1006
|
-
throw new Error('PORT env variable not referenced in entry point');
|
|
1007
|
-
});
|
|
1008
|
-
|
|
1009
|
-
t('DATABASE_URL env referenced', 'environment', 'P1', async () => {
|
|
1010
|
-
const candidates = ['src/index.ts','src/index.js','.env.example','.env.sample'];
|
|
1011
|
-
for (const c of candidates) {
|
|
1012
|
-
const fp = path.join(projectDir, c);
|
|
1013
|
-
if (await fs.pathExists(fp)) {
|
|
1014
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1015
|
-
if (/DATABASE_URL|MONGO_URI|DB_HOST|CONNECTION_STRING/i.test(content)) return;
|
|
1016
|
-
}
|
|
1017
|
-
}
|
|
1018
|
-
throw new Error('No database connection env variable referenced');
|
|
1019
|
-
});
|
|
1020
|
-
|
|
1021
|
-
t('JWT_SECRET env referenced', 'environment', 'P1', async () => {
|
|
1022
|
-
const candidates = ['src/index.ts','src/index.js','.env.example','.env.sample','src/config'];
|
|
1023
|
-
for (const c of candidates) {
|
|
1024
|
-
const fp = path.join(projectDir, c);
|
|
1025
|
-
if (await fs.pathExists(fp)) {
|
|
1026
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1027
|
-
if (/JWT_SECRET|JWT_KEY|TOKEN_SECRET/i.test(content)) return;
|
|
1028
|
-
}
|
|
1029
|
-
}
|
|
1030
|
-
throw new Error('JWT_SECRET not referenced — auth may use hardcoded secret');
|
|
1031
|
-
});
|
|
1032
|
-
|
|
1033
|
-
t('NODE_ENV handling', 'environment', 'P2', async () => {
|
|
1034
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts','src/app.js'];
|
|
1035
|
-
for (const c of candidates) {
|
|
1036
|
-
const fp = path.join(projectDir, c);
|
|
1037
|
-
if (await fs.pathExists(fp)) {
|
|
1038
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1039
|
-
if (content.includes('NODE_ENV')) return;
|
|
1040
|
-
}
|
|
1041
|
-
}
|
|
1042
|
-
throw new Error('NODE_ENV not referenced — no environment mode handling');
|
|
1043
|
-
});
|
|
1044
|
-
|
|
1045
|
-
t('No hardcoded passwords', 'environment', 'P0', async () => {
|
|
1046
|
-
const pattern = /(?:password|passwd|pwd)\s*[:=]\s*['"][^'"]{4,}['"]/i;
|
|
1047
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts','src/config.ts','src/config.js'];
|
|
1048
|
-
for (const c of candidates) {
|
|
1049
|
-
const fp = path.join(projectDir, c);
|
|
1050
|
-
if (await fs.pathExists(fp)) {
|
|
1051
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1052
|
-
if (pattern.test(content)) throw new Error(`Hardcoded password in ${c}`);
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
});
|
|
1056
|
-
|
|
1057
|
-
t('No hardcoded API keys', 'environment', 'P0', async () => {
|
|
1058
|
-
const pattern = /(?:apikey|api_key|apiSecret|secret)\s*[:=]\s*['"][a-zA-Z0-9]{16,}['"]/i;
|
|
1059
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts','src/config.ts'];
|
|
1060
|
-
for (const c of candidates) {
|
|
1061
|
-
const fp = path.join(projectDir, c);
|
|
1062
|
-
if (await fs.pathExists(fp)) {
|
|
1063
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1064
|
-
if (pattern.test(content)) throw new Error(`Hardcoded API key in ${c}`);
|
|
1065
|
-
}
|
|
1066
|
-
}
|
|
1067
|
-
});
|
|
1068
|
-
|
|
1069
|
-
// ── Security (15 tests) ───────────────────────────────────────────────
|
|
1070
|
-
t('Password hashing library', 'security', 'P0', async () => {
|
|
1071
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1072
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1073
|
-
if (!['bcrypt','bcryptjs','argon2','scrypt','crypto'].some(d => deps[d]))
|
|
1074
|
-
throw new Error('No password hashing library found (bcrypt/argon2)');
|
|
1075
|
-
});
|
|
1076
|
-
|
|
1077
|
-
t('JWT library present', 'security', 'P0', async () => {
|
|
1078
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1079
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1080
|
-
if (!['jsonwebtoken','jose','@auth/core','passport-jwt'].some(d => deps[d]))
|
|
1081
|
-
throw new Error('No JWT library found');
|
|
1082
|
-
});
|
|
1083
|
-
|
|
1084
|
-
t('Rate limiting library', 'security', 'P1', async () => {
|
|
1085
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1086
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1087
|
-
if (!['express-rate-limit','rate-limiter-flexible','@nestjs/throttler','fastapi-limiter'].some(d => deps[d]))
|
|
1088
|
-
throw new Error('No rate limiting library found');
|
|
1089
|
-
});
|
|
1090
|
-
|
|
1091
|
-
t('CORS library configured', 'cors', 'P1', async () => {
|
|
1092
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts','src/app.js','index.js'];
|
|
1093
|
-
for (const c of candidates) {
|
|
1094
|
-
const fp = path.join(projectDir, c);
|
|
1095
|
-
if (await fs.pathExists(fp)) {
|
|
1096
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1097
|
-
if (/cors|CORS/i.test(content)) return;
|
|
1098
|
-
}
|
|
1099
|
-
}
|
|
1100
|
-
throw new Error('No CORS configuration detected');
|
|
1101
|
-
});
|
|
1102
|
-
|
|
1103
|
-
t('Helmet or security middleware', 'security', 'P1', async () => {
|
|
1104
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1105
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1106
|
-
if (!['helmet','@fastify/helmet','django-security'].some(d => deps[d]))
|
|
1107
|
-
throw new Error('No security headers middleware (helmet) found');
|
|
1108
|
-
});
|
|
1109
|
-
|
|
1110
|
-
t('Input validation library', 'security', 'P1', async () => {
|
|
1111
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1112
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1113
|
-
if (!['zod','joi','yup','class-validator','express-validator','marshmallow'].some(d => deps[d]))
|
|
1114
|
-
throw new Error('No input validation library (zod/joi/yup) found');
|
|
1115
|
-
});
|
|
1116
|
-
|
|
1117
|
-
t('XSS prevention library', 'security', 'P1', async () => {
|
|
1118
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1119
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1120
|
-
if (!['xss','dompurify','sanitize-html','isomorphic-dompurify'].some(d => deps[d]))
|
|
1121
|
-
throw new Error('No XSS prevention library found');
|
|
1122
|
-
});
|
|
1123
|
-
|
|
1124
|
-
t('No eval() usage', 'security', 'P0', async () => {
|
|
1125
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts','src/app.js'];
|
|
1126
|
-
for (const c of candidates) {
|
|
1127
|
-
const fp = path.join(projectDir, c);
|
|
1128
|
-
if (await fs.pathExists(fp)) {
|
|
1129
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1130
|
-
if (/\beval\s*\(/.test(content)) throw new Error(`eval() found in ${c} — security risk`);
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
});
|
|
1134
|
-
|
|
1135
|
-
t('No dangerous packages', 'security', 'P0', async () => {
|
|
1136
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1137
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1138
|
-
const found = DANGEROUS_PACKAGES.filter(d => deps[d]);
|
|
1139
|
-
if (found.length > 0) throw new Error(`Dangerous packages found: ${found.join(', ')}`);
|
|
1140
|
-
});
|
|
1141
|
-
|
|
1142
|
-
t('CSRF protection configured', 'security', 'P1', async () => {
|
|
1143
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1144
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1145
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts'];
|
|
1146
|
-
let hasCSRF = ['csurf','csrf','@nestjs/csrf'].some(d => deps[d]);
|
|
1147
|
-
if (!hasCSRF) {
|
|
1148
|
-
for (const c of candidates) {
|
|
1149
|
-
const fp = path.join(projectDir, c);
|
|
1150
|
-
if (await fs.pathExists(fp)) {
|
|
1151
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1152
|
-
if (/csrf|CSRF/i.test(content)) { hasCSRF = true; break; }
|
|
1153
|
-
}
|
|
1154
|
-
}
|
|
1155
|
-
}
|
|
1156
|
-
if (!hasCSRF) throw new Error('No CSRF protection found');
|
|
1157
|
-
});
|
|
1158
|
-
|
|
1159
|
-
t('SQL injection prevention', 'security', 'P0', async () => {
|
|
1160
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1161
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1162
|
-
const hasORM = ['prisma','typeorm','sequelize','mongoose','sqlalchemy','hibernate'].some(d => deps[d]);
|
|
1163
|
-
if (!hasORM) {
|
|
1164
|
-
const candidates = ['src/index.ts','src/index.js'];
|
|
1165
|
-
for (const c of candidates) {
|
|
1166
|
-
const fp = path.join(projectDir, c);
|
|
1167
|
-
if (await fs.pathExists(fp)) {
|
|
1168
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1169
|
-
if (/query\s*\+\s*req\.|query\s*\+\s*params/i.test(content))
|
|
1170
|
-
throw new Error(`Potential SQL injection: string concatenation in query in ${c}`);
|
|
1171
|
-
}
|
|
1172
|
-
}
|
|
1173
|
-
}
|
|
1174
|
-
});
|
|
1175
|
-
|
|
1176
|
-
t('Sensitive files not exposed', 'security', 'P0', async () => {
|
|
1177
|
-
const dangerous = ['.env','.git/config','config/database.yml'];
|
|
1178
|
-
for (const f of dangerous) {
|
|
1179
|
-
const fp = path.join(projectDir, 'public', f);
|
|
1180
|
-
if (await fs.pathExists(fp)) throw new Error(`Sensitive file in public/: ${f}`);
|
|
1181
|
-
}
|
|
1182
|
-
});
|
|
1183
|
-
|
|
1184
|
-
t('Cookie security settings', 'security', 'P1', async () => {
|
|
1185
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts'];
|
|
1186
|
-
for (const c of candidates) {
|
|
1187
|
-
const fp = path.join(projectDir, c);
|
|
1188
|
-
if (await fs.pathExists(fp)) {
|
|
1189
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1190
|
-
if (/cookie/.test(content)) {
|
|
1191
|
-
if (!content.includes('httpOnly') && !content.includes('secure'))
|
|
1192
|
-
throw new Error(`Cookie without httpOnly/secure flags in ${c}`);
|
|
1193
|
-
return;
|
|
1194
|
-
}
|
|
1195
|
-
}
|
|
1196
|
-
}
|
|
1197
|
-
});
|
|
1198
|
-
|
|
1199
|
-
t('Dependency vulnerability check', 'dependency', 'P1', async () => {
|
|
1200
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1201
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1202
|
-
const vulns = [];
|
|
1203
|
-
for (const { name, below, reason } of VULN_PATTERNS) {
|
|
1204
|
-
const ver = deps[name];
|
|
1205
|
-
if (ver && semverLt(ver, below))
|
|
1206
|
-
vulns.push(`${name}@${ver} (${reason} — upgrade to ${below}+)`);
|
|
1207
|
-
}
|
|
1208
|
-
if (vulns.length > 0) throw new Error(`Vulnerable deps: ${vulns.join('; ')}`);
|
|
1209
|
-
});
|
|
1210
|
-
|
|
1211
|
-
t('No .npmrc with auth tokens in repo', 'security', 'P0', async () => {
|
|
1212
|
-
const fp = path.join(projectDir, '.npmrc');
|
|
1213
|
-
if (await fs.pathExists(fp)) {
|
|
1214
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1215
|
-
if (content.includes('_authToken') || content.includes('//registry'))
|
|
1216
|
-
throw new Error('.npmrc contains auth tokens — must not be committed');
|
|
1217
|
-
}
|
|
1218
|
-
});
|
|
1219
|
-
|
|
1220
|
-
// ── Database (10 tests) ───────────────────────────────────────────────
|
|
1221
|
-
t('Database schema/models exist', 'database', 'P1', async () => {
|
|
1222
|
-
const candidates = [
|
|
1223
|
-
'prisma/schema.prisma','schema.prisma',
|
|
1224
|
-
'src/models','models','src/entities','entities',
|
|
1225
|
-
];
|
|
1226
|
-
for (const c of candidates) {
|
|
1227
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1228
|
-
}
|
|
1229
|
-
throw new Error('No database schema/models directory found');
|
|
1230
|
-
});
|
|
1231
|
-
|
|
1232
|
-
t('Prisma schema valid fields', 'database', 'P1', async () => {
|
|
1233
|
-
const fp = path.join(projectDir, 'prisma', 'schema.prisma');
|
|
1234
|
-
if (!await fs.pathExists(fp)) return;
|
|
1235
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1236
|
-
if (!content.includes('model ')) throw new Error('Prisma schema has no models defined');
|
|
1237
|
-
if (!content.includes('datasource')) throw new Error('Prisma schema missing datasource block');
|
|
1238
|
-
if (!content.includes('generator')) throw new Error('Prisma schema missing generator block');
|
|
1239
|
-
});
|
|
1240
|
-
|
|
1241
|
-
t('Prisma migrations directory', 'database', 'P2', async () => {
|
|
1242
|
-
const prismaPath = path.join(projectDir, 'prisma', 'schema.prisma');
|
|
1243
|
-
if (!await fs.pathExists(prismaPath)) return;
|
|
1244
|
-
if (!await fs.pathExists(path.join(projectDir, 'prisma', 'migrations')))
|
|
1245
|
-
throw new Error('Prisma migrations directory missing — run prisma migrate dev');
|
|
1246
|
-
});
|
|
1247
|
-
|
|
1248
|
-
t('Database seeder file', 'database', 'P2', async () => {
|
|
1249
|
-
const candidates = [
|
|
1250
|
-
'prisma/seed.ts','prisma/seed.js','src/seeders','seeders','src/database/seed',
|
|
1251
|
-
];
|
|
1252
|
-
for (const c of candidates) {
|
|
1253
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1254
|
-
}
|
|
1255
|
-
throw new Error('No database seeder found');
|
|
1256
|
-
});
|
|
1257
|
-
|
|
1258
|
-
t('Models have ID fields', 'database', 'P1', async () => {
|
|
1259
|
-
const fp = path.join(projectDir, 'prisma', 'schema.prisma');
|
|
1260
|
-
if (!await fs.pathExists(fp)) return;
|
|
1261
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1262
|
-
const models = content.match(/model\s+\w+\s*{[^}]+}/g) || [];
|
|
1263
|
-
const missing = models.filter(m => !/@id/.test(m)).map(m => (m.match(/model\s+(\w+)/) || [])[1]);
|
|
1264
|
-
if (missing.length > 0) throw new Error(`Models missing @id: ${missing.join(', ')}`);
|
|
1265
|
-
});
|
|
1266
|
-
|
|
1267
|
-
t('Models have timestamps', 'database', 'P2', async () => {
|
|
1268
|
-
const fp = path.join(projectDir, 'prisma', 'schema.prisma');
|
|
1269
|
-
if (!await fs.pathExists(fp)) return;
|
|
1270
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1271
|
-
if (!content.includes('createdAt') && !content.includes('updatedAt'))
|
|
1272
|
-
throw new Error('No timestamp fields (createdAt/updatedAt) in Prisma schema');
|
|
1273
|
-
});
|
|
1274
|
-
|
|
1275
|
-
t('Database connection pool configured', 'database', 'P2', async () => {
|
|
1276
|
-
const candidates = ['src/index.ts','src/index.js','src/db.ts','src/database.ts'];
|
|
1277
|
-
for (const c of candidates) {
|
|
1278
|
-
const fp = path.join(projectDir, c);
|
|
1279
|
-
if (await fs.pathExists(fp)) {
|
|
1280
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1281
|
-
if (/pool|connection_limit|maxPoolSize/i.test(content)) return;
|
|
1282
|
-
}
|
|
1283
|
-
}
|
|
1284
|
-
throw new Error('No connection pool configuration found');
|
|
1285
|
-
});
|
|
1286
|
-
|
|
1287
|
-
t('No raw SQL string concatenation', 'database', 'P0', async () => {
|
|
1288
|
-
const candidates = ['src/repositories','src/models','src/services'];
|
|
1289
|
-
for (const c of candidates) {
|
|
1290
|
-
const dir = path.join(projectDir, c);
|
|
1291
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1292
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1293
|
-
for (const f of files) {
|
|
1294
|
-
if (!/\.(ts|js)$/.test(f)) continue;
|
|
1295
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1296
|
-
if (/query\s*\(`[^`]*\$\{req\./i.test(content))
|
|
1297
|
-
throw new Error(`Raw SQL with user input in ${c}/${f}`);
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
});
|
|
1301
|
-
|
|
1302
|
-
t('ORM or query builder used', 'database', 'P1', async () => {
|
|
1303
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1304
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1305
|
-
if (!['prisma','typeorm','sequelize','mongoose','knex','mikro-orm','drizzle-orm'].some(d => deps[d]))
|
|
1306
|
-
throw new Error('No ORM or query builder found');
|
|
1307
|
-
});
|
|
1308
|
-
|
|
1309
|
-
t('Database index strategy', 'database', 'P3', async () => {
|
|
1310
|
-
const fp = path.join(projectDir, 'prisma', 'schema.prisma');
|
|
1311
|
-
if (!await fs.pathExists(fp)) return;
|
|
1312
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1313
|
-
if (!content.includes('@@index') && !content.includes('@unique') && !content.includes('@@unique'))
|
|
1314
|
-
throw new Error('No indexes defined in Prisma schema — query performance may be poor');
|
|
1315
|
-
});
|
|
1316
|
-
|
|
1317
|
-
// ── TypeScript (8 tests) ──────────────────────────────────────────────
|
|
1318
|
-
t('tsconfig.json present', 'typescript', 'P1', async () => {
|
|
1319
|
-
const tsp = path.join(projectDir, 'tsconfig.json');
|
|
1320
|
-
if (!await fs.pathExists(tsp)) throw new Error('tsconfig.json missing');
|
|
1321
|
-
});
|
|
1322
|
-
|
|
1323
|
-
t('tsconfig strict mode', 'typescript', 'P2', async () => {
|
|
1324
|
-
const tsp = path.join(projectDir, 'tsconfig.json');
|
|
1325
|
-
if (!await fs.pathExists(tsp)) return;
|
|
1326
|
-
const tsconfig = await fs.readJson(tsp).catch(() => ({}));
|
|
1327
|
-
if (!tsconfig.compilerOptions?.strict)
|
|
1328
|
-
throw new Error('TypeScript strict mode not enabled in tsconfig.json');
|
|
1329
|
-
});
|
|
1330
|
-
|
|
1331
|
-
t('No "any" types in main files', 'typescript', 'P2', async () => {
|
|
1332
|
-
const candidates = ['src/index.ts','src/app.ts','src/server.ts'];
|
|
1333
|
-
let anyCount = 0;
|
|
1334
|
-
for (const c of candidates) {
|
|
1335
|
-
const fp = path.join(projectDir, c);
|
|
1336
|
-
if (await fs.pathExists(fp)) {
|
|
1337
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1338
|
-
anyCount += (content.match(/:\s*any\b/g) || []).length;
|
|
1339
|
-
}
|
|
1340
|
-
}
|
|
1341
|
-
if (anyCount > 5) throw new Error(`${anyCount} "any" type usages detected — use proper types`);
|
|
1342
|
-
});
|
|
1343
|
-
|
|
1344
|
-
t('TypeScript paths configured', 'typescript', 'P3', async () => {
|
|
1345
|
-
const tsp = path.join(projectDir, 'tsconfig.json');
|
|
1346
|
-
if (!await fs.pathExists(tsp)) return;
|
|
1347
|
-
const tsconfig = await fs.readJson(tsp).catch(() => ({}));
|
|
1348
|
-
if (!tsconfig.compilerOptions?.paths && !tsconfig.compilerOptions?.baseUrl)
|
|
1349
|
-
throw new Error('No path aliases configured in tsconfig');
|
|
1350
|
-
});
|
|
1351
|
-
|
|
1352
|
-
t('TypeScript compiler target >= ES2020', 'typescript', 'P2', async () => {
|
|
1353
|
-
const tsp = path.join(projectDir, 'tsconfig.json');
|
|
1354
|
-
if (!await fs.pathExists(tsp)) return;
|
|
1355
|
-
const tsconfig = await fs.readJson(tsp).catch(() => ({}));
|
|
1356
|
-
const target = tsconfig.compilerOptions?.target || '';
|
|
1357
|
-
const oldTargets = ['es5','es6','es2015','es2016','es2017','es2018','es2019'];
|
|
1358
|
-
if (oldTargets.includes(target.toLowerCase()))
|
|
1359
|
-
throw new Error(`TypeScript target ${target} is outdated — use ES2020+`);
|
|
1360
|
-
});
|
|
1361
|
-
|
|
1362
|
-
t('No @ts-ignore abuse', 'typescript', 'P2', async () => {
|
|
1363
|
-
const candidates = ['src/index.ts','src/app.ts','src/server.ts'];
|
|
1364
|
-
let count = 0;
|
|
1365
|
-
for (const c of candidates) {
|
|
1366
|
-
const fp = path.join(projectDir, c);
|
|
1367
|
-
if (await fs.pathExists(fp)) {
|
|
1368
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1369
|
-
count += (content.match(/@ts-ignore/g) || []).length;
|
|
1370
|
-
}
|
|
1371
|
-
}
|
|
1372
|
-
if (count > 3) throw new Error(`${count} @ts-ignore comments found — fix type errors instead`);
|
|
1373
|
-
});
|
|
1374
|
-
|
|
1375
|
-
t('TypeScript declaration files', 'typescript', 'P3', async () => {
|
|
1376
|
-
const tsp = path.join(projectDir, 'tsconfig.json');
|
|
1377
|
-
if (!await fs.pathExists(tsp)) return;
|
|
1378
|
-
const tsconfig = await fs.readJson(tsp).catch(() => ({}));
|
|
1379
|
-
if (tsconfig.compilerOptions?.declaration === false)
|
|
1380
|
-
throw new Error('TypeScript declaration generation disabled');
|
|
1381
|
-
});
|
|
1382
|
-
|
|
1383
|
-
t('ESLint or TSLint configured', 'code-quality', 'P2', async () => {
|
|
1384
|
-
const candidates = ['.eslintrc','.eslintrc.json','.eslintrc.js','.eslintrc.ts','eslint.config.js'];
|
|
1385
|
-
for (const c of candidates) {
|
|
1386
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1387
|
-
}
|
|
1388
|
-
throw new Error('No ESLint configuration found');
|
|
1389
|
-
});
|
|
1390
|
-
|
|
1391
|
-
// ── Logging (6 tests) ─────────────────────────────────────────────────
|
|
1392
|
-
t('Logging library present', 'logging', 'P1', async () => {
|
|
1393
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1394
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1395
|
-
if (!['winston','pino','bunyan','morgan','log4js','loglevel'].some(d => deps[d]))
|
|
1396
|
-
throw new Error('No structured logging library found (winston/pino)');
|
|
1397
|
-
});
|
|
1398
|
-
|
|
1399
|
-
t('HTTP request logging (morgan/pino-http)', 'logging', 'P2', async () => {
|
|
1400
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1401
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1402
|
-
if (!['morgan','pino-http','http-pino'].some(d => deps[d]))
|
|
1403
|
-
throw new Error('No HTTP request logging middleware (morgan/pino-http)');
|
|
1404
|
-
});
|
|
1405
|
-
|
|
1406
|
-
t('No console.log in production code', 'logging', 'P2', async () => {
|
|
1407
|
-
const candidates = ['src/routes','src/controllers','src/services'];
|
|
1408
|
-
let count = 0;
|
|
1409
|
-
for (const c of candidates) {
|
|
1410
|
-
const dir = path.join(projectDir, c);
|
|
1411
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1412
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1413
|
-
for (const f of files) {
|
|
1414
|
-
if (!/\.(ts|js)$/.test(f)) continue;
|
|
1415
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1416
|
-
count += (content.match(/console\.log\(/g) || []).length;
|
|
1417
|
-
}
|
|
1418
|
-
}
|
|
1419
|
-
if (count > 10) throw new Error(`${count} console.log() in source files — use logger instead`);
|
|
1420
|
-
});
|
|
1421
|
-
|
|
1422
|
-
t('Error logging configured', 'logging', 'P1', async () => {
|
|
1423
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts','src/middleware'];
|
|
1424
|
-
for (const c of candidates) {
|
|
1425
|
-
const fp = path.join(projectDir, c);
|
|
1426
|
-
if (await fs.pathExists(fp)) {
|
|
1427
|
-
const content = typeof fp === 'string' && (await fs.stat(fp)).isFile()
|
|
1428
|
-
? await fs.readFile(fp, 'utf8').catch(() => '')
|
|
1429
|
-
: '';
|
|
1430
|
-
if (/error.*log|log.*error|\.error\(/i.test(content)) return;
|
|
1431
|
-
}
|
|
1432
|
-
}
|
|
1433
|
-
throw new Error('No error logging configured');
|
|
1434
|
-
});
|
|
1435
|
-
|
|
1436
|
-
t('Log level configuration', 'logging', 'P3', async () => {
|
|
1437
|
-
const candidates = ['src/index.ts','src/index.js','src/logger.ts','src/logger.js'];
|
|
1438
|
-
for (const c of candidates) {
|
|
1439
|
-
const fp = path.join(projectDir, c);
|
|
1440
|
-
if (await fs.pathExists(fp)) {
|
|
1441
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1442
|
-
if (/LOG_LEVEL|logLevel|level.*process\.env/i.test(content)) return;
|
|
1443
|
-
}
|
|
1444
|
-
}
|
|
1445
|
-
throw new Error('Log level not configurable via environment');
|
|
1446
|
-
});
|
|
1447
|
-
|
|
1448
|
-
t('Separate logger module', 'logging', 'P3', async () => {
|
|
1449
|
-
const candidates = ['src/logger.ts','src/logger.js','src/utils/logger.ts','src/utils/logger.js','src/lib/logger.ts'];
|
|
1450
|
-
for (const c of candidates) {
|
|
1451
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1452
|
-
}
|
|
1453
|
-
throw new Error('No dedicated logger module found');
|
|
1454
|
-
});
|
|
1455
|
-
|
|
1456
|
-
// ── Error Handling (8 tests) ──────────────────────────────────────────
|
|
1457
|
-
t('Global error handler middleware', 'error-handling', 'P0', async () => {
|
|
1458
|
-
const candidates = ['src/index.ts','src/index.js','src/app.ts','src/app.js','src/middleware'];
|
|
1459
|
-
for (const c of candidates) {
|
|
1460
|
-
const fp = path.join(projectDir, c);
|
|
1461
|
-
if (await fs.pathExists(fp)) {
|
|
1462
|
-
const stat = await fs.stat(fp);
|
|
1463
|
-
const content = stat.isFile()
|
|
1464
|
-
? await fs.readFile(fp, 'utf8').catch(() => '')
|
|
1465
|
-
: '';
|
|
1466
|
-
if (/err.*next|next.*err|errorHandler|error.*middleware/i.test(content)) return;
|
|
1467
|
-
}
|
|
1468
|
-
}
|
|
1469
|
-
throw new Error('No global error handler middleware found');
|
|
1470
|
-
});
|
|
1471
|
-
|
|
1472
|
-
t('Try-catch in async routes', 'error-handling', 'P1', async () => {
|
|
1473
|
-
const candidates = ['src/routes','src/controllers'];
|
|
1474
|
-
let hasTryCatch = false;
|
|
1475
|
-
for (const c of candidates) {
|
|
1476
|
-
const dir = path.join(projectDir, c);
|
|
1477
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1478
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1479
|
-
for (const f of files) {
|
|
1480
|
-
if (!/\.(ts|js)$/.test(f)) continue;
|
|
1481
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1482
|
-
if (/try\s*\{[\s\S]*\}\s*catch/.test(content)) { hasTryCatch = true; break; }
|
|
1483
|
-
}
|
|
1484
|
-
}
|
|
1485
|
-
if (!hasTryCatch) throw new Error('No try-catch blocks in routes/controllers');
|
|
1486
|
-
});
|
|
1487
|
-
|
|
1488
|
-
t('Async error wrapper utility', 'error-handling', 'P2', async () => {
|
|
1489
|
-
const candidates = ['src/utils','src/helpers','src/middleware'];
|
|
1490
|
-
for (const c of candidates) {
|
|
1491
|
-
const dir = path.join(projectDir, c);
|
|
1492
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1493
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1494
|
-
for (const f of files) {
|
|
1495
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1496
|
-
if (/asyncHandler|catchAsync|wrapAsync/i.test(content)) return;
|
|
1497
|
-
}
|
|
1498
|
-
}
|
|
1499
|
-
throw new Error('No async error wrapper utility (asyncHandler/catchAsync)');
|
|
1500
|
-
});
|
|
1501
|
-
|
|
1502
|
-
t('HTTP error codes correct', 'error-handling', 'P2', async () => {
|
|
1503
|
-
const candidates = ['src/routes','src/controllers'];
|
|
1504
|
-
for (const c of candidates) {
|
|
1505
|
-
const dir = path.join(projectDir, c);
|
|
1506
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1507
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1508
|
-
for (const f of files) {
|
|
1509
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1510
|
-
if (/res\.status\(200\).*catch|res\.send\(.*error/i.test(content))
|
|
1511
|
-
throw new Error(`Returning 200 in catch block in ${c}/${f}`);
|
|
1512
|
-
}
|
|
1513
|
-
}
|
|
1514
|
-
});
|
|
1515
|
-
|
|
1516
|
-
t('No unhandledRejection without handler', 'error-handling', 'P1', async () => {
|
|
1517
|
-
const candidates = ['src/index.ts','src/index.js','index.js','server.ts'];
|
|
1518
|
-
for (const c of candidates) {
|
|
1519
|
-
const fp = path.join(projectDir, c);
|
|
1520
|
-
if (await fs.pathExists(fp)) {
|
|
1521
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1522
|
-
if (content.includes('unhandledRejection') || content.includes('uncaughtException')) return;
|
|
1523
|
-
}
|
|
1524
|
-
}
|
|
1525
|
-
throw new Error('No unhandledRejection/uncaughtException handler in entry point');
|
|
1526
|
-
});
|
|
1527
|
-
|
|
1528
|
-
t('Process exit handling', 'error-handling', 'P2', async () => {
|
|
1529
|
-
const candidates = ['src/index.ts','src/index.js','index.js'];
|
|
1530
|
-
for (const c of candidates) {
|
|
1531
|
-
const fp = path.join(projectDir, c);
|
|
1532
|
-
if (await fs.pathExists(fp)) {
|
|
1533
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1534
|
-
if (/SIGTERM|SIGINT|graceful/i.test(content)) return;
|
|
1535
|
-
}
|
|
1536
|
-
}
|
|
1537
|
-
throw new Error('No graceful shutdown handler (SIGTERM/SIGINT)');
|
|
1538
|
-
});
|
|
1539
|
-
|
|
1540
|
-
t('Custom error classes', 'error-handling', 'P2', async () => {
|
|
1541
|
-
const candidates = ['src/errors','src/exceptions','src/utils/errors.ts','src/utils/errors.js'];
|
|
1542
|
-
for (const c of candidates) {
|
|
1543
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1544
|
-
}
|
|
1545
|
-
throw new Error('No custom error classes directory found');
|
|
1546
|
-
});
|
|
1547
|
-
|
|
1548
|
-
t('Validation error responses', 'error-handling', 'P1', async () => {
|
|
1549
|
-
const candidates = ['src/middleware','src/validators'];
|
|
1550
|
-
for (const c of candidates) {
|
|
1551
|
-
const dir = path.join(projectDir, c);
|
|
1552
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1553
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1554
|
-
for (const f of files) {
|
|
1555
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1556
|
-
if (/validationError|ValidationError|400/i.test(content)) return;
|
|
1557
|
-
}
|
|
1558
|
-
}
|
|
1559
|
-
throw new Error('No validation error handling found (400 responses)');
|
|
1560
|
-
});
|
|
1561
|
-
|
|
1562
|
-
// ── Docker & CI (10 tests) ────────────────────────────────────────────
|
|
1563
|
-
t('Dockerfile present', 'docker', 'P1', async () => {
|
|
1564
|
-
if (!await fs.pathExists(path.join(projectDir, 'Dockerfile')))
|
|
1565
|
-
throw new Error('Dockerfile missing');
|
|
1566
|
-
});
|
|
1567
|
-
|
|
1568
|
-
t('docker-compose.yml present', 'docker', 'P1', async () => {
|
|
1569
|
-
const candidates = ['docker-compose.yml','docker-compose.yaml'];
|
|
1570
|
-
for (const c of candidates) {
|
|
1571
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1572
|
-
}
|
|
1573
|
-
throw new Error('docker-compose.yml missing');
|
|
1574
|
-
});
|
|
1575
|
-
|
|
1576
|
-
t('Dockerfile uses non-root user', 'docker', 'P1', async () => {
|
|
1577
|
-
const fp = path.join(projectDir, 'Dockerfile');
|
|
1578
|
-
if (!await fs.pathExists(fp)) return;
|
|
1579
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1580
|
-
if (!content.includes('USER') || content.includes('USER root'))
|
|
1581
|
-
throw new Error('Dockerfile does not set non-root USER — security risk');
|
|
1582
|
-
});
|
|
1583
|
-
|
|
1584
|
-
t('Dockerfile uses specific base image tag', 'docker', 'P2', async () => {
|
|
1585
|
-
const fp = path.join(projectDir, 'Dockerfile');
|
|
1586
|
-
if (!await fs.pathExists(fp)) return;
|
|
1587
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1588
|
-
const fromLine = content.match(/^FROM\s+([^\s]+)/m)?.[1] || '';
|
|
1589
|
-
if (fromLine.endsWith(':latest'))
|
|
1590
|
-
throw new Error('Dockerfile uses :latest tag — pin to specific version');
|
|
1591
|
-
});
|
|
1592
|
-
|
|
1593
|
-
t('Dockerfile multi-stage build', 'docker', 'P2', async () => {
|
|
1594
|
-
const fp = path.join(projectDir, 'Dockerfile');
|
|
1595
|
-
if (!await fs.pathExists(fp)) return;
|
|
1596
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1597
|
-
const fromCount = (content.match(/^FROM /gm) || []).length;
|
|
1598
|
-
if (fromCount < 2) throw new Error('Dockerfile not using multi-stage build — image may be large');
|
|
1599
|
-
});
|
|
1600
|
-
|
|
1601
|
-
t('.dockerignore present', 'docker', 'P2', async () => {
|
|
1602
|
-
if (!await fs.pathExists(path.join(projectDir, '.dockerignore')))
|
|
1603
|
-
throw new Error('.dockerignore missing — node_modules may be included in image');
|
|
1604
|
-
});
|
|
1605
|
-
|
|
1606
|
-
t('GitHub Actions workflow', 'ci', 'P1', async () => {
|
|
1607
|
-
if (!await fs.pathExists(path.join(projectDir, '.github', 'workflows')))
|
|
1608
|
-
throw new Error('No GitHub Actions workflows directory');
|
|
1609
|
-
});
|
|
1610
|
-
|
|
1611
|
-
t('CI workflow has test step', 'ci', 'P2', async () => {
|
|
1612
|
-
const wfDir = path.join(projectDir, '.github', 'workflows');
|
|
1613
|
-
if (!await fs.pathExists(wfDir)) return;
|
|
1614
|
-
const files = await fs.readdir(wfDir).catch(() => []);
|
|
1615
|
-
for (const f of files) {
|
|
1616
|
-
const content = await fs.readFile(path.join(wfDir, f), 'utf8').catch(() => '');
|
|
1617
|
-
if (/run.*test|npm.*test|yarn.*test/i.test(content)) return;
|
|
1618
|
-
}
|
|
1619
|
-
throw new Error('No test step in CI workflow');
|
|
1620
|
-
});
|
|
1621
|
-
|
|
1622
|
-
t('CI workflow has lint step', 'ci', 'P2', async () => {
|
|
1623
|
-
const wfDir = path.join(projectDir, '.github', 'workflows');
|
|
1624
|
-
if (!await fs.pathExists(wfDir)) return;
|
|
1625
|
-
const files = await fs.readdir(wfDir).catch(() => []);
|
|
1626
|
-
for (const f of files) {
|
|
1627
|
-
const content = await fs.readFile(path.join(wfDir, f), 'utf8').catch(() => '');
|
|
1628
|
-
if (/run.*lint|npm.*lint|eslint/i.test(content)) return;
|
|
1629
|
-
}
|
|
1630
|
-
throw new Error('No lint step in CI workflow');
|
|
1631
|
-
});
|
|
1632
|
-
|
|
1633
|
-
t('CI workflow has build step', 'ci', 'P2', async () => {
|
|
1634
|
-
const wfDir = path.join(projectDir, '.github', 'workflows');
|
|
1635
|
-
if (!await fs.pathExists(wfDir)) return;
|
|
1636
|
-
const files = await fs.readdir(wfDir).catch(() => []);
|
|
1637
|
-
for (const f of files) {
|
|
1638
|
-
const content = await fs.readFile(path.join(wfDir, f), 'utf8').catch(() => '');
|
|
1639
|
-
if (/run.*build|npm.*build|yarn.*build/i.test(content)) return;
|
|
1640
|
-
}
|
|
1641
|
-
throw new Error('No build step in CI workflow');
|
|
1642
|
-
});
|
|
1643
|
-
|
|
1644
|
-
// ── Testing (8 tests) ─────────────────────────────────────────────────
|
|
1645
|
-
t('Test directory exists', 'e2e', 'P1', async () => {
|
|
1646
|
-
const testDirs = ['tests','test','__tests__','spec','src/__tests__'];
|
|
1647
|
-
for (const d of testDirs) {
|
|
1648
|
-
if (await fs.pathExists(path.join(projectDir, d))) return;
|
|
1649
|
-
}
|
|
1650
|
-
throw new Error('No test directory found');
|
|
1651
|
-
});
|
|
1652
|
-
|
|
1653
|
-
t('Test files exist', 'e2e', 'P1', async () => {
|
|
1654
|
-
const testDirs = ['tests','test','__tests__','spec'];
|
|
1655
|
-
for (const d of testDirs) {
|
|
1656
|
-
const dir = path.join(projectDir, d);
|
|
1657
|
-
if (await fs.pathExists(dir)) {
|
|
1658
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1659
|
-
const testFiles = files.filter(f => /\.(test|spec)\.(ts|js)$/.test(f));
|
|
1660
|
-
if (testFiles.length > 0) return;
|
|
1661
|
-
}
|
|
1662
|
-
}
|
|
1663
|
-
throw new Error('No test files (.test.ts/.spec.ts) found');
|
|
1664
|
-
});
|
|
1665
|
-
|
|
1666
|
-
t('Test framework configured', 'e2e', 'P1', async () => {
|
|
1667
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1668
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1669
|
-
if (!['jest','vitest','mocha','jasmine','@nestjs/testing','pytest','junit'].some(d => deps[d]))
|
|
1670
|
-
throw new Error('No test framework found (jest/vitest/mocha)');
|
|
1671
|
-
});
|
|
1672
|
-
|
|
1673
|
-
t('Test coverage configured', 'e2e', 'P2', async () => {
|
|
1674
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1675
|
-
const hasJestConfig = !!pkg.jest;
|
|
1676
|
-
const hasVitestConfig = await fs.pathExists(path.join(projectDir, 'vitest.config.ts')) ||
|
|
1677
|
-
await fs.pathExists(path.join(projectDir, 'vitest.config.js'));
|
|
1678
|
-
if (!hasJestConfig && !hasVitestConfig)
|
|
1679
|
-
throw new Error('No test coverage configuration (jest/vitest config)');
|
|
1680
|
-
});
|
|
1681
|
-
|
|
1682
|
-
t('API integration tests', 'e2e', 'P1', async () => {
|
|
1683
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1684
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1685
|
-
if (!['supertest','axios-mock-adapter','nock','httpretty'].some(d => deps[d]))
|
|
1686
|
-
throw new Error('No API testing library (supertest/nock) found');
|
|
1687
|
-
});
|
|
1688
|
-
|
|
1689
|
-
t('E2E test directory', 'e2e', 'P2', async () => {
|
|
1690
|
-
const candidates = ['e2e','tests/e2e','test/e2e','cypress','playwright'];
|
|
1691
|
-
for (const c of candidates) {
|
|
1692
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1693
|
-
}
|
|
1694
|
-
throw new Error('No E2E test directory found');
|
|
1695
|
-
});
|
|
1696
|
-
|
|
1697
|
-
t('Mock/fixture files', 'e2e', 'P3', async () => {
|
|
1698
|
-
const candidates = ['tests/fixtures','test/fixtures','src/__mocks__','__fixtures__','mocks'];
|
|
1699
|
-
for (const c of candidates) {
|
|
1700
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1701
|
-
}
|
|
1702
|
-
throw new Error('No test fixtures/mocks directory found');
|
|
1703
|
-
});
|
|
1704
|
-
|
|
1705
|
-
t('Minimum test file count', 'e2e', 'P2', async () => {
|
|
1706
|
-
const testDirs = ['tests','test','__tests__','spec'];
|
|
1707
|
-
let total = 0;
|
|
1708
|
-
for (const d of testDirs) {
|
|
1709
|
-
const dir = path.join(projectDir, d);
|
|
1710
|
-
if (await fs.pathExists(dir)) {
|
|
1711
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1712
|
-
total += files.filter(f => /\.(test|spec)\.(ts|js)$/.test(f)).length;
|
|
1713
|
-
}
|
|
1714
|
-
}
|
|
1715
|
-
if (total < 3) throw new Error(`Only ${total} test file(s) — need at least 3`);
|
|
1716
|
-
});
|
|
1717
|
-
|
|
1718
|
-
// ── API Documentation (5 tests) ───────────────────────────────────────
|
|
1719
|
-
t('API documentation library', 'api-contract', 'P2', async () => {
|
|
1720
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1721
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1722
|
-
if (!['swagger-ui-express','swagger-jsdoc','@nestjs/swagger','fastapi','springdoc-openapi'].some(d => deps[d]))
|
|
1723
|
-
throw new Error('No API documentation library (swagger-ui-express)');
|
|
1724
|
-
});
|
|
1725
|
-
|
|
1726
|
-
t('OpenAPI spec file', 'api-contract', 'P3', async () => {
|
|
1727
|
-
const candidates = ['openapi.json','openapi.yaml','swagger.json','swagger.yaml','api-spec.json'];
|
|
1728
|
-
for (const c of candidates) {
|
|
1729
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1730
|
-
}
|
|
1731
|
-
throw new Error('No OpenAPI/Swagger spec file found');
|
|
1732
|
-
});
|
|
1733
|
-
|
|
1734
|
-
t('API versioning in routes', 'api-contract', 'P2', async () => {
|
|
1735
|
-
const candidates = ['src/routes','src/controllers'];
|
|
1736
|
-
for (const c of candidates) {
|
|
1737
|
-
const dir = path.join(projectDir, c);
|
|
1738
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1739
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1740
|
-
for (const f of files) {
|
|
1741
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1742
|
-
if (/v1|v2|api\/v/i.test(content)) return;
|
|
1743
|
-
}
|
|
1744
|
-
}
|
|
1745
|
-
throw new Error('No API versioning in routes (e.g. /api/v1/)');
|
|
1746
|
-
});
|
|
1747
|
-
|
|
1748
|
-
t('Request/response DTOs defined', 'api-contract', 'P2', async () => {
|
|
1749
|
-
const candidates = ['src/dto','src/dtos','src/types','src/interfaces','dto'];
|
|
1750
|
-
for (const c of candidates) {
|
|
1751
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
1752
|
-
}
|
|
1753
|
-
throw new Error('No DTO/interface definitions found');
|
|
1754
|
-
});
|
|
1755
|
-
|
|
1756
|
-
t('Response envelope pattern', 'api-contract', 'P3', async () => {
|
|
1757
|
-
const candidates = ['src/utils','src/helpers','src/middleware'];
|
|
1758
|
-
for (const c of candidates) {
|
|
1759
|
-
const dir = path.join(projectDir, c);
|
|
1760
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1761
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1762
|
-
for (const f of files) {
|
|
1763
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1764
|
-
if (/success.*true|status.*success|data.*message/i.test(content)) return;
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
throw new Error('No response envelope pattern (success/data/message) found');
|
|
1768
|
-
});
|
|
1769
|
-
|
|
1770
|
-
// ── Performance + System (6 tests) ────────────────────────────────────
|
|
1771
|
-
t('Heap memory acceptable', 'performance', 'P1', async () => {
|
|
1772
|
-
const heapMB = process.memoryUsage().heapUsed / 1024 / 1024;
|
|
1773
|
-
if (heapMB > 512) throw new Error(`Heap ${heapMB.toFixed(0)}MB exceeds 512MB`);
|
|
1774
|
-
});
|
|
1775
|
-
|
|
1776
|
-
t('RSS memory acceptable', 'performance', 'P2', async () => {
|
|
1777
|
-
const rssMB = process.memoryUsage().rss / 1024 / 1024;
|
|
1778
|
-
if (rssMB > 1024) throw new Error(`RSS ${rssMB.toFixed(0)}MB exceeds 1GB`);
|
|
1779
|
-
});
|
|
9
|
+
import * as p from '@clack/prompts';
|
|
10
|
+
import chalk from 'chalk';
|
|
11
|
+
import fs from 'fs-extra';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
import os from 'node:os';
|
|
14
|
+
import { performance } from 'node:perf_hooks';
|
|
15
|
+
import { EventEmitter } from 'node:events';
|
|
1780
16
|
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
17
|
+
import { SmartCrawler } from './browser/crawler.js';
|
|
18
|
+
import { BrowserInteractor } from './browser/interactions.js';
|
|
19
|
+
import { ScreenshotCapture } from './browser/screenshot.js';
|
|
20
|
+
import { RealAPIValidator } from './analyzers/api.js';
|
|
21
|
+
import { SecurityScanner } from './analyzers/security.js';
|
|
22
|
+
import { PerformanceProfiler } from './analyzers/performance.js';
|
|
23
|
+
import { AccessibilityChecker} from './analyzers/accessibility.js';
|
|
24
|
+
import { SEOScanner } from './analyzers/seo.js';
|
|
25
|
+
import { HTMLReporter } from './reporters/html.js';
|
|
26
|
+
import { TerminalDashboard } from './reporters/terminal.js';
|
|
27
|
+
import { JSONReporter } from './reporters/json.js';
|
|
28
|
+
import { AIClassifier } from './utils/ai-classifier.js';
|
|
1785
29
|
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
30
|
+
// ── Constants ─────────────────────────────────────────────────────────────
|
|
31
|
+
export const VERSION = '12.0.0';
|
|
32
|
+
export const QA_DIR = path.join(process.cwd(), '.backlist', 'qa');
|
|
33
|
+
export const REPORT_DIR = path.join(QA_DIR, 'reports');
|
|
34
|
+
export const HISTORY_FILE = path.join(QA_DIR, 'history.json');
|
|
35
|
+
export const SCREENSHOT_DIR = path.join(REPORT_DIR, 'screenshots');
|
|
1789
36
|
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
}
|
|
37
|
+
// ── Utilities ─────────────────────────────────────────────────────────────
|
|
38
|
+
export function timestamp() { return new Date().toISOString(); }
|
|
39
|
+
export function shortId() { return Math.random().toString(36).slice(2, 9); }
|
|
40
|
+
export function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
41
|
+
export function formatDuration(ms) {
|
|
42
|
+
if (ms < 1000) return `${Math.round(ms)}ms`;
|
|
43
|
+
return `${(ms / 1000).toFixed(2)}s`;
|
|
44
|
+
}
|
|
45
|
+
export function formatBytes(b) {
|
|
46
|
+
if (!b || b < 0) return '0B';
|
|
47
|
+
if (b < 1024) return `${b}B`;
|
|
48
|
+
if (b < 1024 * 1024) return `${(b / 1024).toFixed(1)}KB`;
|
|
49
|
+
return `${(b / 1024 / 1024).toFixed(1)}MB`;
|
|
50
|
+
}
|
|
1796
51
|
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
52
|
+
// ── QA Session ────────────────────────────────────────────────────────────
|
|
53
|
+
export class QASession {
|
|
54
|
+
id;
|
|
55
|
+
startedAt;
|
|
56
|
+
urls = {};
|
|
57
|
+
results = []; // real test results only
|
|
58
|
+
bugs = []; // real detected bugs only
|
|
59
|
+
screenshots = []; // real screenshots
|
|
60
|
+
consoleErrors = [];
|
|
61
|
+
networkLog = [];
|
|
62
|
+
apiLog = [];
|
|
63
|
+
routeMap = [];
|
|
64
|
+
perfMetrics = {};
|
|
65
|
+
secFindings = [];
|
|
66
|
+
a11yResults = [];
|
|
67
|
+
seoResults = [];
|
|
68
|
+
|
|
69
|
+
constructor(urls) {
|
|
70
|
+
this.id = `QA-${shortId()}`;
|
|
71
|
+
this.startedAt = timestamp();
|
|
72
|
+
this.urls = urls;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
addResult(result) {
|
|
76
|
+
this.results.push(result);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
addBug(bug) {
|
|
80
|
+
this.bugs.push({ ...bug, id: `BUG-${shortId()}`, createdAt: timestamp() });
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
getSummary() {
|
|
84
|
+
const passed = this.results.filter(r => r.status === 'PASS').length;
|
|
85
|
+
const failed = this.results.filter(r => r.status === 'FAIL').length;
|
|
86
|
+
const skipped = this.results.filter(r => r.status === 'SKIP').length;
|
|
87
|
+
const flaky = this.results.filter(r => r.status === 'FLAKY').length;
|
|
88
|
+
const total = this.results.length;
|
|
89
|
+
return {
|
|
90
|
+
total, passed, failed, skipped, flaky,
|
|
91
|
+
passRate: total > 0 ? ((passed + flaky) / total * 100).toFixed(1) : '0.0',
|
|
92
|
+
duration: Date.now() - new Date(this.startedAt).getTime(),
|
|
1809
93
|
};
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
1813
96
|
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
97
|
+
// ── Main QA Engine ────────────────────────────────────────────────────────
|
|
98
|
+
export class QAEngine extends EventEmitter {
|
|
99
|
+
#session;
|
|
100
|
+
#terminal;
|
|
101
|
+
#crawler;
|
|
102
|
+
#interactor;
|
|
103
|
+
#screenshotter;
|
|
104
|
+
#apiValidator;
|
|
105
|
+
#security;
|
|
106
|
+
#performance;
|
|
107
|
+
#a11y;
|
|
108
|
+
#seo;
|
|
109
|
+
#aiClassifier;
|
|
110
|
+
#aborted = false;
|
|
111
|
+
|
|
112
|
+
constructor(session, options = {}) {
|
|
113
|
+
super();
|
|
114
|
+
this.#session = session;
|
|
115
|
+
this.#terminal = new TerminalDashboard(session);
|
|
116
|
+
this.#screenshotter = new ScreenshotCapture(SCREENSHOT_DIR);
|
|
117
|
+
this.#aiClassifier = new AIClassifier();
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async init() {
|
|
121
|
+
// Dynamically import Playwright — optional peer dependency
|
|
122
|
+
let playwright;
|
|
123
|
+
try {
|
|
124
|
+
playwright = await import('playwright');
|
|
125
|
+
} catch {
|
|
126
|
+
throw new Error(
|
|
127
|
+
'Playwright not installed. Run: npm install playwright && npx playwright install chromium'
|
|
128
|
+
);
|
|
1819
129
|
}
|
|
1820
|
-
throw new Error('No Prettier configuration found');
|
|
1821
|
-
});
|
|
1822
|
-
|
|
1823
|
-
t('Husky/lint-staged configured', 'code-quality', 'P3', async () => {
|
|
1824
|
-
const pkg = await fs.readJson(path.join(projectDir, 'package.json')).catch(() => ({}));
|
|
1825
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1826
|
-
if (!deps['husky'] && !deps['lint-staged'] && !pkg['lint-staged'])
|
|
1827
|
-
throw new Error('No Husky/lint-staged pre-commit hooks configured');
|
|
1828
|
-
});
|
|
1829
130
|
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
for (const f of files) {
|
|
1838
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1839
|
-
count += (content.match(/\/\/\s*(TODO|FIXME|HACK|XXX)/g) || []).length;
|
|
1840
|
-
}
|
|
1841
|
-
}
|
|
1842
|
-
if (count > 5) throw new Error(`${count} TODO/FIXME comments in production code`);
|
|
1843
|
-
});
|
|
131
|
+
this.#crawler = new SmartCrawler(playwright);
|
|
132
|
+
this.#interactor = new BrowserInteractor(playwright, this.#session);
|
|
133
|
+
this.#apiValidator = new RealAPIValidator(this.#session);
|
|
134
|
+
this.#security = new SecurityScanner(this.#session);
|
|
135
|
+
this.#performance = new PerformanceProfiler(this.#session);
|
|
136
|
+
this.#a11y = new AccessibilityChecker(playwright, this.#session);
|
|
137
|
+
this.#seo = new SEOScanner(this.#session);
|
|
1844
138
|
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
const dir = path.join(projectDir, c);
|
|
1849
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1850
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1851
|
-
for (const f of files) {
|
|
1852
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1853
|
-
const lines = content.split('\n');
|
|
1854
|
-
if (lines.length > 500)
|
|
1855
|
-
throw new Error(`${c}/${f} has ${lines.length} lines — split into smaller files`);
|
|
1856
|
-
}
|
|
1857
|
-
}
|
|
1858
|
-
});
|
|
139
|
+
await this.#interactor.launch();
|
|
140
|
+
await this.#screenshotter.init();
|
|
141
|
+
}
|
|
1859
142
|
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
for (const c of candidates) {
|
|
1864
|
-
const dir = path.join(projectDir, c);
|
|
1865
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1866
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1867
|
-
for (const f of files) {
|
|
1868
|
-
const content = await fs.readFile(path.join(dir, f), 'utf8').catch(() => '');
|
|
1869
|
-
count += (content.match(/\/\/.*(?:function|const|let|var|class|return|if\s*\()/g) || []).length;
|
|
1870
|
-
}
|
|
1871
|
-
}
|
|
1872
|
-
if (count > 15) throw new Error(`${count} lines of commented-out code detected`);
|
|
1873
|
-
});
|
|
143
|
+
async run() {
|
|
144
|
+
this.#terminal.start();
|
|
145
|
+
this.emit('session:start', this.#session);
|
|
1874
146
|
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
if (await fs.pathExists(fp)) {
|
|
1880
|
-
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
1881
|
-
const lines = content.split('\n').slice(0, 30);
|
|
1882
|
-
const importLines = lines.filter(l => l.startsWith('import'));
|
|
1883
|
-
if (importLines.length < 2) throw new Error(`${c} has very few imports — may be incomplete`);
|
|
1884
|
-
return;
|
|
1885
|
-
}
|
|
1886
|
-
}
|
|
1887
|
-
});
|
|
147
|
+
try {
|
|
148
|
+
// Phase 1 — Discovery
|
|
149
|
+
this.#terminal.setPhase('🔍 Phase 1: Route Discovery & Crawling');
|
|
150
|
+
await this.#phaseDiscovery();
|
|
1888
151
|
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
if (!deps['madge'] && !deps['dependency-cruiser'])
|
|
1893
|
-
throw new Error('No circular dependency checker (madge/dependency-cruiser) in devDeps');
|
|
1894
|
-
});
|
|
152
|
+
// Phase 2 — API Validation
|
|
153
|
+
this.#terminal.setPhase('📡 Phase 2: Real API Validation');
|
|
154
|
+
await this.#phaseAPIValidation();
|
|
1895
155
|
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
const dir = path.join(projectDir, c);
|
|
1900
|
-
if (!await fs.pathExists(dir)) continue;
|
|
1901
|
-
const files = await fs.readdir(dir).catch(() => []);
|
|
1902
|
-
const mixedCase = files.filter(f => /[A-Z]/.test(f) && /-/.test(f));
|
|
1903
|
-
if (mixedCase.length > 0)
|
|
1904
|
-
throw new Error(`Mixed naming conventions in ${c}: ${mixedCase.join(', ')}`);
|
|
1905
|
-
}
|
|
1906
|
-
});
|
|
156
|
+
// Phase 3 — Browser Interactions
|
|
157
|
+
this.#terminal.setPhase('🖱️ Phase 3: Browser Interaction Testing');
|
|
158
|
+
await this.#phaseBrowserInteractions();
|
|
1907
159
|
|
|
1908
|
-
|
|
1909
|
-
|
|
160
|
+
// Phase 4 — Security Scan
|
|
161
|
+
this.#terminal.setPhase('🛡️ Phase 4: Security Deep Scan');
|
|
162
|
+
await this.#phaseSecurityScan();
|
|
1910
163
|
|
|
1911
|
-
//
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
const t = (name, type, sev, fn) => tests.push({ id: shortId(), name, type, sev, fn });
|
|
164
|
+
// Phase 5 — Performance
|
|
165
|
+
this.#terminal.setPhase('⚡ Phase 5: Performance Profiling');
|
|
166
|
+
await this.#phasePerformance();
|
|
1915
167
|
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
168
|
+
// Phase 6 — Accessibility
|
|
169
|
+
this.#terminal.setPhase('♿ Phase 6: Accessibility Testing');
|
|
170
|
+
await this.#phaseAccessibility();
|
|
1919
171
|
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
const walk = async (dir) => {
|
|
1924
|
-
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
1925
|
-
for (const e of entries) {
|
|
1926
|
-
if (e.isDirectory() && e.name !== 'node_modules') await walk(path.join(dir, e.name));
|
|
1927
|
-
else if (exts.some(x => e.name.endsWith(x))) { found = true; return; }
|
|
1928
|
-
}
|
|
1929
|
-
};
|
|
1930
|
-
await walk(srcDir);
|
|
1931
|
-
if (!found) throw new Error('No component files found (.tsx/.jsx/.vue/.svelte)');
|
|
1932
|
-
});
|
|
172
|
+
// Phase 7 — SEO
|
|
173
|
+
this.#terminal.setPhase('🔎 Phase 7: SEO Validation');
|
|
174
|
+
await this.#phaseSEO();
|
|
1933
175
|
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
if (!entries.some(f => patterns.some(p => f.includes(p))))
|
|
1938
|
-
throw new Error('No styling configuration found');
|
|
1939
|
-
});
|
|
176
|
+
// Phase 8 — AI Bug Classification
|
|
177
|
+
this.#terminal.setPhase('🤖 Phase 8: AI Bug Classification');
|
|
178
|
+
await this.#phaseAIClassification();
|
|
1940
179
|
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
180
|
+
} catch (err) {
|
|
181
|
+
this.emit('engine:error', err);
|
|
182
|
+
throw err;
|
|
183
|
+
} finally {
|
|
184
|
+
this.#terminal.stop();
|
|
185
|
+
await this.#interactor.close().catch(() => {});
|
|
1945
186
|
}
|
|
1946
|
-
throw new Error('No API client/services directory found');
|
|
1947
|
-
});
|
|
1948
|
-
|
|
1949
|
-
t('State management configured', 'ui', 'P2', async () => {
|
|
1950
|
-
const pkg = await fs.readJson(path.join(process.cwd(), 'package.json')).catch(() => ({}));
|
|
1951
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1952
|
-
if (!['redux','@reduxjs/toolkit','zustand','mobx','recoil','jotai','pinia','vuex'].some(d => deps[d]))
|
|
1953
|
-
throw new Error('No state management library found');
|
|
1954
|
-
});
|
|
1955
|
-
|
|
1956
|
-
t('Router configured', 'ui', 'P1', async () => {
|
|
1957
|
-
const pkg = await fs.readJson(path.join(process.cwd(), 'package.json')).catch(() => ({}));
|
|
1958
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1959
|
-
if (!['react-router','react-router-dom','@reach/router','next','nuxt','@angular/router'].some(d => deps[d]))
|
|
1960
|
-
throw new Error('No router library found');
|
|
1961
|
-
});
|
|
1962
187
|
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
1966
|
-
if (!['react-hook-form','formik','vee-validate','@angular/forms'].some(d => deps[d]))
|
|
1967
|
-
throw new Error('No form handling library (react-hook-form/formik)');
|
|
1968
|
-
});
|
|
188
|
+
return this.#session;
|
|
189
|
+
}
|
|
1969
190
|
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
});
|
|
191
|
+
abort() {
|
|
192
|
+
this.#aborted = true;
|
|
193
|
+
this.#terminal.stop();
|
|
194
|
+
this.#interactor.close().catch(() => {});
|
|
195
|
+
}
|
|
1976
196
|
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
const
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
});
|
|
197
|
+
// ── Phase 1: Discovery ─────────────────────────────────────────────────
|
|
198
|
+
async #phaseDiscovery() {
|
|
199
|
+
for (const [label, url] of Object.entries(this.#session.urls)) {
|
|
200
|
+
if (!url) continue;
|
|
201
|
+
this.#terminal.log(`Crawling ${label}: ${url}`);
|
|
1983
202
|
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
});
|
|
203
|
+
const routes = await this.#crawler.crawl(url, {
|
|
204
|
+
maxPages : 60,
|
|
205
|
+
maxDepth : 4,
|
|
206
|
+
onRoute : (route) => {
|
|
207
|
+
this.#session.routeMap.push(route);
|
|
208
|
+
this.#terminal.log(` Found: ${route.url} (${route.type})`);
|
|
209
|
+
},
|
|
210
|
+
});
|
|
1990
211
|
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
212
|
+
this.#addResult({
|
|
213
|
+
name : `[${label}] Route Discovery`,
|
|
214
|
+
type : 'discovery',
|
|
215
|
+
category: 'crawl',
|
|
216
|
+
status : routes.length > 0 ? 'PASS' : 'FAIL',
|
|
217
|
+
message : routes.length > 0
|
|
218
|
+
? `Discovered ${routes.length} routes`
|
|
219
|
+
: 'No routes discovered — site may be unreachable',
|
|
220
|
+
data : { routeCount: routes.length, routes },
|
|
221
|
+
url,
|
|
222
|
+
label,
|
|
223
|
+
});
|
|
1995
224
|
}
|
|
1996
|
-
|
|
1997
|
-
});
|
|
225
|
+
}
|
|
1998
226
|
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
throw new Error('No Loading/Skeleton component found');
|
|
2005
|
-
});
|
|
227
|
+
// ── Phase 2: Real API Validation ───────────────────────────────────────
|
|
228
|
+
async #phaseAPIValidation() {
|
|
229
|
+
const apiRoutes = this.#session.routeMap.filter(r =>
|
|
230
|
+
r.type === 'api' || r.url.includes('/api/')
|
|
231
|
+
);
|
|
2006
232
|
|
|
2007
|
-
|
|
2008
|
-
}
|
|
233
|
+
this.#terminal.log(`Validating ${apiRoutes.length} API endpoints...`);
|
|
2009
234
|
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
await sleep(20);
|
|
2034
|
-
}});
|
|
2035
|
-
}
|
|
235
|
+
for (const route of apiRoutes) {
|
|
236
|
+
if (this.#aborted) break;
|
|
237
|
+
this.#terminal.setCurrentTest(`API: ${route.url}`);
|
|
238
|
+
|
|
239
|
+
const result = await this.#apiValidator.probe(route.url);
|
|
240
|
+
this.#session.apiLog.push(result);
|
|
241
|
+
|
|
242
|
+
this.#addResult({
|
|
243
|
+
name : `API: ${route.url}`,
|
|
244
|
+
type : 'api',
|
|
245
|
+
category: 'api-validation',
|
|
246
|
+
status : result.pass ? 'PASS' : 'FAIL',
|
|
247
|
+
message : result.message,
|
|
248
|
+
data : {
|
|
249
|
+
statusCode : result.statusCode,
|
|
250
|
+
responseTime: result.responseTime,
|
|
251
|
+
contentType: result.contentType,
|
|
252
|
+
body : result.body?.slice(0, 500),
|
|
253
|
+
headers : result.headers,
|
|
254
|
+
},
|
|
255
|
+
url : route.url,
|
|
256
|
+
duration: result.responseTime,
|
|
257
|
+
});
|
|
2036
258
|
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
}});
|
|
259
|
+
if (!result.pass) {
|
|
260
|
+
this.#session.addBug({
|
|
261
|
+
title : `API Failure: ${route.url}`,
|
|
262
|
+
severity : result.statusCode >= 500 ? 'P0' : 'P1',
|
|
263
|
+
type : 'api',
|
|
264
|
+
description: result.message,
|
|
265
|
+
evidence : result,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
2047
268
|
}
|
|
2048
269
|
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
270
|
+
// Detect APIs from network traffic
|
|
271
|
+
const discoveredAPIs = await this.#apiValidator.discoverFromNetworkLog(
|
|
272
|
+
this.#session.networkLog
|
|
273
|
+
);
|
|
274
|
+
for (const api of discoveredAPIs) {
|
|
275
|
+
if (!apiRoutes.find(r => r.url === api.url)) {
|
|
276
|
+
this.#session.apiLog.push(api);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
2052
279
|
}
|
|
2053
|
-
return tests;
|
|
2054
|
-
}
|
|
2055
280
|
|
|
2056
|
-
//
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
#active = false;
|
|
2062
|
-
#startTime = Date.now();
|
|
2063
|
-
#lastResults = [];
|
|
2064
|
-
#runningTest = null;
|
|
2065
|
-
#bugs = [];
|
|
2066
|
-
#log = [];
|
|
2067
|
-
|
|
2068
|
-
start() {
|
|
2069
|
-
this.#active = true;
|
|
2070
|
-
this.#startTime = Date.now();
|
|
2071
|
-
process.stdout.write(CURSOR_HIDE);
|
|
2072
|
-
this.render({});
|
|
2073
|
-
}
|
|
281
|
+
// ── Phase 3: Browser Interactions ─────────────────────────────────────
|
|
282
|
+
async #phaseBrowserInteractions() {
|
|
283
|
+
const pageRoutes = this.#session.routeMap.filter(r =>
|
|
284
|
+
r.type === 'page' || r.type === 'unknown'
|
|
285
|
+
);
|
|
2074
286
|
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
287
|
+
for (const route of pageRoutes.slice(0, 25)) {
|
|
288
|
+
if (this.#aborted) break;
|
|
289
|
+
this.#terminal.setCurrentTest(`Browser: ${route.url}`);
|
|
290
|
+
|
|
291
|
+
const result = await this.#interactor.testPage(route.url, {
|
|
292
|
+
onConsoleError: (err) => {
|
|
293
|
+
this.#session.consoleErrors.push({ url: route.url, ...err });
|
|
294
|
+
},
|
|
295
|
+
onNetworkEvent: (event) => {
|
|
296
|
+
this.#session.networkLog.push({ url: route.url, ...event });
|
|
297
|
+
},
|
|
298
|
+
});
|
|
2080
299
|
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
}
|
|
300
|
+
// Real screenshot on failure
|
|
301
|
+
if (!result.pass || result.consoleErrors.length > 0) {
|
|
302
|
+
const screenshot = await this.#screenshotter.capture(
|
|
303
|
+
result.page,
|
|
304
|
+
`fail-${shortId()}`
|
|
305
|
+
);
|
|
306
|
+
if (screenshot) {
|
|
307
|
+
result.screenshotPath = screenshot;
|
|
308
|
+
this.#session.screenshots.push({ url: route.url, path: screenshot, reason: result.failReason });
|
|
309
|
+
}
|
|
310
|
+
}
|
|
2093
311
|
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
312
|
+
this.#addResult({
|
|
313
|
+
name : `Page: ${route.url}`,
|
|
314
|
+
type : 'browser',
|
|
315
|
+
category: 'interaction',
|
|
316
|
+
status : result.pass ? (result.consoleErrors.length > 0 ? 'FLAKY' : 'PASS') : 'FAIL',
|
|
317
|
+
message : result.message,
|
|
318
|
+
data : {
|
|
319
|
+
loadTime : result.loadTime,
|
|
320
|
+
consoleErrors : result.consoleErrors,
|
|
321
|
+
networkErrors : result.networkErrors,
|
|
322
|
+
interactedElements: result.interactedElements,
|
|
323
|
+
screenshotPath: result.screenshotPath,
|
|
324
|
+
jsErrors : result.jsErrors,
|
|
325
|
+
resourcesFailed: result.resourcesFailed,
|
|
326
|
+
renderTime : result.renderTime,
|
|
327
|
+
domContentLoaded: result.domContentLoaded,
|
|
328
|
+
},
|
|
329
|
+
url : route.url,
|
|
330
|
+
duration: result.loadTime,
|
|
331
|
+
screenshotPath: result.screenshotPath,
|
|
332
|
+
});
|
|
2101
333
|
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
const w = Math.min(process.stdout.columns || 80, 90);
|
|
2114
|
-
const bar = '─'.repeat(w - 2);
|
|
2115
|
-
|
|
2116
|
-
lines.push(chalk.hex('#00F5FF').bold(`┌${bar}┐`));
|
|
2117
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.hex('#BF40FF').bold(` ⚡ BACKLIST MAXIMUM QA — v${VERSION}`.padEnd(w - 2)) + chalk.hex('#00F5FF').bold('│'));
|
|
2118
|
-
lines.push(chalk.hex('#00F5FF').bold(`├${bar}┤`));
|
|
2119
|
-
|
|
2120
|
-
const metrics = [
|
|
2121
|
-
`${chalk.green('✓')} ${chalk.white.bold(passed)} passed`,
|
|
2122
|
-
`${chalk.red('✗')} ${chalk.white.bold(failed)} failed`,
|
|
2123
|
-
`${chalk.yellow('⚠')} ${chalk.white.bold(flaky)} flaky`,
|
|
2124
|
-
`${chalk.cyan('🐛')} ${chalk.white.bold(this.#bugs.length)} bugs`,
|
|
2125
|
-
`${chalk.gray('⏱')} ${chalk.white(elapsed + 's')}`,
|
|
2126
|
-
].map(m => m.padEnd(22)).join(' ');
|
|
2127
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + ' ' + metrics.slice(0, w - 4) + chalk.hex('#00F5FF').bold('│'));
|
|
2128
|
-
|
|
2129
|
-
const pBar = buildProgressBar(passRate, 30);
|
|
2130
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + ` [${pBar}] ${chalk.white.bold(passRate + '%')} (${total} tests run)`.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
2131
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + ` ${DIM('Heap')} ${chalk.white(sys.heapMB + 'MB')} ${DIM('RSS')} ${chalk.white(sys.rss)} ${DIM('Up')} ${chalk.white(sys.uptime + 's')} ${DIM('Node')} ${chalk.white(process.version)}`.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
2132
|
-
lines.push(chalk.hex('#00F5FF').bold(`├${bar}┤`));
|
|
2133
|
-
|
|
2134
|
-
const runLine = this.#runningTest
|
|
2135
|
-
? ` ${chalk.cyan('⟳')} ${chalk.cyan('Running:')} ${chalk.white(this.#runningTest.slice(0, w - 18))}`
|
|
2136
|
-
: ` ${chalk.gray('⊘ Idle...')}`;
|
|
2137
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + runLine.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
2138
|
-
lines.push(chalk.hex('#00F5FF').bold(`├${bar}┤`));
|
|
2139
|
-
|
|
2140
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.gray(' Recent results:').padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
2141
|
-
const recent = results.slice(-6);
|
|
2142
|
-
for (const r of recent) {
|
|
2143
|
-
const type = chalk.gray(`[${(r.type || '').padEnd(13)}]`);
|
|
2144
|
-
const dur = chalk.gray(formatDuration(r.duration));
|
|
2145
|
-
const row = ` ${colorStatus(r.status)} ${type} ${chalk.white(r.name.slice(0, w - 44))} ${dur}`;
|
|
2146
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + row.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
2147
|
-
}
|
|
2148
|
-
for (let i = recent.length; i < 6; i++) {
|
|
2149
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + ' '.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
2150
|
-
}
|
|
334
|
+
// Real console errors → bugs
|
|
335
|
+
for (const err of result.consoleErrors) {
|
|
336
|
+
this.#session.addBug({
|
|
337
|
+
title : `JS Error: ${err.text?.slice(0, 80)}`,
|
|
338
|
+
severity : err.type === 'error' ? 'P1' : 'P2',
|
|
339
|
+
type : 'javascript',
|
|
340
|
+
description: err.text,
|
|
341
|
+
url : route.url,
|
|
342
|
+
evidence : err,
|
|
343
|
+
});
|
|
344
|
+
}
|
|
2151
345
|
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
346
|
+
// Real network failures → bugs
|
|
347
|
+
for (const nErr of result.networkErrors) {
|
|
348
|
+
this.#session.addBug({
|
|
349
|
+
title : `Network Failure: ${nErr.url}`,
|
|
350
|
+
severity : 'P2',
|
|
351
|
+
type : 'network',
|
|
352
|
+
description: `${nErr.method} ${nErr.url} → ${nErr.failure}`,
|
|
353
|
+
url : route.url,
|
|
354
|
+
evidence : nErr,
|
|
355
|
+
});
|
|
356
|
+
}
|
|
2162
357
|
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + (' ' + entry).slice(0, w - 2).padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
2168
|
-
}
|
|
2169
|
-
for (let i = recentLogs.length; i < 4; i++) {
|
|
2170
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + ' '.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
2171
|
-
}
|
|
358
|
+
// Test forms on the page
|
|
359
|
+
if (result.forms && result.forms.length > 0) {
|
|
360
|
+
await this.#testForms(route.url, result.forms, result.page);
|
|
361
|
+
}
|
|
2172
362
|
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
363
|
+
// Test auth flows
|
|
364
|
+
if (this.#isAuthPage(route.url)) {
|
|
365
|
+
await this.#testAuthFlow(route.url, result.page);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
2176
368
|
}
|
|
2177
|
-
}
|
|
2178
369
|
|
|
2179
|
-
//
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
#results = [];
|
|
2184
|
-
#running = false;
|
|
2185
|
-
#aborted = false;
|
|
370
|
+
// ── Phase 4: Security ─────────────────────────────────────────────────
|
|
371
|
+
async #phaseSecurityScan() {
|
|
372
|
+
for (const [label, url] of Object.entries(this.#session.urls)) {
|
|
373
|
+
if (!url) continue;
|
|
2186
374
|
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
this.#aborted = false;
|
|
2190
|
-
this.#results = [];
|
|
375
|
+
const findings = await this.#security.scan(url);
|
|
376
|
+
this.#session.secFindings.push(...findings);
|
|
2191
377
|
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
378
|
+
for (const finding of findings) {
|
|
379
|
+
this.#addResult({
|
|
380
|
+
name : `Security: ${finding.check}`,
|
|
381
|
+
type : 'security',
|
|
382
|
+
category: finding.category,
|
|
383
|
+
status : finding.pass ? 'PASS' : 'FAIL',
|
|
384
|
+
message : finding.detail,
|
|
385
|
+
data : finding.evidence,
|
|
386
|
+
url,
|
|
387
|
+
label,
|
|
388
|
+
severity: finding.severity,
|
|
389
|
+
});
|
|
2199
390
|
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
title : test.name,
|
|
2210
|
-
severity: test.sev || this.#classifySeverity(test.type, result.error),
|
|
2211
|
-
status : 'OPEN',
|
|
391
|
+
if (!finding.pass && (finding.severity === 'P0' || finding.severity === 'P1')) {
|
|
392
|
+
this.#session.addBug({
|
|
393
|
+
title : `Security: ${finding.check}`,
|
|
394
|
+
severity : finding.severity,
|
|
395
|
+
type : 'security',
|
|
396
|
+
description: finding.detail,
|
|
397
|
+
url,
|
|
398
|
+
evidence : finding.evidence,
|
|
399
|
+
recommendation: finding.recommendation,
|
|
2212
400
|
});
|
|
2213
|
-
dashboard.addLog(chalk.red(`✗ FAIL: ${test.name}`));
|
|
2214
|
-
} else {
|
|
2215
|
-
dashboard.addLog(chalk.green(`✓ ${result.status}: ${test.name} (${formatDuration(result.duration)})`));
|
|
2216
401
|
}
|
|
2217
|
-
dashboard.render({});
|
|
2218
|
-
await sleep(40);
|
|
2219
402
|
}
|
|
2220
403
|
}
|
|
2221
|
-
|
|
2222
|
-
this.#running = false;
|
|
2223
|
-
return [...this.#results];
|
|
2224
404
|
}
|
|
2225
405
|
|
|
2226
|
-
|
|
406
|
+
// ── Phase 5: Performance ──────────────────────────────────────────────
|
|
407
|
+
async #phasePerformance() {
|
|
408
|
+
for (const [label, url] of Object.entries(this.#session.urls)) {
|
|
409
|
+
if (!url) continue;
|
|
410
|
+
|
|
411
|
+
const metrics = await this.#performance.profile(url);
|
|
412
|
+
this.#session.perfMetrics[label] = metrics;
|
|
413
|
+
|
|
414
|
+
// Core Web Vitals as real test results
|
|
415
|
+
const vitals = [
|
|
416
|
+
{ name: 'LCP', value: metrics.lcp, threshold: 2500, unit: 'ms' },
|
|
417
|
+
{ name: 'FID', value: metrics.fid, threshold: 100, unit: 'ms' },
|
|
418
|
+
{ name: 'CLS', value: metrics.cls, threshold: 0.1, unit: '' },
|
|
419
|
+
{ name: 'FCP', value: metrics.fcp, threshold: 1800, unit: 'ms' },
|
|
420
|
+
{ name: 'TTFB', value: metrics.ttfb, threshold: 800, unit: 'ms' },
|
|
421
|
+
{ name: 'TTI', value: metrics.tti, threshold: 3800, unit: 'ms' },
|
|
422
|
+
{ name: 'TBT', value: metrics.tbt, threshold: 200, unit: 'ms' },
|
|
423
|
+
];
|
|
424
|
+
|
|
425
|
+
for (const vital of vitals) {
|
|
426
|
+
const pass = vital.value !== null && vital.value <= vital.threshold;
|
|
427
|
+
const na = vital.value === null;
|
|
428
|
+
|
|
429
|
+
this.#addResult({
|
|
430
|
+
name : `[${label}] ${vital.name} — Core Web Vital`,
|
|
431
|
+
type : 'performance',
|
|
432
|
+
category: 'web-vitals',
|
|
433
|
+
status : na ? 'SKIP' : (pass ? 'PASS' : 'FAIL'),
|
|
434
|
+
message : na
|
|
435
|
+
? `${vital.name} not measurable`
|
|
436
|
+
: `${vital.name}: ${vital.value}${vital.unit} (threshold: ${vital.threshold}${vital.unit})`,
|
|
437
|
+
data : { value: vital.value, threshold: vital.threshold, unit: vital.unit },
|
|
438
|
+
url,
|
|
439
|
+
label,
|
|
440
|
+
duration: vital.value,
|
|
441
|
+
});
|
|
2227
442
|
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
443
|
+
if (!na && !pass) {
|
|
444
|
+
this.#session.addBug({
|
|
445
|
+
title : `Poor ${vital.name}: ${vital.value}${vital.unit} (>${vital.threshold}${vital.unit})`,
|
|
446
|
+
severity : vital.name === 'LCP' || vital.name === 'CLS' ? 'P1' : 'P2',
|
|
447
|
+
type : 'performance',
|
|
448
|
+
description: `${vital.name} exceeds threshold on ${label}`,
|
|
449
|
+
url,
|
|
450
|
+
evidence : { value: vital.value, threshold: vital.threshold },
|
|
451
|
+
recommendation: `Optimize ${vital.name} — see https://web.dev/vitals`,
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
}
|
|
2236
455
|
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
} catch (err) {
|
|
2251
|
-
lastError = err.message;
|
|
2252
|
-
if (attempt < FLAKY_RETRY_COUNT) await sleep(300);
|
|
456
|
+
// Real resource analysis
|
|
457
|
+
for (const resource of (metrics.slowResources || [])) {
|
|
458
|
+
this.#addResult({
|
|
459
|
+
name : `[${label}] Slow resource: ${resource.url.split('/').pop()}`,
|
|
460
|
+
type : 'performance',
|
|
461
|
+
category: 'resource',
|
|
462
|
+
status : 'FAIL',
|
|
463
|
+
message : `${resource.url} took ${resource.duration}ms (${formatBytes(resource.size)})`,
|
|
464
|
+
data : resource,
|
|
465
|
+
url,
|
|
466
|
+
label,
|
|
467
|
+
duration: resource.duration,
|
|
468
|
+
});
|
|
2253
469
|
}
|
|
2254
470
|
}
|
|
2255
|
-
|
|
2256
|
-
return { id, name, type, sev, status: 'FAIL', duration: Date.now() - start, retries: FLAKY_RETRY_COUNT, error: lastError };
|
|
2257
471
|
}
|
|
2258
|
-
}
|
|
2259
472
|
|
|
2260
|
-
//
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
const startedAt = timestamp();
|
|
2266
|
-
const allTests = [];
|
|
2267
|
-
const probes = [];
|
|
473
|
+
// ── Phase 6: Accessibility ────────────────────────────────────────────
|
|
474
|
+
async #phaseAccessibility() {
|
|
475
|
+
const pageRoutes = this.#session.routeMap
|
|
476
|
+
.filter(r => r.type === 'page' || r.type === 'unknown')
|
|
477
|
+
.slice(0, 15);
|
|
2268
478
|
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
479
|
+
for (const route of pageRoutes) {
|
|
480
|
+
if (this.#aborted) break;
|
|
481
|
+
this.#terminal.setCurrentTest(`A11y: ${route.url}`);
|
|
482
|
+
|
|
483
|
+
const result = await this.#a11y.check(route.url);
|
|
484
|
+
this.#session.a11yResults.push({ url: route.url, ...result });
|
|
485
|
+
|
|
486
|
+
for (const violation of result.violations) {
|
|
487
|
+
this.#addResult({
|
|
488
|
+
name : `A11y [${violation.impact}]: ${violation.description}`,
|
|
489
|
+
type : 'accessibility',
|
|
490
|
+
category: violation.category || 'wcag',
|
|
491
|
+
status : 'FAIL',
|
|
492
|
+
message : `${violation.nodes} element(s) affected — ${violation.help}`,
|
|
493
|
+
data : {
|
|
494
|
+
impact : violation.impact,
|
|
495
|
+
wcagTags: violation.tags,
|
|
496
|
+
nodes : violation.affectedNodes,
|
|
497
|
+
helpUrl : violation.helpUrl,
|
|
498
|
+
},
|
|
499
|
+
url : route.url,
|
|
500
|
+
severity: violation.impact === 'critical' ? 'P0'
|
|
501
|
+
: violation.impact === 'serious' ? 'P1'
|
|
502
|
+
: violation.impact === 'moderate' ? 'P2' : 'P3',
|
|
503
|
+
});
|
|
2273
504
|
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
505
|
+
if (violation.impact === 'critical' || violation.impact === 'serious') {
|
|
506
|
+
this.#session.addBug({
|
|
507
|
+
title : `A11y: ${violation.description}`,
|
|
508
|
+
severity : violation.impact === 'critical' ? 'P0' : 'P1',
|
|
509
|
+
type : 'accessibility',
|
|
510
|
+
description: `${violation.nodes} element(s): ${violation.help}`,
|
|
511
|
+
url : route.url,
|
|
512
|
+
evidence : violation.affectedNodes,
|
|
513
|
+
recommendation: violation.helpUrl,
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
}
|
|
2279
517
|
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
518
|
+
// Passes also recorded as real results
|
|
519
|
+
for (const pass of (result.passes || []).slice(0, 5)) {
|
|
520
|
+
this.#addResult({
|
|
521
|
+
name : `A11y Pass: ${pass.description}`,
|
|
522
|
+
type : 'accessibility',
|
|
523
|
+
category: 'wcag',
|
|
524
|
+
status : 'PASS',
|
|
525
|
+
message : `${pass.nodes} element(s) verified`,
|
|
526
|
+
data : pass,
|
|
527
|
+
url : route.url,
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
}
|
|
2289
531
|
}
|
|
2290
532
|
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
533
|
+
// ── Phase 7: SEO ──────────────────────────────────────────────────────
|
|
534
|
+
async #phaseSEO() {
|
|
535
|
+
const pageRoutes = this.#session.routeMap
|
|
536
|
+
.filter(r => r.type === 'page' || r.type === 'unknown')
|
|
537
|
+
.slice(0, 20);
|
|
2294
538
|
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
539
|
+
for (const route of pageRoutes) {
|
|
540
|
+
if (this.#aborted) break;
|
|
541
|
+
this.#terminal.setCurrentTest(`SEO: ${route.url}`);
|
|
542
|
+
|
|
543
|
+
const result = await this.#seo.scan(route.url);
|
|
544
|
+
this.#session.seoResults.push({ url: route.url, ...result });
|
|
545
|
+
|
|
546
|
+
for (const check of result.checks) {
|
|
547
|
+
this.#addResult({
|
|
548
|
+
name : `SEO: ${check.name} — ${new URL(route.url).pathname}`,
|
|
549
|
+
type : 'seo',
|
|
550
|
+
category: check.category,
|
|
551
|
+
status : check.pass ? 'PASS' : 'FAIL',
|
|
552
|
+
message : check.detail,
|
|
553
|
+
data : check.data,
|
|
554
|
+
url : route.url,
|
|
555
|
+
severity: check.severity,
|
|
556
|
+
});
|
|
2298
557
|
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
status : 'OPEN',
|
|
2311
|
-
description: r.error || '',
|
|
2312
|
-
type : r.type,
|
|
2313
|
-
createdAt : timestamp(),
|
|
2314
|
-
});
|
|
558
|
+
if (!check.pass && (check.severity === 'P0' || check.severity === 'P1')) {
|
|
559
|
+
this.#session.addBug({
|
|
560
|
+
title : `SEO: ${check.name}`,
|
|
561
|
+
severity : check.severity,
|
|
562
|
+
type : 'seo',
|
|
563
|
+
description: check.detail,
|
|
564
|
+
url : route.url,
|
|
565
|
+
recommendation: check.recommendation,
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
}
|
|
2315
569
|
}
|
|
2316
|
-
});
|
|
2317
|
-
|
|
2318
|
-
if (dashboard) dashboard.start();
|
|
2319
|
-
const results = await runner.run(allTests, dashboard);
|
|
2320
|
-
if (dashboard) dashboard.stop();
|
|
2321
|
-
|
|
2322
|
-
const routeScans = [];
|
|
2323
|
-
for (const { probe, label } of probes) {
|
|
2324
|
-
if (!silent) console.log(chalk.gray(`\n Route scan for ${label} (${COMMON_ROUTES.length} routes)...`));
|
|
2325
|
-
const ps = await probe.probeRoutes(COMMON_ROUTES);
|
|
2326
|
-
routeScans.push(...ps.map(r => ({ label, ...r })));
|
|
2327
570
|
}
|
|
2328
571
|
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
if (!silent) printResultsSummary(results, autoBugs);
|
|
2334
|
-
|
|
2335
|
-
const run = {
|
|
2336
|
-
id: runId, type: 'url-qa', version: VERSION, startedAt, duration,
|
|
2337
|
-
urls: probes.map(p => ({ label: p.label, url: p.url })),
|
|
2338
|
-
results, bugReports: autoBugs, summary, coverage, routeScans,
|
|
2339
|
-
};
|
|
2340
|
-
|
|
2341
|
-
await saveRun(run);
|
|
2342
|
-
const reportFile = await exportReport(run);
|
|
2343
|
-
if (!silent && reportFile) console.log(chalk.gray(` 📄 Report: ${reportFile}`));
|
|
2344
|
-
|
|
2345
|
-
return run;
|
|
2346
|
-
}
|
|
2347
|
-
|
|
2348
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
2349
|
-
// Automated QA — Full Maximum Scan
|
|
2350
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
2351
|
-
export async function runAutomatedQA({ continuous = false, localUrl, prodUrl } = {}) {
|
|
2352
|
-
const runOnce = async () => {
|
|
2353
|
-
const runId = `AQA-${shortId()}`;
|
|
2354
|
-
const startedAt = timestamp();
|
|
2355
|
-
|
|
2356
|
-
console.log('');
|
|
2357
|
-
console.log(chalk.hex('#BF40FF').bold(` ── 🤖 MAXIMUM Automated QA v${VERSION} ────────────────`));
|
|
2358
|
-
console.log('');
|
|
2359
|
-
|
|
2360
|
-
let endpoints = [];
|
|
2361
|
-
try {
|
|
2362
|
-
const { analyzeFrontend } = await import('../analyzer.js');
|
|
2363
|
-
endpoints = await analyzeFrontend(path.join(process.cwd(), 'src'));
|
|
2364
|
-
} catch {}
|
|
2365
|
-
|
|
2366
|
-
const allTests = [
|
|
2367
|
-
...buildFullSystemTests(),
|
|
2368
|
-
...buildEndpointTests(endpoints),
|
|
2369
|
-
...buildUITests(),
|
|
2370
|
-
];
|
|
572
|
+
// ── Phase 8: AI Classification ────────────────────────────────────────
|
|
573
|
+
async #phaseAIClassification() {
|
|
574
|
+
this.#terminal.log(`AI classifying ${this.#session.bugs.length} bugs...`);
|
|
2371
575
|
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
576
|
+
for (const bug of this.#session.bugs) {
|
|
577
|
+
const classification = await this.#aiClassifier.classify(bug, this.#session);
|
|
578
|
+
bug.aiSeverity = classification.severity;
|
|
579
|
+
bug.aiCategory = classification.category;
|
|
580
|
+
bug.aiRecommendation = classification.recommendation;
|
|
581
|
+
bug.aiConfidence = classification.confidence;
|
|
582
|
+
}
|
|
2375
583
|
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
// ← UNLIMITED bugs
|
|
2381
|
-
runner.on('result', r => {
|
|
2382
|
-
if (r.status === 'FAIL') {
|
|
2383
|
-
autoBugs.push({
|
|
2384
|
-
id : `AUTO-${shortId()}`,
|
|
2385
|
-
title : `Automated: ${r.name}`,
|
|
2386
|
-
severity : r.sev || (r.type === 'security' || r.type === 'auth' ? 'P0' :
|
|
2387
|
-
r.type === 'database' || r.type === 'error-handling' ? 'P1' :
|
|
2388
|
-
r.type === 'e2e' ? 'P1' : 'P2'),
|
|
2389
|
-
status : 'OPEN',
|
|
2390
|
-
description: r.error || '',
|
|
2391
|
-
type : r.type,
|
|
2392
|
-
createdAt : timestamp(),
|
|
2393
|
-
});
|
|
2394
|
-
}
|
|
584
|
+
// Sort bugs by AI-determined severity
|
|
585
|
+
this.#session.bugs.sort((a, b) => {
|
|
586
|
+
const order = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
587
|
+
return (order[a.aiSeverity || a.severity] || 3) - (order[b.aiSeverity || b.severity] || 3);
|
|
2395
588
|
});
|
|
589
|
+
}
|
|
2396
590
|
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
591
|
+
// ── Form Testing ──────────────────────────────────────────────────────
|
|
592
|
+
async #testForms(url, forms, page) {
|
|
593
|
+
for (const form of forms.slice(0, 3)) {
|
|
594
|
+
this.#terminal.setCurrentTest(`Form: ${url} — ${form.action || 'unknown'}`);
|
|
595
|
+
|
|
596
|
+
const result = await this.#interactor.testForm(page, form);
|
|
597
|
+
|
|
598
|
+
this.#addResult({
|
|
599
|
+
name : `Form test: ${url} → ${form.action || 'inline'}`,
|
|
600
|
+
type : 'form',
|
|
601
|
+
category: 'user-flow',
|
|
602
|
+
status : result.pass ? 'PASS' : 'FAIL',
|
|
603
|
+
message : result.message,
|
|
604
|
+
data : {
|
|
605
|
+
fields : form.fields,
|
|
606
|
+
action : form.action,
|
|
607
|
+
method : form.method,
|
|
608
|
+
validationOk: result.validationOk,
|
|
609
|
+
submissionOk: result.submissionOk,
|
|
610
|
+
errors : result.errors,
|
|
611
|
+
},
|
|
612
|
+
url,
|
|
613
|
+
duration: result.duration,
|
|
614
|
+
});
|
|
2400
615
|
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
616
|
+
if (!result.pass) {
|
|
617
|
+
this.#session.addBug({
|
|
618
|
+
title : `Form broken: ${form.action || url}`,
|
|
619
|
+
severity : 'P1',
|
|
620
|
+
type : 'form',
|
|
621
|
+
description: result.message,
|
|
622
|
+
url,
|
|
623
|
+
evidence : result.errors,
|
|
624
|
+
});
|
|
2407
625
|
}
|
|
2408
626
|
}
|
|
2409
|
-
|
|
2410
|
-
const duration = Date.now() - new Date(startedAt).getTime();
|
|
2411
|
-
const summary = buildSummary(results);
|
|
2412
|
-
const coverage = buildCoverageMatrix(results);
|
|
2413
|
-
|
|
2414
|
-
printResultsSummary(results, autoBugs);
|
|
2415
|
-
|
|
2416
|
-
const run = {
|
|
2417
|
-
id: runId, type: 'automated', version: VERSION, startedAt, duration,
|
|
2418
|
-
results, bugReports: autoBugs, summary, coverage,
|
|
2419
|
-
urls: [localUrl, prodUrl].filter(Boolean).map((u, i) => ({
|
|
2420
|
-
label: i === 0 ? 'localhost' : 'production', url: u,
|
|
2421
|
-
})),
|
|
2422
|
-
};
|
|
2423
|
-
|
|
2424
|
-
await saveRun(run);
|
|
2425
|
-
const reportFile = await exportReport(run);
|
|
2426
|
-
if (reportFile) console.log(chalk.gray(` 📄 Report: ${reportFile}`));
|
|
2427
|
-
await printRunDiff(run);
|
|
2428
|
-
|
|
2429
|
-
p.outro(chalk.hex('#00F5FF').bold(
|
|
2430
|
-
`Run ${runId} — ${results.length} tests · ${autoBugs.length} bugs · ${formatDuration(duration)}`
|
|
2431
|
-
));
|
|
2432
|
-
return run;
|
|
2433
|
-
};
|
|
2434
|
-
|
|
2435
|
-
if (!continuous) { await runOnce(); return; }
|
|
2436
|
-
|
|
2437
|
-
console.log(chalk.cyan(` ⚡ Continuous mode — every ${WATCH_INTERVAL_MS / 1000}s. Ctrl+C to stop.\n`));
|
|
2438
|
-
let i = 0;
|
|
2439
|
-
while (true) {
|
|
2440
|
-
console.log(chalk.gray(`\n ── Iteration ${++i} ── ${new Date().toLocaleTimeString()}`));
|
|
2441
|
-
await runOnce();
|
|
2442
|
-
await sleep(WATCH_INTERVAL_MS);
|
|
2443
627
|
}
|
|
2444
|
-
}
|
|
2445
628
|
|
|
2446
|
-
//
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
function buildCoverageMatrix(results) {
|
|
2450
|
-
const matrix = {};
|
|
2451
|
-
for (const r of results) {
|
|
2452
|
-
if (!matrix[r.type]) matrix[r.type] = { total: 0, passed: 0, failed: 0, skipped: 0, flaky: 0 };
|
|
2453
|
-
matrix[r.type].total++;
|
|
2454
|
-
if (r.status === 'PASS') matrix[r.type].passed++;
|
|
2455
|
-
if (r.status === 'FAIL') matrix[r.type].failed++;
|
|
2456
|
-
if (r.status === 'SKIP') matrix[r.type].skipped++;
|
|
2457
|
-
if (r.status === 'FLAKY') { matrix[r.type].flaky++; matrix[r.type].passed++; }
|
|
2458
|
-
}
|
|
2459
|
-
return matrix;
|
|
2460
|
-
}
|
|
629
|
+
// ── Auth Flow Testing ─────────────────────────────────────────────────
|
|
630
|
+
async #testAuthFlow(url, page) {
|
|
631
|
+
this.#terminal.setCurrentTest(`Auth flow: ${url}`);
|
|
2461
632
|
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
};
|
|
2470
|
-
}
|
|
633
|
+
const result = await this.#interactor.testAuthFlow(page, url, {
|
|
634
|
+
testCredentials: [
|
|
635
|
+
{ username: 'test@example.com', password: 'wrong-password-test', expectFail: true },
|
|
636
|
+
{ username: 'invalid@test.com', password: 'wrong123', expectFail: true },
|
|
637
|
+
{ username: '', password: '', expectFail: true },
|
|
638
|
+
],
|
|
639
|
+
});
|
|
2471
640
|
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
641
|
+
this.#addResult({
|
|
642
|
+
name : `Auth flow: ${url}`,
|
|
643
|
+
type : 'auth',
|
|
644
|
+
category: 'authentication',
|
|
645
|
+
status : result.pass ? 'PASS' : 'FAIL',
|
|
646
|
+
message : result.message,
|
|
647
|
+
data : result.details,
|
|
648
|
+
url,
|
|
649
|
+
duration: result.duration,
|
|
650
|
+
});
|
|
2476
651
|
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
652
|
+
if (!result.pass) {
|
|
653
|
+
this.#session.addBug({
|
|
654
|
+
title : `Auth flow issue: ${url}`,
|
|
655
|
+
severity : 'P0',
|
|
656
|
+
type : 'auth',
|
|
657
|
+
description: result.message,
|
|
658
|
+
url,
|
|
659
|
+
evidence : result.details,
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
}
|
|
2480
663
|
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
console.log(` Tests: ${chalk.white.bold(results.length)} total`);
|
|
2484
|
-
console.log(` Pass rate: [${buildProgressBar(passRate, 24)}] ${chalk.white.bold(passRate + '%')}`);
|
|
2485
|
-
console.log(` ${chalk.green('✓')} ${passed} passed ${chalk.red('✗')} ${failed} failed`);
|
|
2486
|
-
console.log(` Bugs: ${chalk.white.bold(bugs.length)} total — ${chalk.red.bold(sevGroups.P0.length)} P0 · ${chalk.yellow.bold(sevGroups.P1.length)} P1 · ${chalk.cyan(sevGroups.P2.length)} P2 · ${chalk.gray(sevGroups.P3.length)} P3`);
|
|
2487
|
-
|
|
2488
|
-
if (failed > 0) {
|
|
2489
|
-
console.log('');
|
|
2490
|
-
console.log(chalk.red.bold(' Failed Tests:'));
|
|
2491
|
-
results.filter(r => r.status === 'FAIL').forEach(f => {
|
|
2492
|
-
const sev = f.sev ? ` [${colorSeverity(f.sev)}]` : '';
|
|
2493
|
-
console.log(chalk.red(` ✗${sev} [${f.type}] ${f.name}`));
|
|
2494
|
-
if (f.error) console.log(chalk.gray(` → ${f.error}`));
|
|
2495
|
-
});
|
|
664
|
+
#isAuthPage(url) {
|
|
665
|
+
return /\/(login|signin|auth|register|signup)/i.test(url);
|
|
2496
666
|
}
|
|
2497
667
|
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
668
|
+
// ── Add real result ────────────────────────────────────────────────────
|
|
669
|
+
#addResult(result) {
|
|
670
|
+
const r = {
|
|
671
|
+
id : shortId(),
|
|
672
|
+
timestamp: timestamp(),
|
|
673
|
+
duration : result.duration || 0,
|
|
674
|
+
...result,
|
|
675
|
+
};
|
|
676
|
+
this.#session.addResult(r);
|
|
677
|
+
this.#terminal.addResult(r);
|
|
678
|
+
this.emit('result', r);
|
|
679
|
+
return r;
|
|
2505
680
|
}
|
|
2506
|
-
console.log('');
|
|
2507
681
|
}
|
|
2508
682
|
|
|
2509
683
|
// ─────────────────────────────────────────────────────────────────────────
|
|
2510
|
-
//
|
|
684
|
+
// Public API — exported functions
|
|
2511
685
|
// ─────────────────────────────────────────────────────────────────────────
|
|
2512
|
-
function buildHTMLReport(runData) {
|
|
2513
|
-
const { id, startedAt, duration, results, bugReports, coverage, summary, urls = [], routeScans = [] } = runData;
|
|
2514
|
-
const passRate = summary.total > 0 ? ((summary.passed / summary.total) * 100).toFixed(1) : 0;
|
|
2515
|
-
const statusColor = passRate >= 90 ? '#22c55e' : passRate >= 70 ? '#f59e0b' : '#ef4444';
|
|
2516
|
-
|
|
2517
|
-
const typeColors = {
|
|
2518
|
-
'happy-path' : ['#064e3b','#34d399'],
|
|
2519
|
-
'validation' : ['#1e3a5f','#60a5fa'],
|
|
2520
|
-
'auth' : ['#3b1f5e','#c084fc'],
|
|
2521
|
-
'edge-case' : ['#3b2a1a','#f59e0b'],
|
|
2522
|
-
'performance' : ['#1a2a3b','#38bdf8'],
|
|
2523
|
-
'security' : ['#450a0a','#f87171'],
|
|
2524
|
-
'e2e' : ['#1a3b2a','#4ade80'],
|
|
2525
|
-
'ui' : ['#2a1a3b','#a78bfa'],
|
|
2526
|
-
'http' : ['#0f2a3b','#38bdf8'],
|
|
2527
|
-
'seo' : ['#1a2e0f','#86efac'],
|
|
2528
|
-
'a11y' : ['#2e1a0f','#fca5a5'],
|
|
2529
|
-
'links' : ['#0f1a2e','#93c5fd'],
|
|
2530
|
-
'database' : ['#1a1a3b','#818cf8'],
|
|
2531
|
-
'docker' : ['#0f2a2a','#2dd4bf'],
|
|
2532
|
-
'ci' : ['#1a2e1a','#86efac'],
|
|
2533
|
-
'code-quality' : ['#2e2a0f','#fde68a'],
|
|
2534
|
-
'typescript' : ['#0f1a3b','#60a5fa'],
|
|
2535
|
-
'environment' : ['#2a0f1a','#f9a8d4'],
|
|
2536
|
-
'logging' : ['#1a2a1a','#4ade80'],
|
|
2537
|
-
'error-handling': ['#3b1a1a','#fca5a5'],
|
|
2538
|
-
'api-contract' : ['#1a0f2e','#c4b5fd'],
|
|
2539
|
-
'dependency' : ['#2a1a0f','#fdba74'],
|
|
2540
|
-
'cors' : ['#0f2a1a','#6ee7b7'],
|
|
2541
|
-
'rate-limit' : ['#2a0f2a','#d8b4fe'],
|
|
2542
|
-
'file-structure': ['#0f1a1a','#7dd3fc'],
|
|
2543
|
-
};
|
|
2544
|
-
|
|
2545
|
-
const badgeStyle = (type) => {
|
|
2546
|
-
const [bg, fg] = typeColors[type] ?? ['#1e293b','#94a3b8'];
|
|
2547
|
-
return `background:${bg};color:${fg};padding:2px 8px;border-radius:4px;font-size:.7rem;font-weight:500`;
|
|
2548
|
-
};
|
|
2549
|
-
|
|
2550
|
-
const covBars = Object.entries(coverage).map(([type, d]) => {
|
|
2551
|
-
const pct = d.total ? ((d.passed / d.total) * 100).toFixed(0) : 0;
|
|
2552
|
-
const [, fg] = typeColors[type] ?? ['','#94a3b8'];
|
|
2553
|
-
return `<div style="display:flex;align-items:center;gap:.75rem;margin-bottom:.5rem">
|
|
2554
|
-
<div style="width:130px;font-size:.75rem;color:#94a3b8">${type}</div>
|
|
2555
|
-
<div style="flex:1;background:#2d2d4e;border-radius:4px;height:7px;overflow:hidden">
|
|
2556
|
-
<div style="height:100%;width:${pct}%;background:${fg};border-radius:4px"></div>
|
|
2557
|
-
</div>
|
|
2558
|
-
<div style="width:70px;text-align:right;font-size:.75rem;color:#64748b">${d.passed}/${d.total} (${pct}%)</div>
|
|
2559
|
-
</div>`;
|
|
2560
|
-
}).join('');
|
|
2561
|
-
|
|
2562
|
-
const rows = results.map(r => `<tr class="${r.status.toLowerCase()}">
|
|
2563
|
-
<td>${r.name}</td>
|
|
2564
|
-
<td><span style="${badgeStyle(r.type)}">${r.type}</span></td>
|
|
2565
|
-
<td><span class="status status-${r.status.toLowerCase()}">${r.status}</span></td>
|
|
2566
|
-
<td>${r.sev ? `<span class="sev-${(r.sev||'').toLowerCase()}">${r.sev}</span>` : '—'}</td>
|
|
2567
|
-
<td>${r.duration}ms</td>
|
|
2568
|
-
<td>${r.retries > 0 ? `<span style="background:#422006;color:#fb923c;padding:2px 6px;border-radius:3px;font-size:.7rem">${r.retries}x</span>` : '—'}</td>
|
|
2569
|
-
<td class="err">${r.error ? `<code>${r.error}</code>` : '—'}</td>
|
|
2570
|
-
</tr>`).join('');
|
|
2571
|
-
|
|
2572
|
-
// All bug reports — no limit
|
|
2573
|
-
const sevOrder = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
2574
|
-
const sortedBugs = [...bugReports].sort((a, b) => (sevOrder[a.severity] || 3) - (sevOrder[b.severity] || 3));
|
|
2575
|
-
const bugCards = sortedBugs.length
|
|
2576
|
-
? sortedBugs.map(b => `
|
|
2577
|
-
<div class="bug-card bug-${(b.severity||'p3').toLowerCase()}">
|
|
2578
|
-
<div class="bug-header">
|
|
2579
|
-
<span class="bug-id">${b.id}</span>
|
|
2580
|
-
<span class="bug-sev">${b.severity}</span>
|
|
2581
|
-
<span class="bug-type">[${b.type || 'general'}]</span>
|
|
2582
|
-
<span class="bug-st">${b.status}</span>
|
|
2583
|
-
</div>
|
|
2584
|
-
<div class="bug-title">${b.title}</div>
|
|
2585
|
-
${b.description ? `<div class="bug-desc">${b.description}</div>` : ''}
|
|
2586
|
-
</div>`).join('')
|
|
2587
|
-
: '<p style="color:#34d399;text-align:center;padding:1rem">No bugs 🎉</p>';
|
|
2588
|
-
|
|
2589
|
-
const sevCounts = { P0: 0, P1: 0, P2: 0, P3: 0 };
|
|
2590
|
-
bugReports.forEach(b => { if (sevCounts[b.severity] !== undefined) sevCounts[b.severity]++; });
|
|
2591
|
-
|
|
2592
|
-
const routeCards = routeScans.length
|
|
2593
|
-
? routeScans.map(r => `
|
|
2594
|
-
<div style="display:flex;align-items:center;gap:.75rem;padding:.4rem .75rem;border-bottom:1px solid #1a1a2e;font-size:.78rem">
|
|
2595
|
-
<span style="font-family:monospace;min-width:36px;color:${r.status>=200&&r.status<400?'#34d399':r.status>=500?'#f87171':'#f59e0b'}">${r.status||'ERR'}</span>
|
|
2596
|
-
<span style="flex:1;color:#94a3b8;font-family:monospace">${r.route}</span>
|
|
2597
|
-
<span style="color:#64748b">${r.duration}ms</span>
|
|
2598
|
-
<span style="font-size:.68rem;padding:1px 5px;background:#1e293b;color:#64748b;border-radius:3px">${r.label}</span>
|
|
2599
|
-
</div>`).join('')
|
|
2600
|
-
: '<p style="color:#64748b;font-size:.85rem;padding:.5rem">No route scans.</p>';
|
|
2601
|
-
|
|
2602
|
-
const urlCards = urls.length
|
|
2603
|
-
? urls.map(u => `
|
|
2604
|
-
<div style="background:#1e1e30;border:1px solid #2d2d4e;border-radius:8px;padding:.75rem 1rem;margin-bottom:.5rem;display:flex;justify-content:space-between;align-items:center">
|
|
2605
|
-
<span style="font-size:.75rem;color:#64748b;text-transform:uppercase">${u.label}</span>
|
|
2606
|
-
<a href="${u.url}" target="_blank" style="font-size:.8rem;color:#60a5fa">${u.url}</a>
|
|
2607
|
-
</div>`).join('')
|
|
2608
|
-
: '';
|
|
2609
|
-
|
|
2610
|
-
const chartLabels = JSON.stringify(Object.keys(coverage));
|
|
2611
|
-
const chartPassed = JSON.stringify(Object.values(coverage).map(d => d.passed));
|
|
2612
|
-
const chartFailed = JSON.stringify(Object.values(coverage).map(d => d.failed));
|
|
2613
|
-
const chartFlaky = JSON.stringify(Object.values(coverage).map(d => d.flaky));
|
|
2614
|
-
|
|
2615
|
-
return `<!DOCTYPE html>
|
|
2616
|
-
<html lang="en">
|
|
2617
|
-
<head>
|
|
2618
|
-
<meta charset="UTF-8">
|
|
2619
|
-
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
2620
|
-
<title>Backlist MAXIMUM QA v${VERSION} — ${id}</title>
|
|
2621
|
-
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.4.1/chart.umd.js"></script>
|
|
2622
|
-
<style>
|
|
2623
|
-
*{box-sizing:border-box;margin:0;padding:0}
|
|
2624
|
-
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0a0a12;color:#e2e8f0;font-size:14px;line-height:1.6}
|
|
2625
|
-
header{background:#0f0f1e;border-bottom:2px solid #00f5ff33;padding:1.5rem 2rem;display:flex;align-items:center;justify-content:space-between}
|
|
2626
|
-
header h1{font-size:1.3rem;font-weight:700;color:#00f5ff}
|
|
2627
|
-
header p{color:#64748b;font-size:.82rem;margin-top:4px}
|
|
2628
|
-
.badge{font-size:.72rem;color:#BF40FF;padding:3px 12px;border:1px solid #BF40FF;border-radius:20px}
|
|
2629
|
-
.container{max-width:1300px;margin:0 auto;padding:2rem}
|
|
2630
|
-
.metrics{display:grid;grid-template-columns:repeat(auto-fit,minmax(130px,1fr));gap:.75rem;margin-bottom:1.5rem}
|
|
2631
|
-
.mc{background:#1e1e30;border:1px solid #2d2d4e;border-radius:10px;padding:1rem 1.25rem;transition:border .2s}
|
|
2632
|
-
.mc:hover{border-color:#00f5ff44}
|
|
2633
|
-
.ml{font-size:.68rem;color:#64748b;text-transform:uppercase;letter-spacing:.08em}
|
|
2634
|
-
.mv{font-size:1.9rem;font-weight:700;margin-top:4px}
|
|
2635
|
-
.section{background:#1e1e30;border:1px solid #2d2d4e;border-radius:10px;padding:1.5rem;margin-bottom:1.25rem}
|
|
2636
|
-
.section-title{font-size:.92rem;font-weight:600;margin-bottom:1rem;color:#cbd5e1;border-bottom:1px solid #2d2d4e;padding-bottom:.75rem;display:flex;justify-content:space-between;align-items:center}
|
|
2637
|
-
.grid2{display:grid;grid-template-columns:1fr 1fr;gap:1rem}
|
|
2638
|
-
.grid3{display:grid;grid-template-columns:1fr 1fr 1fr;gap:1rem}
|
|
2639
|
-
table{width:100%;border-collapse:collapse;font-size:.8rem}
|
|
2640
|
-
th{text-align:left;color:#64748b;font-weight:500;padding:.5rem .75rem;border-bottom:1px solid #2d2d4e;white-space:nowrap}
|
|
2641
|
-
td{padding:.45rem .75rem;border-bottom:1px solid #1a1a2e;vertical-align:top;word-break:break-word}
|
|
2642
|
-
tr.fail td{background:rgba(239,68,68,.04)}
|
|
2643
|
-
tr.flaky td{background:rgba(245,158,11,.03)}
|
|
2644
|
-
.status{display:inline-block;padding:2px 8px;border-radius:4px;font-size:.7rem;font-weight:600}
|
|
2645
|
-
.status-pass{background:#064e3b;color:#34d399}
|
|
2646
|
-
.status-fail{background:#450a0a;color:#f87171}
|
|
2647
|
-
.status-skip{background:#1e293b;color:#94a3b8}
|
|
2648
|
-
.status-flaky{background:#422006;color:#fbbf24}
|
|
2649
|
-
.sev-p0{background:#450a0a;color:#f87171;padding:2px 6px;border-radius:3px;font-size:.7rem;font-weight:700}
|
|
2650
|
-
.sev-p1{background:#422006;color:#fbbf24;padding:2px 6px;border-radius:3px;font-size:.7rem;font-weight:700}
|
|
2651
|
-
.sev-p2{background:#1e3a5f;color:#60a5fa;padding:2px 6px;border-radius:3px;font-size:.7rem}
|
|
2652
|
-
.sev-p3{background:#1e293b;color:#94a3b8;padding:2px 6px;border-radius:3px;font-size:.7rem}
|
|
2653
|
-
.err code{font-size:.7rem;color:#f87171;background:#1a0a0a;padding:2px 6px;border-radius:3px}
|
|
2654
|
-
.bug-card{border-radius:8px;padding:.9rem 1rem;margin-bottom:.6rem;border-left:3px solid}
|
|
2655
|
-
.bug-p0{background:rgba(239,68,68,.08);border-color:#ef4444}
|
|
2656
|
-
.bug-p1{background:rgba(245,158,11,.07);border-color:#f59e0b}
|
|
2657
|
-
.bug-p2{background:rgba(96,165,250,.07);border-color:#60a5fa}
|
|
2658
|
-
.bug-p3{background:rgba(148,163,184,.06);border-color:#64748b}
|
|
2659
|
-
.bug-header{display:flex;gap:.6rem;align-items:center;margin-bottom:.4rem;flex-wrap:wrap}
|
|
2660
|
-
.bug-id{font-family:monospace;font-size:.72rem;color:#475569}
|
|
2661
|
-
.bug-sev{font-size:.7rem;font-weight:700;color:#f87171}
|
|
2662
|
-
.bug-type{font-size:.68rem;color:#64748b}
|
|
2663
|
-
.bug-st{font-size:.68rem;padding:2px 6px;border-radius:4px;background:#1e293b;color:#94a3b8}
|
|
2664
|
-
.bug-title{font-weight:600;margin-bottom:.2rem;font-size:.88rem}
|
|
2665
|
-
.bug-desc{font-size:.78rem;color:#94a3b8}
|
|
2666
|
-
.chart-wrap{position:relative;height:280px}
|
|
2667
|
-
.summary-bar{display:flex;gap:.5rem;align-items:center;margin:.5rem 0}
|
|
2668
|
-
footer{text-align:center;color:#334155;font-size:.7rem;padding:2rem;border-top:1px solid #1e293b;margin-top:2rem}
|
|
2669
|
-
@media(max-width:768px){.grid2,.grid3{grid-template-columns:1fr}}
|
|
2670
|
-
</style>
|
|
2671
|
-
</head>
|
|
2672
|
-
<body>
|
|
2673
|
-
<header>
|
|
2674
|
-
<div>
|
|
2675
|
-
<h1>🧪 Backlist MAXIMUM QA Report</h1>
|
|
2676
|
-
<p>Run: ${id} · ${new Date(startedAt).toLocaleString()} · ${formatDuration(duration)} · ${results.length} tests · ${bugReports.length} bugs</p>
|
|
2677
|
-
</div>
|
|
2678
|
-
<span class="badge">v${VERSION} MAXIMUM</span>
|
|
2679
|
-
</header>
|
|
2680
|
-
<div class="container">
|
|
2681
|
-
|
|
2682
|
-
${urlCards ? `<div class="section"><div class="section-title">Target URLs</div>${urlCards}</div>` : ''}
|
|
2683
|
-
|
|
2684
|
-
<div class="metrics">
|
|
2685
|
-
<div class="mc"><div class="ml">Pass Rate</div><div class="mv" style="color:${statusColor}">${passRate}%</div></div>
|
|
2686
|
-
<div class="mc"><div class="ml">Total Tests</div><div class="mv">${summary.total}</div></div>
|
|
2687
|
-
<div class="mc"><div class="ml">Passed</div><div class="mv" style="color:#34d399">${summary.passed}</div></div>
|
|
2688
|
-
<div class="mc"><div class="ml">Failed</div><div class="mv" style="color:#f87171">${summary.failed}</div></div>
|
|
2689
|
-
<div class="mc"><div class="ml">Flaky</div><div class="mv" style="color:#fbbf24">${summary.flaky}</div></div>
|
|
2690
|
-
<div class="mc"><div class="ml">Total Bugs</div><div class="mv" style="color:#c084fc">${bugReports.length}</div></div>
|
|
2691
|
-
<div class="mc"><div class="ml">P0 Critical</div><div class="mv" style="color:#ef4444">${sevCounts.P0}</div></div>
|
|
2692
|
-
<div class="mc"><div class="ml">P1 High</div><div class="mv" style="color:#f59e0b">${sevCounts.P1}</div></div>
|
|
2693
|
-
<div class="mc"><div class="ml">P2 Medium</div><div class="mv" style="color:#60a5fa">${sevCounts.P2}</div></div>
|
|
2694
|
-
<div class="mc"><div class="ml">P3 Low</div><div class="mv" style="color:#94a3b8">${sevCounts.P3}</div></div>
|
|
2695
|
-
<div class="mc"><div class="ml">Categories</div><div class="mv">${Object.keys(coverage).length}</div></div>
|
|
2696
|
-
<div class="mc"><div class="ml">Duration</div><div class="mv" style="font-size:1.3rem">${formatDuration(duration)}</div></div>
|
|
2697
|
-
</div>
|
|
2698
|
-
|
|
2699
|
-
<div class="grid2">
|
|
2700
|
-
<div class="section">
|
|
2701
|
-
<div class="section-title">Coverage by Category <span style="font-size:.75rem;color:#64748b">${Object.keys(coverage).length} types</span></div>
|
|
2702
|
-
${covBars}
|
|
2703
|
-
</div>
|
|
2704
|
-
<div class="section">
|
|
2705
|
-
<div class="section-title">Pass / Fail / Flaky by Type</div>
|
|
2706
|
-
<div class="chart-wrap"><canvas id="typeChart"></canvas></div>
|
|
2707
|
-
</div>
|
|
2708
|
-
</div>
|
|
2709
|
-
|
|
2710
|
-
<div class="section">
|
|
2711
|
-
<div class="section-title">
|
|
2712
|
-
Bug Reports
|
|
2713
|
-
<span style="font-size:.78rem;color:#64748b">${bugReports.length} bugs total — unlimited — sorted by severity</span>
|
|
2714
|
-
</div>
|
|
2715
|
-
${bugCards}
|
|
2716
|
-
</div>
|
|
2717
|
-
|
|
2718
|
-
<div class="section">
|
|
2719
|
-
<div class="section-title">
|
|
2720
|
-
Route Scan
|
|
2721
|
-
<span style="font-size:.78rem;color:#64748b">${routeScans.length} routes probed</span>
|
|
2722
|
-
</div>
|
|
2723
|
-
${routeCards}
|
|
2724
|
-
</div>
|
|
2725
|
-
|
|
2726
|
-
<div class="section">
|
|
2727
|
-
<div class="section-title">
|
|
2728
|
-
All Test Results
|
|
2729
|
-
<span style="font-size:.78rem;color:#64748b">${results.length} tests</span>
|
|
2730
|
-
</div>
|
|
2731
|
-
<table>
|
|
2732
|
-
<thead>
|
|
2733
|
-
<tr>
|
|
2734
|
-
<th>Test Name</th><th>Type</th><th>Status</th>
|
|
2735
|
-
<th>Sev</th><th>Duration</th><th>Retries</th><th>Error</th>
|
|
2736
|
-
</tr>
|
|
2737
|
-
</thead>
|
|
2738
|
-
<tbody>${rows}</tbody>
|
|
2739
|
-
</table>
|
|
2740
|
-
</div>
|
|
2741
|
-
</div>
|
|
2742
|
-
|
|
2743
|
-
<footer>
|
|
2744
|
-
Backlist MAXIMUM QA v${VERSION} — ${results.length} tests · ${bugReports.length} bugs · Generated ${new Date().toLocaleString()}
|
|
2745
|
-
</footer>
|
|
2746
|
-
|
|
2747
|
-
<script>
|
|
2748
|
-
new Chart(document.getElementById('typeChart'), {
|
|
2749
|
-
type: 'bar',
|
|
2750
|
-
data: {
|
|
2751
|
-
labels: ${chartLabels},
|
|
2752
|
-
datasets: [
|
|
2753
|
-
{ label: 'Passed', data: ${chartPassed}, backgroundColor: '#34d399', borderRadius: 3 },
|
|
2754
|
-
{ label: 'Failed', data: ${chartFailed}, backgroundColor: '#f87171', borderRadius: 3 },
|
|
2755
|
-
{ label: 'Flaky', data: ${chartFlaky}, backgroundColor: '#fbbf24', borderRadius: 3 },
|
|
2756
|
-
],
|
|
2757
|
-
},
|
|
2758
|
-
options: {
|
|
2759
|
-
responsive: true,
|
|
2760
|
-
maintainAspectRatio: false,
|
|
2761
|
-
plugins: {
|
|
2762
|
-
legend: { labels: { color: '#94a3b8', font: { size: 11 } } },
|
|
2763
|
-
},
|
|
2764
|
-
scales: {
|
|
2765
|
-
x: { stacked: true, ticks: { color: '#64748b', font: { size: 10 } }, grid: { color: '#1e293b' } },
|
|
2766
|
-
y: { stacked: true, ticks: { color: '#64748b', stepSize: 1, font: { size: 10 } }, grid: { color: '#1e293b' } },
|
|
2767
|
-
},
|
|
2768
|
-
},
|
|
2769
|
-
});
|
|
2770
|
-
</script>
|
|
2771
|
-
</body>
|
|
2772
|
-
</html>`;
|
|
2773
|
-
}
|
|
2774
686
|
|
|
2775
|
-
// ── History helpers ────────────────────────────────────────────────────────
|
|
2776
687
|
export async function initQASystem() {
|
|
2777
688
|
await fs.ensureDir(QA_DIR);
|
|
2778
689
|
await fs.ensureDir(REPORT_DIR);
|
|
690
|
+
await fs.ensureDir(SCREENSHOT_DIR);
|
|
2779
691
|
if (!await fs.pathExists(HISTORY_FILE)) {
|
|
2780
|
-
await fs.writeJson(HISTORY_FILE, { runs: [] }, { spaces: 2 });
|
|
692
|
+
await fs.writeJson(HISTORY_FILE, { runs: [], version: VERSION }, { spaces: 2 });
|
|
2781
693
|
}
|
|
2782
694
|
}
|
|
2783
695
|
|
|
2784
|
-
async function
|
|
2785
|
-
|
|
2786
|
-
|
|
696
|
+
export async function saveSession(session) {
|
|
697
|
+
const history = await loadHistory();
|
|
698
|
+
const summary = session.getSummary();
|
|
699
|
+
history.runs.unshift({
|
|
700
|
+
id : session.id,
|
|
701
|
+
startedAt: session.startedAt,
|
|
702
|
+
urls : session.urls,
|
|
703
|
+
summary,
|
|
704
|
+
version : VERSION,
|
|
705
|
+
bugCount : session.bugs.length,
|
|
706
|
+
screenshotCount: session.screenshots.length,
|
|
707
|
+
});
|
|
708
|
+
if (history.runs.length > 100) history.runs = history.runs.slice(0, 100);
|
|
709
|
+
await fs.writeJson(HISTORY_FILE, history, { spaces: 2 });
|
|
2787
710
|
}
|
|
2788
711
|
|
|
2789
|
-
async function
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
if (hist.runs.length > 100) hist.runs = hist.runs.slice(0, 100); // keep more history
|
|
2793
|
-
await fs.writeJson(HISTORY_FILE, hist, { spaces: 2 });
|
|
712
|
+
export async function loadHistory() {
|
|
713
|
+
try { return await fs.readJson(HISTORY_FILE); }
|
|
714
|
+
catch { return { runs: [], version: VERSION }; }
|
|
2794
715
|
}
|
|
2795
716
|
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
console.error(chalk.gray(` [warn] Report write failed: ${err.message}`));
|
|
717
|
+
// ── URL QA entry point ────────────────────────────────────────────────────
|
|
718
|
+
export async function runUrlQA({ localUrl, stagingUrl, prodUrl, options = {} } = {}) {
|
|
719
|
+
const urls = {};
|
|
720
|
+
if (localUrl) urls.localhost = localUrl;
|
|
721
|
+
if (stagingUrl) urls.staging = stagingUrl;
|
|
722
|
+
if (prodUrl) urls.production = prodUrl;
|
|
723
|
+
|
|
724
|
+
if (Object.keys(urls).length === 0) {
|
|
725
|
+
console.log(chalk.red(' No URLs provided.'));
|
|
2806
726
|
return null;
|
|
2807
727
|
}
|
|
728
|
+
|
|
729
|
+
const session = new QASession(urls);
|
|
730
|
+
const engine = new QAEngine(session, options);
|
|
731
|
+
|
|
732
|
+
await engine.init();
|
|
733
|
+
await engine.run();
|
|
734
|
+
await saveSession(session);
|
|
735
|
+
|
|
736
|
+
const htmlReporter = new HTMLReporter(session);
|
|
737
|
+
const jsonReporter = new JSONReporter(session);
|
|
738
|
+
|
|
739
|
+
const htmlPath = await htmlReporter.generate(REPORT_DIR);
|
|
740
|
+
const jsonPath = await jsonReporter.generate(REPORT_DIR);
|
|
741
|
+
|
|
742
|
+
return { session, htmlPath, jsonPath };
|
|
2808
743
|
}
|
|
2809
744
|
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
const
|
|
2814
|
-
if (
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
745
|
+
// ── Automated QA entry point ──────────────────────────────────────────────
|
|
746
|
+
export async function runAutomatedQA({ continuous = false, localUrl, prodUrl, stagingUrl } = {}) {
|
|
747
|
+
const runOnce = async () => {
|
|
748
|
+
const urls = {};
|
|
749
|
+
if (localUrl) urls.localhost = localUrl;
|
|
750
|
+
if (stagingUrl) urls.staging = stagingUrl;
|
|
751
|
+
if (prodUrl) urls.production = prodUrl;
|
|
752
|
+
|
|
753
|
+
if (Object.keys(urls).length === 0) {
|
|
754
|
+
console.log(chalk.yellow(' No URLs configured. Skipping URL-based tests.'));
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
const session = new QASession(urls);
|
|
758
|
+
const engine = new QAEngine(session);
|
|
759
|
+
|
|
760
|
+
await engine.init();
|
|
761
|
+
await engine.run();
|
|
762
|
+
await saveSession(session);
|
|
763
|
+
|
|
764
|
+
const htmlPath = await new HTMLReporter(session).generate(REPORT_DIR);
|
|
765
|
+
const jsonPath = await new JSONReporter(session).generate(REPORT_DIR);
|
|
766
|
+
|
|
767
|
+
const summary = session.getSummary();
|
|
768
|
+
console.log(chalk.hex('#00F5FF').bold(
|
|
769
|
+
`\n ✓ Run ${session.id} — ${summary.total} tests · ${summary.failed} failed · ` +
|
|
770
|
+
`${session.bugs.length} bugs · ${formatDuration(summary.duration)}`
|
|
771
|
+
));
|
|
772
|
+
if (htmlPath) console.log(chalk.gray(` 📄 Report: ${htmlPath}`));
|
|
773
|
+
return session;
|
|
774
|
+
};
|
|
775
|
+
|
|
776
|
+
if (!continuous) return runOnce();
|
|
777
|
+
|
|
778
|
+
console.log(chalk.cyan(' ⚡ Continuous mode — re-runs every 60s. Ctrl+C to stop.\n'));
|
|
779
|
+
let i = 0;
|
|
780
|
+
while (true) {
|
|
781
|
+
console.log(chalk.gray(`\n ── Run #${++i} ── ${new Date().toLocaleTimeString()}`));
|
|
782
|
+
await runOnce();
|
|
783
|
+
await sleep(60_000);
|
|
784
|
+
}
|
|
2823
785
|
}
|
|
2824
786
|
|
|
2825
|
-
//
|
|
2826
|
-
// Manual QA Flow
|
|
2827
|
-
// ─────────────────────────────────────────────────────────────────────────
|
|
787
|
+
// ── Manual QA ─────────────────────────────────────────────────────────────
|
|
2828
788
|
export async function runManualQA() {
|
|
2829
|
-
const runId = `MQA-${shortId()}`;
|
|
2830
|
-
const startedAt = timestamp();
|
|
2831
|
-
const runner = new TestRunner();
|
|
2832
|
-
const bugs = [];
|
|
2833
|
-
const manualResults = [];
|
|
2834
|
-
|
|
2835
789
|
console.log('');
|
|
790
|
+
|
|
2836
791
|
const action = await p.select({
|
|
2837
|
-
message: 'Manual QA —
|
|
792
|
+
message: 'Manual QA — what to run?',
|
|
2838
793
|
options: [
|
|
2839
|
-
{ value: 'url
|
|
2840
|
-
{ value: '
|
|
2841
|
-
{ value: '
|
|
2842
|
-
{ value: '
|
|
2843
|
-
{ value: '
|
|
2844
|
-
{ value: '
|
|
2845
|
-
{ value: 'ui-tests', label: '🖥️ UI/Frontend Tests', hint: '12 frontend checks' },
|
|
2846
|
-
{ value: 'new-test', label: '✏️ Custom Test', hint: 'Create and run your own' },
|
|
2847
|
-
{ value: 'log-bug', label: '🐛 Log Bug Report', hint: 'Unlimited bug logging' },
|
|
794
|
+
{ value: 'full-url', label: '🌐 Full URL-Based Real Scan', hint: 'Browser + API + Security + Perf + SEO + A11y' },
|
|
795
|
+
{ value: 'security', label: '🛡️ Security Only', hint: 'Real HTTP security header + vuln scan' },
|
|
796
|
+
{ value: 'perf', label: '⚡ Performance Only', hint: 'Real Core Web Vitals measurement' },
|
|
797
|
+
{ value: 'a11y', label: '♿ Accessibility Only', hint: 'Real axe-core WCAG scan' },
|
|
798
|
+
{ value: 'seo', label: '🔎 SEO Only', hint: 'Real meta, og, robots, sitemap scan' },
|
|
799
|
+
{ value: 'api', label: '📡 API Only', hint: 'Real endpoint probe + contract validation' },
|
|
2848
800
|
],
|
|
2849
801
|
});
|
|
2850
802
|
if (p.isCancel(action)) { p.cancel('Cancelled.'); return; }
|
|
2851
803
|
|
|
2852
|
-
const
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
if (p.isCancel(localUrl)) { p.cancel('Cancelled.'); return; }
|
|
2858
|
-
const run = await runUrlQA({
|
|
2859
|
-
localUrl: String(localUrl).trim() || undefined,
|
|
2860
|
-
prodUrl : String(prodUrl).trim() || undefined,
|
|
2861
|
-
});
|
|
2862
|
-
if (run) manualResults.push(...run.results);
|
|
2863
|
-
|
|
2864
|
-
} else if (action === 'log-bug') {
|
|
2865
|
-
await logBugInteractive(bugs);
|
|
2866
|
-
|
|
2867
|
-
} else if (action === 'new-test') {
|
|
2868
|
-
await createAndRunTestInteractive(runner, manualResults, dashboard);
|
|
2869
|
-
|
|
2870
|
-
} else if (action === 'full-scan') {
|
|
2871
|
-
dashboard.start();
|
|
2872
|
-
const results = await runner.run([...buildFullSystemTests(), ...buildUITests()], dashboard);
|
|
2873
|
-
manualResults.push(...results);
|
|
2874
|
-
dashboard.stop();
|
|
2875
|
-
printResultsSummary(results);
|
|
2876
|
-
|
|
2877
|
-
} else if (action === 'ui-tests') {
|
|
2878
|
-
dashboard.start();
|
|
2879
|
-
const results = await runner.run(buildUITests(), dashboard);
|
|
2880
|
-
manualResults.push(...results);
|
|
2881
|
-
dashboard.stop();
|
|
2882
|
-
printResultsSummary(results);
|
|
2883
|
-
|
|
2884
|
-
} else if (action === 'security-scan') {
|
|
2885
|
-
const all = buildFullSystemTests();
|
|
2886
|
-
const sec = all.filter(t => ['security','auth','cors','rate-limit','environment'].includes(t.type));
|
|
2887
|
-
dashboard.start();
|
|
2888
|
-
const results = await runner.run(sec, dashboard);
|
|
2889
|
-
manualResults.push(...results);
|
|
2890
|
-
dashboard.stop();
|
|
2891
|
-
printResultsSummary(results);
|
|
2892
|
-
|
|
2893
|
-
} else if (action === 'db-scan') {
|
|
2894
|
-
const all = buildFullSystemTests();
|
|
2895
|
-
const db = all.filter(t => t.type === 'database');
|
|
2896
|
-
dashboard.start();
|
|
2897
|
-
const results = await runner.run(db, dashboard);
|
|
2898
|
-
manualResults.push(...results);
|
|
2899
|
-
dashboard.stop();
|
|
2900
|
-
printResultsSummary(results);
|
|
2901
|
-
|
|
2902
|
-
} else if (action === 'docker-scan') {
|
|
2903
|
-
const all = buildFullSystemTests();
|
|
2904
|
-
const docker = all.filter(t => ['docker','ci'].includes(t.type));
|
|
2905
|
-
dashboard.start();
|
|
2906
|
-
const results = await runner.run(docker, dashboard);
|
|
2907
|
-
manualResults.push(...results);
|
|
2908
|
-
dashboard.stop();
|
|
2909
|
-
printResultsSummary(results);
|
|
2910
|
-
|
|
2911
|
-
} else if (action === 'code-quality') {
|
|
2912
|
-
const all = buildFullSystemTests();
|
|
2913
|
-
const cq = all.filter(t => ['code-quality','typescript','logging','error-handling'].includes(t.type));
|
|
2914
|
-
dashboard.start();
|
|
2915
|
-
const results = await runner.run(cq, dashboard);
|
|
2916
|
-
manualResults.push(...results);
|
|
2917
|
-
dashboard.stop();
|
|
2918
|
-
printResultsSummary(results);
|
|
2919
|
-
}
|
|
804
|
+
const localUrl = await p.text({
|
|
805
|
+
message : 'Localhost URL:',
|
|
806
|
+
placeholder: 'http://localhost:3000',
|
|
807
|
+
});
|
|
808
|
+
if (p.isCancel(localUrl)) { p.cancel('Cancelled.'); return; }
|
|
2920
809
|
|
|
2921
|
-
const
|
|
2922
|
-
|
|
810
|
+
const prodUrl = await p.text({
|
|
811
|
+
message : 'Production URL (blank to skip):',
|
|
812
|
+
placeholder: 'https://yoursite.com',
|
|
813
|
+
});
|
|
2923
814
|
|
|
2924
|
-
const
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
const run = {
|
|
2928
|
-
id: runId, type: 'manual', version: VERSION, startedAt, duration,
|
|
2929
|
-
results: manualResults, bugReports: bugs, summary, coverage,
|
|
815
|
+
const urls = {
|
|
816
|
+
localhost : String(localUrl).trim() || undefined,
|
|
817
|
+
production: !p.isCancel(prodUrl) ? String(prodUrl).trim() || undefined : undefined,
|
|
2930
818
|
};
|
|
2931
|
-
await saveRun(run);
|
|
2932
|
-
const reportFile = await exportReport(run);
|
|
2933
819
|
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
820
|
+
const session = new QASession(urls);
|
|
821
|
+
const engine = new QAEngine(session);
|
|
822
|
+
await engine.init();
|
|
2937
823
|
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
if (p.isCancel(title)) return;
|
|
2941
|
-
const severity = await p.select({
|
|
2942
|
-
message: 'Severity:',
|
|
2943
|
-
options: Object.entries(SEVERITY_LEVELS).map(([k, v]) => ({ value: k, label: `${k} — ${v}` })),
|
|
2944
|
-
});
|
|
2945
|
-
if (p.isCancel(severity)) return;
|
|
2946
|
-
const type = await p.select({
|
|
2947
|
-
message: 'Category:',
|
|
2948
|
-
options: TEST_TYPES.map(t => ({ value: t, label: t })),
|
|
2949
|
-
});
|
|
2950
|
-
if (p.isCancel(type)) return;
|
|
2951
|
-
const description = await p.text({ message: 'Description:', placeholder: 'Steps to reproduce…' });
|
|
2952
|
-
bugs.push({
|
|
2953
|
-
id: `BUG-${shortId()}`, title: String(title),
|
|
2954
|
-
severity: String(severity), type: String(type), status: 'OPEN',
|
|
2955
|
-
description: p.isCancel(description) ? '' : String(description),
|
|
2956
|
-
createdAt: timestamp(),
|
|
2957
|
-
});
|
|
2958
|
-
console.log(chalk.green(` ✓ Bug logged: ${colorSeverity(String(severity))} — unlimited bug tracking active`));
|
|
2959
|
-
}
|
|
824
|
+
// Only run selected phases
|
|
825
|
+
await engine.runPhase(action);
|
|
2960
826
|
|
|
2961
|
-
|
|
2962
|
-
const
|
|
2963
|
-
if (
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
message: 'Severity:',
|
|
2968
|
-
options: ['P0','P1','P2','P3'].map(s => ({ value: s, label: s })),
|
|
2969
|
-
});
|
|
2970
|
-
if (p.isCancel(sev)) return;
|
|
2971
|
-
const expectPass = await p.confirm({ message: 'Should this test pass?' });
|
|
2972
|
-
|
|
2973
|
-
dashboard.start();
|
|
2974
|
-
const [result] = await runner.run([{
|
|
2975
|
-
id: shortId(), name: String(name), type: String(type), sev: String(sev),
|
|
2976
|
-
fn: async () => {
|
|
2977
|
-
await sleep(400 + Math.random() * 300);
|
|
2978
|
-
if (!expectPass) throw new Error('Test manually marked as failure');
|
|
2979
|
-
},
|
|
2980
|
-
}], dashboard);
|
|
2981
|
-
results.push(result);
|
|
2982
|
-
dashboard.stop();
|
|
2983
|
-
console.log(` ${colorStatus(result.status)} ${result.name} ${chalk.gray(formatDuration(result.duration))}`);
|
|
827
|
+
await saveSession(session);
|
|
828
|
+
const htmlPath = await new HTMLReporter(session).generate(REPORT_DIR);
|
|
829
|
+
if (htmlPath) {
|
|
830
|
+
p.outro(chalk.hex('#00F5FF').bold('✓ QA complete'));
|
|
831
|
+
console.log(chalk.gray(` 📄 Report: ${htmlPath}`));
|
|
832
|
+
}
|
|
2984
833
|
}
|
|
2985
834
|
|
|
2986
|
-
// ── Post-
|
|
835
|
+
// ── Post-generation validation ────────────────────────────────────────────
|
|
2987
836
|
export async function autoRunPostGeneration(options = {}) {
|
|
2988
837
|
console.log('');
|
|
2989
|
-
console.log(chalk.hex('#00F5FF').bold(` ── 🔬 Post-Generation
|
|
2990
|
-
console.log(chalk.gray(
|
|
838
|
+
console.log(chalk.hex('#00F5FF').bold(` ── 🔬 Post-Generation Real QA v${VERSION} ──`));
|
|
839
|
+
console.log(chalk.gray(' Note: Start your server first, then provide its URL'));
|
|
2991
840
|
console.log('');
|
|
2992
841
|
|
|
2993
|
-
const
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
const autoBugs = [];
|
|
2998
|
-
|
|
2999
|
-
console.log(chalk.gray(` ${tests.length} tests — no limit on bug reports\n`));
|
|
3000
|
-
|
|
3001
|
-
runner.on('result', r => {
|
|
3002
|
-
if (r.status === 'FAIL') {
|
|
3003
|
-
autoBugs.push({
|
|
3004
|
-
id: `POST-${shortId()}`, title: r.name,
|
|
3005
|
-
severity : r.sev || (r.type === 'security' ? 'P0' : r.type === 'database' ? 'P1' : 'P2'),
|
|
3006
|
-
status : 'OPEN', description: r.error || '',
|
|
3007
|
-
type : r.type, createdAt: timestamp(),
|
|
3008
|
-
});
|
|
3009
|
-
}
|
|
842
|
+
const url = await p.text({
|
|
843
|
+
message : 'Server URL to validate:',
|
|
844
|
+
placeholder: 'http://localhost:3000',
|
|
845
|
+
defaultValue: 'http://localhost:3000',
|
|
3010
846
|
});
|
|
847
|
+
if (p.isCancel(url)) { p.cancel('Cancelled.'); return; }
|
|
3011
848
|
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
const summary = buildSummary(results);
|
|
3017
|
-
const coverage = buildCoverageMatrix(results);
|
|
3018
|
-
const run = {
|
|
3019
|
-
id: `POST-${shortId()}`, type: 'post-generation', version: VERSION,
|
|
3020
|
-
startedAt: timestamp(), duration: 0,
|
|
3021
|
-
results, bugReports: autoBugs, summary, coverage,
|
|
3022
|
-
};
|
|
3023
|
-
|
|
3024
|
-
await saveRun(run);
|
|
3025
|
-
const reportFile = await exportReport(run);
|
|
3026
|
-
printResultsSummary(results, autoBugs);
|
|
3027
|
-
|
|
3028
|
-
if (autoBugs.length > 0) {
|
|
3029
|
-
console.log(chalk.red.bold(` ⚠ ${autoBugs.length} issue(s) found:`));
|
|
3030
|
-
autoBugs.forEach(b => console.log(chalk.red(` ${colorSeverity(b.severity)} [${b.type}] ${b.title}`)));
|
|
3031
|
-
console.log('');
|
|
849
|
+
const result = await runUrlQA({ localUrl: String(url).trim() });
|
|
850
|
+
if (result?.htmlPath) {
|
|
851
|
+
console.log(chalk.gray(` 📄 Report: ${result.htmlPath}`));
|
|
3032
852
|
}
|
|
3033
|
-
if (reportFile) console.log(chalk.gray(` 📄 Report: ${reportFile}`));
|
|
3034
853
|
}
|
|
3035
854
|
|
|
3036
|
-
// ──
|
|
855
|
+
// ── View History ──────────────────────────────────────────────────────────
|
|
3037
856
|
export async function viewQAHistory() {
|
|
3038
|
-
const
|
|
3039
|
-
if (!
|
|
857
|
+
const history = await loadHistory();
|
|
858
|
+
if (!history.runs?.length) {
|
|
859
|
+
console.log(chalk.yellow('\n No QA history found.\n'));
|
|
860
|
+
return;
|
|
861
|
+
}
|
|
3040
862
|
|
|
3041
863
|
console.log('');
|
|
3042
|
-
console.log(chalk.hex('#00F5FF').bold(' QA History
|
|
3043
|
-
console.log(chalk.gray('
|
|
864
|
+
console.log(chalk.hex('#00F5FF').bold(' QA History (real runs only)'));
|
|
865
|
+
console.log(chalk.gray(' ──────────────────────────────────────────────────────'));
|
|
866
|
+
|
|
867
|
+
for (const run of history.runs.slice(0, 15)) {
|
|
868
|
+
const rate = run.summary?.passRate ?? '–';
|
|
869
|
+
const color = Number(rate) >= 90 ? chalk.green : Number(rate) >= 70 ? chalk.yellow : chalk.red;
|
|
870
|
+
const bugs = run.bugCount ?? 0;
|
|
871
|
+
const shots = run.screenshotCount ?? 0;
|
|
872
|
+
const urlStr = Object.values(run.urls || {}).filter(Boolean).join(', ');
|
|
3044
873
|
|
|
3045
|
-
for (const run of hist.runs.slice(0, 15)) {
|
|
3046
|
-
const passRate = run.summary.total ? ((run.summary.passed / run.summary.total) * 100).toFixed(0) : '–';
|
|
3047
|
-
const rateColor = Number(passRate) >= 90 ? chalk.green : Number(passRate) >= 70 ? chalk.yellow : chalk.red;
|
|
3048
|
-
const bugs = run.bugReports?.length ?? 0;
|
|
3049
874
|
console.log(
|
|
3050
|
-
` ${chalk.gray(run.id.padEnd(
|
|
3051
|
-
|
|
3052
|
-
|
|
875
|
+
` ${chalk.gray(run.id.padEnd(14))} ` +
|
|
876
|
+
`${chalk.gray(new Date(run.startedAt).toLocaleString().padEnd(24))} ` +
|
|
877
|
+
`${color(String(rate + '%').padStart(6))} ` +
|
|
878
|
+
`${chalk.gray(String(run.summary?.total || 0) + ' tests')} ` +
|
|
879
|
+
`${chalk.cyan(bugs + ' bugs')} ` +
|
|
880
|
+
`${chalk.gray(shots + ' shots')} ` +
|
|
881
|
+
`${chalk.dim(urlStr.slice(0, 40))}`
|
|
3053
882
|
);
|
|
3054
883
|
}
|
|
3055
884
|
console.log('');
|
|
3056
885
|
|
|
3057
886
|
const chosen = await p.select({
|
|
3058
|
-
message: '
|
|
887
|
+
message: 'Open a report?',
|
|
3059
888
|
options: [
|
|
3060
|
-
...
|
|
889
|
+
...history.runs.slice(0, 8).map(r => ({
|
|
3061
890
|
value: r.id,
|
|
3062
|
-
label: `${r.id} — ${new Date(r.startedAt).toLocaleString()} — ${r.
|
|
891
|
+
label: `${r.id} — ${new Date(r.startedAt).toLocaleString()} — ${r.bugCount} bugs`,
|
|
3063
892
|
})),
|
|
3064
893
|
{ value: '__back', label: '↩ Back' },
|
|
3065
894
|
],
|
|
3066
895
|
});
|
|
3067
896
|
if (p.isCancel(chosen) || chosen === '__back') return;
|
|
3068
897
|
|
|
3069
|
-
const
|
|
3070
|
-
if (
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
if (run.bugReports?.length) {
|
|
3084
|
-
console.log('');
|
|
3085
|
-
console.log(chalk.bold(` All ${run.bugReports.length} Bug Reports:`));
|
|
3086
|
-
for (const b of run.bugReports) {
|
|
3087
|
-
console.log(` ${colorSeverity(b.severity)} [${b.type || ''}] ${b.title} ${chalk.gray(`[${b.status}]`)}`);
|
|
3088
|
-
if (b.description) console.log(chalk.gray(` → ${b.description}`));
|
|
3089
|
-
}
|
|
898
|
+
const reportPath = path.join(REPORT_DIR, `${chosen.toLowerCase()}.html`);
|
|
899
|
+
if (await fs.pathExists(reportPath)) {
|
|
900
|
+
console.log(chalk.green(` 📄 Report: ${reportPath}`));
|
|
901
|
+
// Open in browser if possible
|
|
902
|
+
try {
|
|
903
|
+
const { exec } = await import('node:child_process');
|
|
904
|
+
const cmd = process.platform === 'darwin' ? 'open'
|
|
905
|
+
: process.platform === 'win32' ? 'start'
|
|
906
|
+
: 'xdg-open';
|
|
907
|
+
exec(`${cmd} "${reportPath}"`);
|
|
908
|
+
} catch {}
|
|
909
|
+
} else {
|
|
910
|
+
console.log(chalk.yellow(' Report file not found — may have been deleted.'));
|
|
3090
911
|
}
|
|
3091
|
-
console.log('');
|
|
3092
912
|
}
|