create-backlist 9.0.0 → 10.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/index.js +355 -170
- package/bin/qa.js +112 -53
- package/package.json +1 -1
- package/src/analyzer.js +39 -0
- package/src/qa/qa-engine.js +664 -401
package/src/qa/qa-engine.js
CHANGED
|
@@ -1,21 +1,24 @@
|
|
|
1
1
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
2
|
-
// Backlist QA Engine — qa-engine.js
|
|
3
|
-
// Full live QA runtime: manual + automated +
|
|
2
|
+
// Backlist QA Engine — qa-engine.js v10.0
|
|
3
|
+
// Full live QA runtime: manual + automated + URL-based browser QA
|
|
4
4
|
// Copyright (c) W.A.H.ISHAN — MIT License
|
|
5
5
|
//
|
|
6
|
-
// NEW in
|
|
7
|
-
// ✦
|
|
8
|
-
// ✦
|
|
9
|
-
// ✦
|
|
10
|
-
// ✦
|
|
11
|
-
// ✦
|
|
12
|
-
// ✦
|
|
13
|
-
// ✦
|
|
14
|
-
// ✦
|
|
15
|
-
// ✦
|
|
16
|
-
// ✦
|
|
17
|
-
// ✦
|
|
18
|
-
// ✦
|
|
6
|
+
// NEW in v10.0:
|
|
7
|
+
// ✦ URL-based QA — test localhost AND production simultaneously
|
|
8
|
+
// ✦ Real HTTP endpoint probing (fetch-based, no browser needed)
|
|
9
|
+
// ✦ Auto route crawler — discovers pages from sitemap / common paths
|
|
10
|
+
// ✦ Response time p50/p95/p99 benchmarking per route
|
|
11
|
+
// ✦ Security header scanner (CSP, CORS, X-Frame, HSTS, etc.)
|
|
12
|
+
// ✦ Auth flow validator (login / protected route / token check)
|
|
13
|
+
// ✦ Broken link detector across crawled pages
|
|
14
|
+
// ✦ SEO meta tag validator per page
|
|
15
|
+
// ✦ Accessibility quick-check (meta viewport, lang attr, alt probes)
|
|
16
|
+
// ✦ Mobile responsiveness header check
|
|
17
|
+
// ✦ Console error simulation via HTML response parsing
|
|
18
|
+
// ✦ Rich HTML report v10 with Chart.js + timeline + per-page cards
|
|
19
|
+
// ✦ Dual-URL diff report (localhost vs production)
|
|
20
|
+
// ✦ JSON CI output with exit-code propagation
|
|
21
|
+
// ✦ All v9.0 features retained
|
|
19
22
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
20
23
|
|
|
21
24
|
import * as p from '@clack/prompts';
|
|
@@ -23,23 +26,46 @@ import chalk from 'chalk';
|
|
|
23
26
|
import fs from 'fs-extra';
|
|
24
27
|
import path from 'node:path';
|
|
25
28
|
import os from 'node:os';
|
|
29
|
+
import http from 'node:http';
|
|
30
|
+
import https from 'node:https';
|
|
31
|
+
import { URL } from 'node:url';
|
|
26
32
|
import { EventEmitter } from 'node:events';
|
|
27
33
|
import { performance } from 'node:perf_hooks';
|
|
28
34
|
|
|
29
35
|
// ── Constants ─────────────────────────────────────────────────────────────
|
|
30
36
|
|
|
37
|
+
const VERSION = '10.0.0';
|
|
31
38
|
const QA_DIR = path.join(process.cwd(), '.backlist', 'qa');
|
|
32
39
|
const HISTORY_FILE = path.join(QA_DIR, 'history.json');
|
|
33
40
|
const REPORT_DIR = path.join(QA_DIR, 'reports');
|
|
34
41
|
|
|
35
42
|
const SEVERITY_LEVELS = { P0: 'Critical', P1: 'High', P2: 'Medium', P3: 'Low' };
|
|
36
|
-
const TEST_TYPES = ['happy-path', 'validation', 'auth', 'edge-case', 'performance', 'security', 'e2e', 'ui'];
|
|
43
|
+
const TEST_TYPES = ['happy-path', 'validation', 'auth', 'edge-case', 'performance', 'security', 'e2e', 'ui', 'seo', 'a11y', 'links', 'http'];
|
|
37
44
|
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
45
|
+
const HTTP_TIMEOUT_MS = 8_000;
|
|
38
46
|
const FLAKY_RETRY_COUNT = 2;
|
|
39
47
|
const WATCH_INTERVAL_MS = 30_000;
|
|
40
48
|
|
|
41
|
-
// ──
|
|
42
|
-
|
|
49
|
+
// ── Common routes to probe ────────────────────────────────────────────────
|
|
50
|
+
const COMMON_ROUTES = [
|
|
51
|
+
'/', '/login', '/register', '/dashboard', '/dashboard/analytics',
|
|
52
|
+
'/dashboard/sales', '/profile', '/settings', '/admin', '/about',
|
|
53
|
+
'/contact', '/api/health', '/api/status', '/api/v1/health',
|
|
54
|
+
'/sitemap.xml', '/robots.txt', '/favicon.ico',
|
|
55
|
+
];
|
|
56
|
+
|
|
57
|
+
// ── Security headers to check ─────────────────────────────────────────────
|
|
58
|
+
const SECURITY_HEADERS = [
|
|
59
|
+
{ header: 'content-security-policy', label: 'CSP', sev: 'P1' },
|
|
60
|
+
{ header: 'x-frame-options', label: 'X-Frame', sev: 'P1' },
|
|
61
|
+
{ header: 'x-content-type-options', label: 'X-Content', sev: 'P2' },
|
|
62
|
+
{ header: 'strict-transport-security', label: 'HSTS', sev: 'P1' },
|
|
63
|
+
{ header: 'referrer-policy', label: 'Referrer', sev: 'P2' },
|
|
64
|
+
{ header: 'permissions-policy', label: 'Permissions', sev: 'P3' },
|
|
65
|
+
{ header: 'access-control-allow-origin', label: 'CORS', sev: 'P2' },
|
|
66
|
+
];
|
|
67
|
+
|
|
68
|
+
// ── ANSI escape helpers ───────────────────────────────────────────────────
|
|
43
69
|
const ESC = '\x1b[';
|
|
44
70
|
const CLEAR_LINE = ESC + '2K\r';
|
|
45
71
|
const CURSOR_UP = (n) => ESC + `${n}A`;
|
|
@@ -49,7 +75,6 @@ const BOLD = chalk.bold;
|
|
|
49
75
|
const DIM = chalk.dim;
|
|
50
76
|
|
|
51
77
|
// ── Utilities ─────────────────────────────────────────────────────────────
|
|
52
|
-
|
|
53
78
|
function timestamp() { return new Date().toISOString(); }
|
|
54
79
|
function shortId() { return Math.random().toString(36).slice(2, 9); }
|
|
55
80
|
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
@@ -82,24 +107,370 @@ function formatDuration(ms) {
|
|
|
82
107
|
}
|
|
83
108
|
|
|
84
109
|
function formatBytes(b) {
|
|
85
|
-
if (b < 1024)
|
|
110
|
+
if (b < 1024) return `${b}B`;
|
|
86
111
|
if (b < 1024 * 1024) return `${(b / 1024).toFixed(1)}KB`;
|
|
87
112
|
return `${(b / 1024 / 1024).toFixed(1)}MB`;
|
|
88
113
|
}
|
|
89
114
|
|
|
90
|
-
// ── System info ────────────────────────────────────────────────────────────
|
|
91
|
-
|
|
92
115
|
function getSystemStats() {
|
|
93
|
-
const mem
|
|
94
|
-
const heapMB
|
|
95
|
-
const rss
|
|
96
|
-
const uptime
|
|
97
|
-
|
|
98
|
-
|
|
116
|
+
const mem = process.memoryUsage();
|
|
117
|
+
const heapMB = (mem.heapUsed / 1024 / 1024).toFixed(1);
|
|
118
|
+
const rss = formatBytes(mem.rss);
|
|
119
|
+
const uptime = process.uptime().toFixed(1);
|
|
120
|
+
return { heapMB, rss, uptime };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
124
|
+
// v10.0: HTTP Probe Engine
|
|
125
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
126
|
+
|
|
127
|
+
export class HttpProbe {
|
|
128
|
+
#baseUrl;
|
|
129
|
+
#agent;
|
|
130
|
+
|
|
131
|
+
constructor(baseUrl) {
|
|
132
|
+
this.#baseUrl = baseUrl.replace(/\/$/, '');
|
|
133
|
+
const isHttps = baseUrl.startsWith('https');
|
|
134
|
+
this.#agent = isHttps
|
|
135
|
+
? new https.Agent({ rejectUnauthorized: false, timeout: HTTP_TIMEOUT_MS })
|
|
136
|
+
: new http.Agent({ timeout: HTTP_TIMEOUT_MS });
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async fetch(route = '/', options = {}) {
|
|
140
|
+
const url = this.#baseUrl + route;
|
|
141
|
+
const t0 = performance.now();
|
|
142
|
+
try {
|
|
143
|
+
const controller = new AbortController();
|
|
144
|
+
const timer = setTimeout(() => controller.abort(), HTTP_TIMEOUT_MS);
|
|
145
|
+
const res = await fetch(url, {
|
|
146
|
+
signal : controller.signal,
|
|
147
|
+
headers: { 'User-Agent': 'Backlist-QA/10.0', ...options.headers },
|
|
148
|
+
method : options.method || 'GET',
|
|
149
|
+
...(options.body ? { body: options.body } : {}),
|
|
150
|
+
});
|
|
151
|
+
clearTimeout(timer);
|
|
152
|
+
const duration = Math.round(performance.now() - t0);
|
|
153
|
+
const headers = {};
|
|
154
|
+
res.headers.forEach((v, k) => { headers[k] = v; });
|
|
155
|
+
const text = options.readBody ? await res.text().catch(() => '') : '';
|
|
156
|
+
return { ok: true, status: res.status, headers, duration, url, text };
|
|
157
|
+
} catch (err) {
|
|
158
|
+
const duration = Math.round(performance.now() - t0);
|
|
159
|
+
return { ok: false, status: 0, headers: {}, duration, url, error: err.message };
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async probeRoutes(routes = COMMON_ROUTES) {
|
|
164
|
+
const results = [];
|
|
165
|
+
for (const route of routes) {
|
|
166
|
+
const r = await this.fetch(route, { readBody: true });
|
|
167
|
+
results.push({ route, ...r });
|
|
168
|
+
}
|
|
169
|
+
return results;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
async checkSecurityHeaders(route = '/') {
|
|
173
|
+
const r = await this.fetch(route);
|
|
174
|
+
if (!r.ok && r.status === 0) return { ok: false, results: [], error: r.error };
|
|
175
|
+
const results = SECURITY_HEADERS.map(({ header, label, sev }) => ({
|
|
176
|
+
header, label, sev,
|
|
177
|
+
present: header in r.headers,
|
|
178
|
+
value : r.headers[header] || null,
|
|
179
|
+
}));
|
|
180
|
+
return { ok: true, results };
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async benchmarkRoute(route = '/', samples = 5) {
|
|
184
|
+
const timings = [];
|
|
185
|
+
for (let i = 0; i < samples; i++) {
|
|
186
|
+
const r = await this.fetch(route);
|
|
187
|
+
if (r.ok || r.status > 0) timings.push(r.duration);
|
|
188
|
+
await sleep(100);
|
|
189
|
+
}
|
|
190
|
+
if (!timings.length) return { p50: 0, p95: 0, p99: 0, avg: 0, samples: 0 };
|
|
191
|
+
timings.sort((a, b) => a - b);
|
|
192
|
+
const p = (pct) => timings[Math.min(Math.floor(timings.length * pct / 100), timings.length - 1)];
|
|
193
|
+
return {
|
|
194
|
+
p50 : p(50),
|
|
195
|
+
p95 : p(95),
|
|
196
|
+
p99 : p(99),
|
|
197
|
+
avg : Math.round(timings.reduce((a, b) => a + b, 0) / timings.length),
|
|
198
|
+
samples: timings.length,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async checkSEO(route = '/') {
|
|
203
|
+
const r = await this.fetch(route, { readBody: true });
|
|
204
|
+
if (!r.ok && r.status === 0) return { ok: false, checks: [] };
|
|
205
|
+
const html = r.text || '';
|
|
206
|
+
const checks = [
|
|
207
|
+
{ name: 'Title tag', pass: /<title[^>]*>[^<]+<\/title>/i.test(html), sev: 'P1' },
|
|
208
|
+
{ name: 'Meta description',pass: /<meta[^>]+name=["']description["'][^>]*>/i.test(html), sev: 'P2' },
|
|
209
|
+
{ name: 'H1 tag', pass: /<h1[^>]*>[^<]+<\/h1>/i.test(html), sev: 'P1' },
|
|
210
|
+
{ name: 'Viewport meta', pass: /<meta[^>]+name=["']viewport["'][^>]*>/i.test(html), sev: 'P1' },
|
|
211
|
+
{ name: 'Lang attribute', pass: /<html[^>]+lang=["'][^"']+["']/i.test(html), sev: 'P2' },
|
|
212
|
+
{ name: 'Canonical link', pass: /<link[^>]+rel=["']canonical["'][^>]*>/i.test(html), sev: 'P2' },
|
|
213
|
+
{ name: 'OG meta tags', pass: /<meta[^>]+property=["']og:/i.test(html), sev: 'P3' },
|
|
214
|
+
];
|
|
215
|
+
return { ok: true, checks, statusCode: r.status };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
get baseUrl() { return this.#baseUrl; }
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
222
|
+
// v10.0: URL-Based QA Test Suite Builder
|
|
223
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
224
|
+
|
|
225
|
+
function buildUrlTestSuite(probe, label = 'target') {
|
|
226
|
+
const tests = [];
|
|
227
|
+
|
|
228
|
+
// ── Connectivity ──────────────────────────────────────────────────────
|
|
229
|
+
tests.push({ id: shortId(), name: `[${label}] Homepage reachable`, type: 'http', sev: 'P0', fn: async () => {
|
|
230
|
+
const r = await probe.fetch('/');
|
|
231
|
+
if (!r.ok && r.status === 0) throw new Error(`Connection failed: ${r.error}`);
|
|
232
|
+
if (r.status >= 500) throw new Error(`Server error: HTTP ${r.status}`);
|
|
233
|
+
}});
|
|
234
|
+
|
|
235
|
+
tests.push({ id: shortId(), name: `[${label}] API health endpoint`, type: 'http', sev: 'P0', fn: async () => {
|
|
236
|
+
const candidates = ['/api/health', '/api/status', '/api/v1/health', '/health'];
|
|
237
|
+
let found = false;
|
|
238
|
+
for (const c of candidates) {
|
|
239
|
+
const r = await probe.fetch(c);
|
|
240
|
+
if (r.status >= 200 && r.status < 400) { found = true; break; }
|
|
241
|
+
}
|
|
242
|
+
if (!found) throw new Error('No reachable API health endpoint found');
|
|
243
|
+
}});
|
|
244
|
+
|
|
245
|
+
tests.push({ id: shortId(), name: `[${label}] 404 handler works`, type: 'http', sev: 'P2', fn: async () => {
|
|
246
|
+
const r = await probe.fetch('/this-page-does-not-exist-qa-test-' + shortId());
|
|
247
|
+
if (r.status !== 404) throw new Error(`Expected 404, got ${r.status}`);
|
|
248
|
+
}});
|
|
249
|
+
|
|
250
|
+
// ── Security Headers ──────────────────────────────────────────────────
|
|
251
|
+
tests.push({ id: shortId(), name: `[${label}] Security headers scan`, type: 'security', sev: 'P1', fn: async () => {
|
|
252
|
+
const scan = await probe.checkSecurityHeaders('/');
|
|
253
|
+
if (!scan.ok) throw new Error(`Could not reach server: ${scan.error}`);
|
|
254
|
+
const critical = scan.results.filter(r => !r.present && (r.sev === 'P0' || r.sev === 'P1'));
|
|
255
|
+
if (critical.length > 0) {
|
|
256
|
+
throw new Error(`Missing critical headers: ${critical.map(r => r.label).join(', ')}`);
|
|
257
|
+
}
|
|
258
|
+
}});
|
|
259
|
+
|
|
260
|
+
tests.push({ id: shortId(), name: `[${label}] HTTPS redirect check`, type: 'security', sev: 'P1', fn: async () => {
|
|
261
|
+
const r = await probe.fetch('/');
|
|
262
|
+
if (probe.baseUrl.startsWith('http://') && r.status === 200) {
|
|
263
|
+
throw new Error('HTTP serving without redirect — HTTPS not enforced');
|
|
264
|
+
}
|
|
265
|
+
}});
|
|
266
|
+
|
|
267
|
+
// ── Authentication ────────────────────────────────────────────────────
|
|
268
|
+
tests.push({ id: shortId(), name: `[${label}] Login page accessible`, type: 'auth', sev: 'P1', fn: async () => {
|
|
269
|
+
const candidates = ['/login', '/auth/login', '/signin', '/auth', '/user/login'];
|
|
270
|
+
let found = false;
|
|
271
|
+
for (const c of candidates) {
|
|
272
|
+
const r = await probe.fetch(c);
|
|
273
|
+
if (r.status >= 200 && r.status < 400) { found = true; break; }
|
|
274
|
+
}
|
|
275
|
+
if (!found) throw new Error('No login page found at common paths');
|
|
276
|
+
}});
|
|
277
|
+
|
|
278
|
+
tests.push({ id: shortId(), name: `[${label}] Protected route redirects`, type: 'auth', sev: 'P0', fn: async () => {
|
|
279
|
+
const candidates = ['/dashboard', '/admin', '/profile', '/settings'];
|
|
280
|
+
for (const c of candidates) {
|
|
281
|
+
const r = await probe.fetch(c);
|
|
282
|
+
if (r.status === 200) {
|
|
283
|
+
const html = r.text || '';
|
|
284
|
+
const hasAuthBlock = /login|signin|unauthorized|forbidden|redirect/i.test(html);
|
|
285
|
+
if (!hasAuthBlock) throw new Error(`${c} appears accessible without auth (HTTP ${r.status})`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}});
|
|
289
|
+
|
|
290
|
+
// ── Performance ───────────────────────────────────────────────────────
|
|
291
|
+
tests.push({ id: shortId(), name: `[${label}] Homepage response time < 3s`, type: 'performance', sev: 'P1', fn: async () => {
|
|
292
|
+
const bench = await probe.benchmarkRoute('/', 3);
|
|
293
|
+
if (bench.avg > 3000) throw new Error(`Avg response ${bench.avg}ms exceeds 3000ms threshold`);
|
|
294
|
+
}});
|
|
295
|
+
|
|
296
|
+
tests.push({ id: shortId(), name: `[${label}] API latency p95 < 1s`, type: 'performance', sev: 'P2', fn: async () => {
|
|
297
|
+
const candidates = ['/api/health', '/api/status', '/api/v1/health'];
|
|
298
|
+
for (const c of candidates) {
|
|
299
|
+
const r = await probe.fetch(c);
|
|
300
|
+
if (r.status > 0) {
|
|
301
|
+
const bench = await probe.benchmarkRoute(c, 3);
|
|
302
|
+
if (bench.p95 > 1000) throw new Error(`p95 latency ${bench.p95}ms on ${c} exceeds 1000ms`);
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}});
|
|
307
|
+
|
|
308
|
+
// ── SEO ───────────────────────────────────────────────────────────────
|
|
309
|
+
tests.push({ id: shortId(), name: `[${label}] Homepage SEO tags`, type: 'seo', sev: 'P1', fn: async () => {
|
|
310
|
+
const seo = await probe.checkSEO('/');
|
|
311
|
+
if (!seo.ok) throw new Error('Could not fetch homepage');
|
|
312
|
+
const failing = seo.checks.filter(c => !c.pass && c.sev === 'P1');
|
|
313
|
+
if (failing.length > 0) throw new Error(`Missing SEO tags: ${failing.map(c => c.name).join(', ')}`);
|
|
314
|
+
}});
|
|
315
|
+
|
|
316
|
+
tests.push({ id: shortId(), name: `[${label}] robots.txt accessible`, type: 'seo', sev: 'P2', fn: async () => {
|
|
317
|
+
const r = await probe.fetch('/robots.txt');
|
|
318
|
+
if (r.status !== 200) throw new Error(`robots.txt returned ${r.status}`);
|
|
319
|
+
}});
|
|
320
|
+
|
|
321
|
+
// ── Accessibility ─────────────────────────────────────────────────────
|
|
322
|
+
tests.push({ id: shortId(), name: `[${label}] Viewport meta tag`, type: 'a11y', sev: 'P1', fn: async () => {
|
|
323
|
+
const r = await probe.fetch('/', { readBody: true });
|
|
324
|
+
if (!r.ok && r.status === 0) throw new Error('Could not fetch homepage');
|
|
325
|
+
if (!/<meta[^>]+name=["']viewport["']/i.test(r.text || '')) {
|
|
326
|
+
throw new Error('Missing viewport meta tag — mobile responsiveness broken');
|
|
327
|
+
}
|
|
328
|
+
}});
|
|
329
|
+
|
|
330
|
+
tests.push({ id: shortId(), name: `[${label}] HTML lang attribute`, type: 'a11y', sev: 'P2', fn: async () => {
|
|
331
|
+
const r = await probe.fetch('/', { readBody: true });
|
|
332
|
+
if (!r.ok && r.status === 0) throw new Error('Could not fetch homepage');
|
|
333
|
+
if (!/<html[^>]+lang=["'][^"']+["']/i.test(r.text || '')) {
|
|
334
|
+
throw new Error('Missing lang attribute on <html> — screen reader accessibility issue');
|
|
335
|
+
}
|
|
336
|
+
}});
|
|
337
|
+
|
|
338
|
+
// ── Common routes ─────────────────────────────────────────────────────
|
|
339
|
+
tests.push({ id: shortId(), name: `[${label}] Core routes return non-500`, type: 'e2e', sev: 'P1', fn: async () => {
|
|
340
|
+
const routes = ['/', '/login', '/about', '/contact'];
|
|
341
|
+
const errors = [];
|
|
342
|
+
for (const route of routes) {
|
|
343
|
+
const r = await probe.fetch(route);
|
|
344
|
+
if (r.status >= 500) errors.push(`${route} → ${r.status}`);
|
|
345
|
+
}
|
|
346
|
+
if (errors.length > 0) throw new Error(`Server errors: ${errors.join(', ')}`);
|
|
347
|
+
}});
|
|
348
|
+
|
|
349
|
+
tests.push({ id: shortId(), name: `[${label}] sitemap.xml or sitemap`, type: 'seo', sev: 'P3', fn: async () => {
|
|
350
|
+
const r = await probe.fetch('/sitemap.xml');
|
|
351
|
+
if (r.status !== 200) throw new Error(`sitemap.xml returned ${r.status}`);
|
|
352
|
+
}});
|
|
353
|
+
|
|
354
|
+
// ── Content-Type ──────────────────────────────────────────────────────
|
|
355
|
+
tests.push({ id: shortId(), name: `[${label}] HTML content-type correct`, type: 'http', sev: 'P2', fn: async () => {
|
|
356
|
+
const r = await probe.fetch('/');
|
|
357
|
+
if (!r.ok && r.status === 0) throw new Error('Connection failed');
|
|
358
|
+
const ct = r.headers['content-type'] || '';
|
|
359
|
+
if (!ct.includes('text/html')) throw new Error(`Expected text/html, got: ${ct}`);
|
|
360
|
+
}});
|
|
361
|
+
|
|
362
|
+
tests.push({ id: shortId(), name: `[${label}] API returns JSON`, type: 'http', sev: 'P2', fn: async () => {
|
|
363
|
+
const candidates = ['/api/health', '/api/status', '/api/v1/health'];
|
|
364
|
+
for (const c of candidates) {
|
|
365
|
+
const r = await probe.fetch(c);
|
|
366
|
+
if (r.status > 0 && r.status < 500) {
|
|
367
|
+
const ct = r.headers['content-type'] || '';
|
|
368
|
+
if (!ct.includes('json')) throw new Error(`${c} does not return JSON (${ct})`);
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}});
|
|
373
|
+
|
|
374
|
+
return tests;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
378
|
+
// v10.0: Run URL-Based QA
|
|
379
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
380
|
+
|
|
381
|
+
export async function runUrlQA({ localUrl, prodUrl, silent = false } = {}) {
|
|
382
|
+
const runId = `UQA-${shortId()}`;
|
|
383
|
+
const startedAt = timestamp();
|
|
384
|
+
const allTests = [];
|
|
385
|
+
const probes = [];
|
|
386
|
+
|
|
387
|
+
if (!localUrl && !prodUrl) {
|
|
388
|
+
console.log(chalk.red(' No URLs provided. Use --url=http://localhost:3000 or pass { localUrl, prodUrl }.'));
|
|
389
|
+
return null;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
if (!silent) {
|
|
393
|
+
console.log('');
|
|
394
|
+
console.log(chalk.hex('#00F5FF').bold(' ── 🌐 URL-Based QA Scan v10.0 ─────────────────────────'));
|
|
395
|
+
console.log('');
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
if (localUrl) {
|
|
399
|
+
const probe = new HttpProbe(localUrl);
|
|
400
|
+
probes.push({ probe, label: 'localhost', url: localUrl });
|
|
401
|
+
if (!silent) console.log(chalk.gray(` → Probing localhost: ${localUrl}`));
|
|
402
|
+
}
|
|
403
|
+
if (prodUrl) {
|
|
404
|
+
const probe = new HttpProbe(prodUrl);
|
|
405
|
+
probes.push({ probe, label: 'production', url: prodUrl });
|
|
406
|
+
if (!silent) console.log(chalk.gray(` → Probing production: ${prodUrl}`));
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
for (const { probe, label } of probes) {
|
|
410
|
+
allTests.push(...buildUrlTestSuite(probe, label));
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
if (!silent) {
|
|
414
|
+
console.log(chalk.gray(`\n Building HTTP test suite: ${allTests.length} tests across ${probes.length} target(s)\n`));
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
const dashboard = silent ? null : new LiveDashboard();
|
|
418
|
+
const runner = new TestRunner();
|
|
419
|
+
const autoBugs = [];
|
|
420
|
+
|
|
421
|
+
runner.on('result', r => {
|
|
422
|
+
if (r.status === 'FAIL') {
|
|
423
|
+
autoBugs.push({
|
|
424
|
+
id : `HTTP-${shortId()}`,
|
|
425
|
+
title : r.name,
|
|
426
|
+
severity : r.sev || (r.type === 'security' || r.type === 'auth' ? 'P0' : 'P2'),
|
|
427
|
+
status : 'OPEN',
|
|
428
|
+
description: r.error || '',
|
|
429
|
+
createdAt : timestamp(),
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
if (dashboard) dashboard.start();
|
|
435
|
+
const results = await runner.run(allTests, dashboard);
|
|
436
|
+
if (dashboard) dashboard.stop();
|
|
437
|
+
|
|
438
|
+
// ── Route scan summary ─────────────────────────────────────────────────
|
|
439
|
+
const routeScans = [];
|
|
440
|
+
for (const { probe, label, url } of probes) {
|
|
441
|
+
if (!silent) {
|
|
442
|
+
console.log(chalk.gray(`\n Crawling routes for ${label}...`));
|
|
443
|
+
}
|
|
444
|
+
const probeResults = await probe.probeRoutes(COMMON_ROUTES.slice(0, 12));
|
|
445
|
+
for (const pr of probeResults) {
|
|
446
|
+
routeScans.push({ label, url, ...pr });
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const duration = Date.now() - new Date(startedAt).getTime();
|
|
451
|
+
const summary = buildSummary(results);
|
|
452
|
+
const coverage = buildCoverageMatrix(results);
|
|
453
|
+
|
|
454
|
+
if (!silent) printResultsSummary(results);
|
|
455
|
+
|
|
456
|
+
const run = {
|
|
457
|
+
id: runId, type: 'url-qa', version: VERSION, startedAt, duration,
|
|
458
|
+
urls : probes.map(p => ({ label: p.label, url: p.url })),
|
|
459
|
+
results, bugReports: autoBugs, summary, coverage, routeScans,
|
|
460
|
+
};
|
|
461
|
+
|
|
462
|
+
await saveRun(run);
|
|
463
|
+
const reportFile = await exportReport(run);
|
|
464
|
+
|
|
465
|
+
if (!silent && reportFile) {
|
|
466
|
+
console.log(chalk.gray(` 📄 URL QA Report: ${reportFile}`));
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
return run;
|
|
99
470
|
}
|
|
100
471
|
|
|
101
472
|
// ─────────────────────────────────────────────────────────────────────────
|
|
102
|
-
// Live Dashboard Renderer
|
|
473
|
+
// Live Dashboard Renderer (v9.0 retained + v10 label)
|
|
103
474
|
// ─────────────────────────────────────────────────────────────────────────
|
|
104
475
|
|
|
105
476
|
class LiveDashboard {
|
|
@@ -132,18 +503,15 @@ class LiveDashboard {
|
|
|
132
503
|
render(summary) {
|
|
133
504
|
if (!this.#active) return;
|
|
134
505
|
this.#clearLines();
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
this.#lines = lines.length;
|
|
506
|
+
const lines = this.#buildLines(summary);
|
|
507
|
+
this.#lines = lines.length;
|
|
138
508
|
process.stdout.write(lines.join('\n') + '\n');
|
|
139
509
|
}
|
|
140
510
|
|
|
141
511
|
#clearLines() {
|
|
142
512
|
if (this.#lines > 0) {
|
|
143
513
|
process.stdout.write(CURSOR_UP(this.#lines) + CLEAR_LINE);
|
|
144
|
-
for (let i = 1; i < this.#lines; i++)
|
|
145
|
-
process.stdout.write('\n' + CLEAR_LINE);
|
|
146
|
-
}
|
|
514
|
+
for (let i = 1; i < this.#lines; i++) process.stdout.write('\n' + CLEAR_LINE);
|
|
147
515
|
process.stdout.write(CURSOR_UP(this.#lines - 1));
|
|
148
516
|
}
|
|
149
517
|
}
|
|
@@ -162,12 +530,10 @@ class LiveDashboard {
|
|
|
162
530
|
const w = Math.min(process.stdout.columns || 80, 88);
|
|
163
531
|
const bar = '─'.repeat(w - 2);
|
|
164
532
|
|
|
165
|
-
// Header
|
|
166
533
|
lines.push(chalk.hex('#00F5FF').bold(`┌${bar}┐`));
|
|
167
|
-
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.hex('#BF40FF').bold(
|
|
534
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.hex('#BF40FF').bold(` ⚡ BACKLIST LIVE QA DASHBOARD v${VERSION}`.padEnd(w - 2)) + chalk.hex('#00F5FF').bold('│'));
|
|
168
535
|
lines.push(chalk.hex('#00F5FF').bold(`├${bar}┤`));
|
|
169
536
|
|
|
170
|
-
// Metrics row
|
|
171
537
|
const metrics = [
|
|
172
538
|
`${chalk.green('✓')} ${chalk.white.bold(passed)} passed`,
|
|
173
539
|
`${chalk.red('✗')} ${chalk.white.bold(failed)} failed`,
|
|
@@ -177,25 +543,19 @@ class LiveDashboard {
|
|
|
177
543
|
].map(m => m.padEnd(20)).join(' ');
|
|
178
544
|
lines.push(chalk.hex('#00F5FF').bold('│') + ' ' + metrics.slice(0, w - 4) + chalk.hex('#00F5FF').bold('│'));
|
|
179
545
|
|
|
180
|
-
// Progress bar
|
|
181
546
|
const pBar = buildProgressBar(passRate, 30);
|
|
182
547
|
lines.push(chalk.hex('#00F5FF').bold('│') + ` Pass rate [${pBar}] ${chalk.white.bold(passRate + '%')} (${total} tests)`.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
183
548
|
|
|
184
|
-
// System health
|
|
185
549
|
const sysLine = ` ${DIM('Heap')} ${chalk.white(sys.heapMB + 'MB')} ${DIM('RSS')} ${chalk.white(sys.rss)} ${DIM('Uptime')} ${chalk.white(sys.uptime + 's')} ${DIM('Node')} ${chalk.white(process.version)}`;
|
|
186
550
|
lines.push(chalk.hex('#00F5FF').bold('│') + sysLine.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
187
|
-
|
|
188
551
|
lines.push(chalk.hex('#00F5FF').bold(`├${'─'.repeat(w - 2)}┤`));
|
|
189
552
|
|
|
190
|
-
// Currently running
|
|
191
553
|
const runLine = this.#runningTest
|
|
192
554
|
? ` ${chalk.cyan('⟳')} ${chalk.cyan('Running:')} ${chalk.white(this.#runningTest.slice(0, w - 16))}`
|
|
193
|
-
: ` ${chalk.gray('⊘ Idle
|
|
555
|
+
: ` ${chalk.gray('⊘ Idle...')}`;
|
|
194
556
|
lines.push(chalk.hex('#00F5FF').bold('│') + runLine.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
195
|
-
|
|
196
557
|
lines.push(chalk.hex('#00F5FF').bold(`├${'─'.repeat(w - 2)}┤`));
|
|
197
558
|
|
|
198
|
-
// Last 5 results
|
|
199
559
|
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.gray(' Recent results:').padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
200
560
|
const recentResults = results.slice(-5);
|
|
201
561
|
for (const r of recentResults) {
|
|
@@ -205,12 +565,10 @@ class LiveDashboard {
|
|
|
205
565
|
const row = ` ${colorStatus(r.status)} ${type} ${chalk.white(name)} ${dur}`;
|
|
206
566
|
lines.push(chalk.hex('#00F5FF').bold('│') + row.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
207
567
|
}
|
|
208
|
-
// Pad to always show 5 lines
|
|
209
568
|
for (let i = recentResults.length; i < 5; i++) {
|
|
210
569
|
lines.push(chalk.hex('#00F5FF').bold('│') + ' '.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
211
570
|
}
|
|
212
571
|
|
|
213
|
-
// Bugs
|
|
214
572
|
lines.push(chalk.hex('#00F5FF').bold(`├${'─'.repeat(w - 2)}┤`));
|
|
215
573
|
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.gray(' Active bugs:').padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
216
574
|
const recentBugs = this.#bugs.slice(-3);
|
|
@@ -222,7 +580,6 @@ class LiveDashboard {
|
|
|
222
580
|
lines.push(chalk.hex('#00F5FF').bold('│') + ' '.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
223
581
|
}
|
|
224
582
|
|
|
225
|
-
// Log
|
|
226
583
|
lines.push(chalk.hex('#00F5FF').bold(`├${'─'.repeat(w - 2)}┤`));
|
|
227
584
|
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.gray(' Event log:').padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
228
585
|
const recentLogs = this.#log.slice(-4);
|
|
@@ -234,14 +591,13 @@ class LiveDashboard {
|
|
|
234
591
|
}
|
|
235
592
|
|
|
236
593
|
lines.push(chalk.hex('#00F5FF').bold(`└${bar}┘`));
|
|
237
|
-
lines.push(DIM(' Press Ctrl+C to stop
|
|
238
|
-
|
|
594
|
+
lines.push(DIM(' Press Ctrl+C to stop'));
|
|
239
595
|
return lines;
|
|
240
596
|
}
|
|
241
597
|
}
|
|
242
598
|
|
|
243
599
|
// ─────────────────────────────────────────────────────────────────────────
|
|
244
|
-
// Test Runner
|
|
600
|
+
// Test Runner (v9 retained + sev field propagation)
|
|
245
601
|
// ─────────────────────────────────────────────────────────────────────────
|
|
246
602
|
|
|
247
603
|
class TestRunner extends EventEmitter {
|
|
@@ -270,17 +626,17 @@ class TestRunner extends EventEmitter {
|
|
|
270
626
|
dashboard.addResult(result);
|
|
271
627
|
if (result.status === 'FAIL') {
|
|
272
628
|
dashboard.addBug({
|
|
273
|
-
id
|
|
274
|
-
title
|
|
275
|
-
severity
|
|
276
|
-
status
|
|
629
|
+
id : `AUTO-${shortId()}`,
|
|
630
|
+
title : `${test.name}`,
|
|
631
|
+
severity: test.sev || this.#classifySeverity(test.type, result.error),
|
|
632
|
+
status : 'OPEN',
|
|
277
633
|
});
|
|
278
634
|
dashboard.addLog(chalk.red(`FAIL: ${test.name} — ${result.error ?? 'unknown'}`));
|
|
279
635
|
} else {
|
|
280
636
|
dashboard.addLog(chalk.green(`${result.status}: ${test.name} (${formatDuration(result.duration)})`));
|
|
281
637
|
}
|
|
282
638
|
dashboard.render({});
|
|
283
|
-
await sleep(
|
|
639
|
+
await sleep(60);
|
|
284
640
|
}
|
|
285
641
|
}
|
|
286
642
|
|
|
@@ -298,7 +654,7 @@ class TestRunner extends EventEmitter {
|
|
|
298
654
|
}
|
|
299
655
|
|
|
300
656
|
async #runOne(test) {
|
|
301
|
-
const { id, name, type, fn, timeout = DEFAULT_TIMEOUT_MS } = test;
|
|
657
|
+
const { id, name, type, sev, fn, timeout = DEFAULT_TIMEOUT_MS } = test;
|
|
302
658
|
const start = Date.now();
|
|
303
659
|
let retries = 0;
|
|
304
660
|
let lastError = null;
|
|
@@ -310,7 +666,7 @@ class TestRunner extends EventEmitter {
|
|
|
310
666
|
sleep(timeout).then(() => { throw new Error(`Timed out after ${timeout}ms`); }),
|
|
311
667
|
]);
|
|
312
668
|
const status = attempt > 0 ? 'FLAKY' : 'PASS';
|
|
313
|
-
return { id, name, type, status, duration: Date.now() - start, retries: attempt, error: null };
|
|
669
|
+
return { id, name, type, sev, status, duration: Date.now() - start, retries: attempt, error: null };
|
|
314
670
|
} catch (err) {
|
|
315
671
|
lastError = err.message;
|
|
316
672
|
retries = attempt;
|
|
@@ -318,145 +674,86 @@ class TestRunner extends EventEmitter {
|
|
|
318
674
|
}
|
|
319
675
|
}
|
|
320
676
|
|
|
321
|
-
return { id, name, type, status: 'FAIL', duration: Date.now() - start, retries, error: lastError };
|
|
677
|
+
return { id, name, type, sev, status: 'FAIL', duration: Date.now() - start, retries, error: lastError };
|
|
322
678
|
}
|
|
323
679
|
}
|
|
324
680
|
|
|
325
681
|
// ─────────────────────────────────────────────────────────────────────────
|
|
326
|
-
//
|
|
682
|
+
// File-System Test Suites (v9 retained)
|
|
327
683
|
// ─────────────────────────────────────────────────────────────────────────
|
|
328
684
|
|
|
329
685
|
function buildEndpointTests(endpoints) {
|
|
330
686
|
const tests = [];
|
|
331
|
-
|
|
332
687
|
for (const ep of endpoints) {
|
|
333
688
|
const label = `${ep.method} ${ep.route}`;
|
|
334
|
-
|
|
335
|
-
tests.push({ id: shortId(), name: `Happy path: ${label}`, type: 'happy-path', fn: async () => {
|
|
689
|
+
tests.push({ id: shortId(), name: `Happy path: ${label}`, type: 'happy-path', sev: 'P2', fn: async () => {
|
|
336
690
|
await sleep(30 + Math.random() * 80);
|
|
337
691
|
if (!ep.route || !ep.method) throw new Error('Endpoint missing route or method');
|
|
338
692
|
}});
|
|
339
|
-
|
|
340
693
|
if (ep.schemaFields && Object.keys(ep.schemaFields).length > 0) {
|
|
341
|
-
tests.push({ id: shortId(), name: `Validation: ${label}`, type: 'validation', fn: async () => {
|
|
694
|
+
tests.push({ id: shortId(), name: `Validation: ${label}`, type: 'validation', sev: 'P2', fn: async () => {
|
|
342
695
|
await sleep(25);
|
|
343
696
|
const missing = Object.entries(ep.schemaFields).filter(([, t]) => !t);
|
|
344
697
|
if (missing.length) throw new Error(`Fields missing types: ${missing.map(([k]) => k).join(', ')}`);
|
|
345
698
|
}});
|
|
346
699
|
}
|
|
347
|
-
|
|
348
700
|
if (/\/admin|\/user|\/auth|\/profile|\/dashboard|\/private/i.test(ep.route)) {
|
|
349
|
-
tests.push({ id: shortId(), name: `Auth guard: ${label}`, type: 'auth', fn: async () => {
|
|
701
|
+
tests.push({ id: shortId(), name: `Auth guard: ${label}`, type: 'auth', sev: 'P0', fn: async () => {
|
|
350
702
|
await sleep(40);
|
|
351
703
|
}});
|
|
352
704
|
}
|
|
353
|
-
|
|
354
|
-
if (ep.pathParams?.length > 0) {
|
|
355
|
-
tests.push({ id: shortId(), name: `Edge case — empty param: ${label}`, type: 'edge-case', fn: async () => {
|
|
356
|
-
await sleep(20);
|
|
357
|
-
if (ep.pathParams.find(p => p.length === 0) !== undefined) throw new Error('Empty path parameter');
|
|
358
|
-
}});
|
|
359
|
-
}
|
|
360
705
|
}
|
|
361
|
-
|
|
362
706
|
return tests;
|
|
363
707
|
}
|
|
364
708
|
|
|
365
709
|
function buildFullSystemTests(projectDir = process.cwd()) {
|
|
366
710
|
const tests = [];
|
|
367
711
|
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
const exists = await fs.pathExists(projectDir);
|
|
371
|
-
if (!exists) throw new Error('Project directory not found');
|
|
712
|
+
tests.push({ id: shortId(), name: 'Project structure integrity', type: 'e2e', sev: 'P1', fn: async () => {
|
|
713
|
+
if (!(await fs.pathExists(projectDir))) throw new Error('Project directory not found');
|
|
372
714
|
}});
|
|
373
|
-
|
|
374
|
-
tests.push({ id: shortId(), name: 'Package.json valid', type: 'validation', fn: async () => {
|
|
715
|
+
tests.push({ id: shortId(), name: 'Package.json valid', type: 'validation', sev: 'P1', fn: async () => {
|
|
375
716
|
const pkgPath = path.join(projectDir, 'package.json');
|
|
376
717
|
if (!(await fs.pathExists(pkgPath))) throw new Error('package.json missing');
|
|
377
718
|
const pkg = await fs.readJson(pkgPath);
|
|
378
719
|
if (!pkg.name) throw new Error('package.json has no name field');
|
|
379
720
|
}});
|
|
380
|
-
|
|
381
|
-
tests.push({ id: shortId(), name: 'Dependencies declared', type: 'validation', fn: async () => {
|
|
721
|
+
tests.push({ id: shortId(), name: 'Dependencies declared', type: 'validation', sev: 'P2', fn: async () => {
|
|
382
722
|
const pkgPath = path.join(projectDir, 'package.json');
|
|
383
723
|
if (!(await fs.pathExists(pkgPath))) return;
|
|
384
|
-
const pkg
|
|
724
|
+
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
385
725
|
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
386
726
|
if (Object.keys(deps).length === 0) throw new Error('No dependencies declared');
|
|
387
727
|
}});
|
|
388
|
-
|
|
389
|
-
// ── API/Backend Tests ────────────────────────────────────────────────
|
|
390
|
-
tests.push({ id: shortId(), name: 'API routes file exists', type: 'happy-path', fn: async () => {
|
|
728
|
+
tests.push({ id: shortId(), name: 'API routes file exists', type: 'happy-path', sev: 'P1', fn: async () => {
|
|
391
729
|
const candidates = ['src/routes', 'routes', 'src/api', 'api', 'src/controllers', 'controllers'];
|
|
392
|
-
for (const c of candidates) {
|
|
393
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
394
|
-
}
|
|
730
|
+
for (const c of candidates) { if (await fs.pathExists(path.join(projectDir, c))) return; }
|
|
395
731
|
throw new Error('No routes/api directory found');
|
|
396
732
|
}});
|
|
397
|
-
|
|
398
|
-
tests.push({ id: shortId(), name: 'Entry point reachable', type: 'happy-path', fn: async () => {
|
|
733
|
+
tests.push({ id: shortId(), name: 'Entry point reachable', type: 'happy-path', sev: 'P0', fn: async () => {
|
|
399
734
|
const candidates = ['src/index.ts', 'src/index.js', 'index.ts', 'index.js', 'main.py', 'main.go', 'Program.cs'];
|
|
400
|
-
for (const c of candidates) {
|
|
401
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
402
|
-
}
|
|
735
|
+
for (const c of candidates) { if (await fs.pathExists(path.join(projectDir, c))) return; }
|
|
403
736
|
throw new Error('No recognisable entry point found');
|
|
404
737
|
}});
|
|
405
|
-
|
|
406
|
-
tests.push({ id: shortId(), name: 'Environment config present', type: 'validation', fn: async () => {
|
|
738
|
+
tests.push({ id: shortId(), name: 'Environment config present', type: 'validation', sev: 'P1', fn: async () => {
|
|
407
739
|
const candidates = ['.env', '.env.example', '.env.sample', 'config.js', 'config.ts', 'appsettings.json'];
|
|
408
|
-
for (const c of candidates) {
|
|
409
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
410
|
-
}
|
|
740
|
+
for (const c of candidates) { if (await fs.pathExists(path.join(projectDir, c))) return; }
|
|
411
741
|
throw new Error('No environment config file found');
|
|
412
742
|
}});
|
|
413
|
-
|
|
414
|
-
// ── Auth Tests ───────────────────────────────────────────────────────
|
|
415
|
-
tests.push({ id: shortId(), name: 'JWT middleware present', type: 'auth', fn: async () => {
|
|
416
|
-
const candidates = ['src/middleware', 'middleware', 'src/middlewares', 'middlewares'];
|
|
417
|
-
for (const c of candidates) {
|
|
418
|
-
if (await fs.pathExists(path.join(projectDir, c))) {
|
|
419
|
-
const files = await fs.readdir(path.join(projectDir, c)).catch(() => []);
|
|
420
|
-
const hasAuth = files.some(f => /auth|jwt|guard|verify/i.test(f));
|
|
421
|
-
if (hasAuth) return;
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
// soft check — not all stacks need middleware dir
|
|
425
|
-
}});
|
|
426
|
-
|
|
427
|
-
tests.push({ id: shortId(), name: 'Password hashing config', type: 'security', fn: async () => {
|
|
428
|
-
await sleep(35);
|
|
429
|
-
// Scan for bcrypt/argon2 usage in package.json
|
|
743
|
+
tests.push({ id: shortId(), name: 'Password hashing library', type: 'security', sev: 'P0', fn: async () => {
|
|
430
744
|
const pkgPath = path.join(projectDir, 'package.json');
|
|
431
745
|
if (!(await fs.pathExists(pkgPath))) return;
|
|
432
746
|
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
433
747
|
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
434
|
-
|
|
435
|
-
|
|
748
|
+
if (!['bcrypt', 'bcryptjs', 'argon2', 'argon2d'].some(d => deps[d]))
|
|
749
|
+
throw new Error('No password hashing library (bcrypt/argon2) found');
|
|
436
750
|
}});
|
|
437
|
-
|
|
438
|
-
// ── Database Tests ───────────────────────────────────────────────────
|
|
439
|
-
tests.push({ id: shortId(), name: 'Database schema defined', type: 'validation', fn: async () => {
|
|
751
|
+
tests.push({ id: shortId(), name: 'Database schema defined', type: 'validation', sev: 'P1', fn: async () => {
|
|
440
752
|
const candidates = ['prisma/schema.prisma', 'schema.prisma', 'src/models', 'models', 'src/entities'];
|
|
441
|
-
for (const c of candidates) {
|
|
442
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
443
|
-
}
|
|
753
|
+
for (const c of candidates) { if (await fs.pathExists(path.join(projectDir, c))) return; }
|
|
444
754
|
throw new Error('No database schema/models directory found');
|
|
445
755
|
}});
|
|
446
|
-
|
|
447
|
-
tests.push({ id: shortId(), name: 'Migration scripts present', type: 'validation', fn: async () => {
|
|
448
|
-
await sleep(20);
|
|
449
|
-
const candidates = ['prisma/migrations', 'migrations', 'db/migrations', 'src/migrations'];
|
|
450
|
-
for (const c of candidates) {
|
|
451
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
452
|
-
}
|
|
453
|
-
// Acceptable if using ORMs that auto-migrate
|
|
454
|
-
}});
|
|
455
|
-
|
|
456
|
-
// ── Security Scan ────────────────────────────────────────────────────
|
|
457
|
-
tests.push({ id: shortId(), name: 'CORS config found', type: 'security', fn: async () => {
|
|
458
|
-
await sleep(40);
|
|
459
|
-
// Scan src/index.* for cors usage
|
|
756
|
+
tests.push({ id: shortId(), name: 'CORS config found', type: 'security', sev: 'P1', fn: async () => {
|
|
460
757
|
const candidates = ['src/index.ts', 'src/index.js', 'src/app.ts', 'src/app.js', 'index.js', 'index.ts'];
|
|
461
758
|
for (const c of candidates) {
|
|
462
759
|
const filePath = path.join(projectDir, c);
|
|
@@ -465,70 +762,42 @@ function buildFullSystemTests(projectDir = process.cwd()) {
|
|
|
465
762
|
if (/cors|CORS/i.test(content)) return;
|
|
466
763
|
}
|
|
467
764
|
}
|
|
468
|
-
throw new Error('No CORS configuration detected
|
|
765
|
+
throw new Error('No CORS configuration detected');
|
|
469
766
|
}});
|
|
470
|
-
|
|
471
|
-
tests.push({ id: shortId(), name: 'Rate limiting configured', type: 'security', fn: async () => {
|
|
472
|
-
await sleep(30);
|
|
767
|
+
tests.push({ id: shortId(), name: 'Rate limiting configured', type: 'security', sev: 'P1', fn: async () => {
|
|
473
768
|
const pkgPath = path.join(projectDir, 'package.json');
|
|
474
769
|
if (!(await fs.pathExists(pkgPath))) return;
|
|
475
770
|
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
476
771
|
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
477
|
-
|
|
478
|
-
|
|
772
|
+
if (!['express-rate-limit', 'rate-limiter-flexible', 'fastapi-limiter', 'throttler'].some(d => deps[d]))
|
|
773
|
+
throw new Error('No rate-limiting library found');
|
|
479
774
|
}});
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
const
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
const content = await fs.readFile(filePath, 'utf8').catch(() => '');
|
|
489
|
-
if (secretPattern.test(content)) throw new Error(`Hardcoded secret detected in ${t}`);
|
|
775
|
+
tests.push({ id: shortId(), name: 'Secrets not hardcoded', type: 'security', sev: 'P0', fn: async () => {
|
|
776
|
+
const pattern = /(?:password|secret|apikey|api_key)\s*=\s*['"][^'"]{6,}['"]/i;
|
|
777
|
+
const targets = ['src/index.ts', 'src/index.js', 'src/app.ts', 'src/app.js'];
|
|
778
|
+
for (const t of targets) {
|
|
779
|
+
const fp = path.join(projectDir, t);
|
|
780
|
+
if (await fs.pathExists(fp)) {
|
|
781
|
+
const content = await fs.readFile(fp, 'utf8').catch(() => '');
|
|
782
|
+
if (pattern.test(content)) throw new Error(`Hardcoded secret detected in ${t}`);
|
|
490
783
|
}
|
|
491
784
|
}
|
|
492
785
|
}});
|
|
493
|
-
|
|
494
|
-
// ── Performance Checks ───────────────────────────────────────────────
|
|
495
|
-
tests.push({ id: shortId(), name: 'Heap memory acceptable', type: 'performance', fn: async () => {
|
|
496
|
-
await sleep(20);
|
|
786
|
+
tests.push({ id: shortId(), name: 'Heap memory acceptable', type: 'performance', sev: 'P2', fn: async () => {
|
|
497
787
|
const heapMB = process.memoryUsage().heapUsed / 1024 / 1024;
|
|
498
|
-
if (heapMB > 512) throw new Error(`Heap
|
|
788
|
+
if (heapMB > 512) throw new Error(`Heap ${heapMB.toFixed(0)}MB exceeds 512MB limit`);
|
|
499
789
|
}});
|
|
500
|
-
|
|
501
|
-
tests.push({ id: shortId(), name: 'File system scan speed', type: 'performance', fn: async () => {
|
|
502
|
-
const t0 = performance.now();
|
|
503
|
-
await fs.readdir(projectDir).catch(() => []);
|
|
504
|
-
const elapsed = performance.now() - t0;
|
|
505
|
-
if (elapsed > 2000) throw new Error(`FS scan too slow: ${elapsed.toFixed(0)}ms`);
|
|
506
|
-
}});
|
|
507
|
-
|
|
508
|
-
tests.push({ id: shortId(), name: 'Node.js version check', type: 'happy-path', fn: async () => {
|
|
509
|
-
const maj = parseInt(process.version.slice(1));
|
|
510
|
-
if (maj < 18) throw new Error(`Node.js ${process.version} — requires v18+`);
|
|
511
|
-
}});
|
|
512
|
-
|
|
513
|
-
// ── Docker / Deploy ──────────────────────────────────────────────────
|
|
514
|
-
tests.push({ id: shortId(), name: 'Dockerfile present', type: 'e2e', fn: async () => {
|
|
790
|
+
tests.push({ id: shortId(), name: 'Dockerfile present', type: 'e2e', sev: 'P2', fn: async () => {
|
|
515
791
|
const candidates = ['Dockerfile', 'Dockerfile.dev', 'docker-compose.yml', 'docker-compose.yaml'];
|
|
516
|
-
for (const c of candidates) {
|
|
517
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
518
|
-
}
|
|
792
|
+
for (const c of candidates) { if (await fs.pathExists(path.join(projectDir, c))) return; }
|
|
519
793
|
throw new Error('No Docker configuration found');
|
|
520
794
|
}});
|
|
521
|
-
|
|
522
|
-
tests.push({ id: shortId(), name: 'CI/CD pipeline configured', type: 'e2e', fn: async () => {
|
|
795
|
+
tests.push({ id: shortId(), name: 'CI/CD pipeline configured', type: 'e2e', sev: 'P2', fn: async () => {
|
|
523
796
|
const ciPaths = ['.github/workflows', '.gitlab-ci.yml', '.circleci', 'Jenkinsfile'];
|
|
524
|
-
for (const c of ciPaths) {
|
|
525
|
-
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
526
|
-
}
|
|
797
|
+
for (const c of ciPaths) { if (await fs.pathExists(path.join(projectDir, c))) return; }
|
|
527
798
|
throw new Error('No CI/CD pipeline detected');
|
|
528
799
|
}});
|
|
529
|
-
|
|
530
|
-
// ── Test Infrastructure ──────────────────────────────────────────────
|
|
531
|
-
tests.push({ id: shortId(), name: 'Test files exist', type: 'e2e', fn: async () => {
|
|
800
|
+
tests.push({ id: shortId(), name: 'Test files exist', type: 'e2e', sev: 'P2', fn: async () => {
|
|
532
801
|
const testDirs = ['tests', 'test', '__tests__', 'spec'];
|
|
533
802
|
for (const d of testDirs) {
|
|
534
803
|
if (await fs.pathExists(path.join(projectDir, d))) {
|
|
@@ -538,42 +807,26 @@ function buildFullSystemTests(projectDir = process.cwd()) {
|
|
|
538
807
|
}
|
|
539
808
|
throw new Error('No test files found');
|
|
540
809
|
}});
|
|
541
|
-
|
|
542
|
-
tests.push({ id: shortId(), name: 'Test script configured', type: 'validation', fn: async () => {
|
|
543
|
-
const pkgPath = path.join(projectDir, 'package.json');
|
|
544
|
-
if (!(await fs.pathExists(pkgPath))) return;
|
|
545
|
-
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
546
|
-
if (!pkg.scripts?.test) throw new Error('No "test" script in package.json');
|
|
547
|
-
}});
|
|
548
|
-
|
|
549
|
-
// ── Swagger / Docs ───────────────────────────────────────────────────
|
|
550
|
-
tests.push({ id: shortId(), name: 'API documentation configured', type: 'happy-path', fn: async () => {
|
|
551
|
-
await sleep(20);
|
|
810
|
+
tests.push({ id: shortId(), name: 'API documentation configured', type: 'happy-path', sev: 'P3', fn: async () => {
|
|
552
811
|
const pkgPath = path.join(projectDir, 'package.json');
|
|
553
812
|
if (!(await fs.pathExists(pkgPath))) return;
|
|
554
813
|
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
555
814
|
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
556
|
-
|
|
557
|
-
|
|
815
|
+
if (!['swagger-ui-express', 'swagger-jsdoc', '@nestjs/swagger', 'fastapi', 'springdoc-openapi'].some(d => deps[d]))
|
|
816
|
+
throw new Error('No API documentation library found');
|
|
558
817
|
}});
|
|
559
|
-
|
|
560
|
-
// ── Logging ──────────────────────────────────────────────────────────
|
|
561
|
-
tests.push({ id: shortId(), name: 'Logging library present', type: 'validation', fn: async () => {
|
|
562
|
-
await sleep(15);
|
|
818
|
+
tests.push({ id: shortId(), name: 'Logging library present', type: 'validation', sev: 'P2', fn: async () => {
|
|
563
819
|
const pkgPath = path.join(projectDir, 'package.json');
|
|
564
820
|
if (!(await fs.pathExists(pkgPath))) return;
|
|
565
821
|
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
566
822
|
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
567
|
-
|
|
568
|
-
|
|
823
|
+
if (!['winston', 'pino', 'morgan', 'log4j', 'structlog'].some(d => deps[d]))
|
|
824
|
+
throw new Error('No structured logging library found');
|
|
569
825
|
}});
|
|
570
|
-
|
|
571
|
-
// ── Baseline (always pass) ────────────────────────────────────────────
|
|
572
|
-
tests.push({ id: shortId(), name: 'QA system operational', type: 'happy-path', fn: async () => {
|
|
826
|
+
tests.push({ id: shortId(), name: 'QA system operational', type: 'happy-path', sev: 'P3', fn: async () => {
|
|
573
827
|
await fs.ensureDir(QA_DIR);
|
|
574
828
|
}});
|
|
575
|
-
|
|
576
|
-
tests.push({ id: shortId(), name: 'Report directory writable', type: 'happy-path', fn: async () => {
|
|
829
|
+
tests.push({ id: shortId(), name: 'Report directory writable', type: 'happy-path', sev: 'P3', fn: async () => {
|
|
577
830
|
await fs.ensureDir(REPORT_DIR);
|
|
578
831
|
const testFile = path.join(REPORT_DIR, `.write-test-${shortId()}`);
|
|
579
832
|
await fs.writeFile(testFile, 'ok');
|
|
@@ -583,14 +836,12 @@ function buildFullSystemTests(projectDir = process.cwd()) {
|
|
|
583
836
|
return tests;
|
|
584
837
|
}
|
|
585
838
|
|
|
586
|
-
// ── UI Simulation tests ───────────────────────────────────────────────────
|
|
587
|
-
|
|
588
839
|
function buildUITests(srcDir = path.join(process.cwd(), 'src')) {
|
|
589
840
|
return [
|
|
590
|
-
{ id: shortId(), name: 'Frontend src directory exists', type: 'ui', fn: async () => {
|
|
591
|
-
if (!(await fs.pathExists(srcDir))) throw new Error(`src
|
|
841
|
+
{ id: shortId(), name: 'Frontend src directory exists', type: 'ui', sev: 'P1', fn: async () => {
|
|
842
|
+
if (!(await fs.pathExists(srcDir))) throw new Error(`src not found: ${srcDir}`);
|
|
592
843
|
}},
|
|
593
|
-
{ id: shortId(), name: 'Component files present', type: 'ui', fn: async () => {
|
|
844
|
+
{ id: shortId(), name: 'Component files present', type: 'ui', sev: 'P1', fn: async () => {
|
|
594
845
|
const exts = ['.tsx', '.jsx', '.vue', '.svelte'];
|
|
595
846
|
let found = false;
|
|
596
847
|
const walk = async (dir) => {
|
|
@@ -601,35 +852,22 @@ function buildUITests(srcDir = path.join(process.cwd(), 'src')) {
|
|
|
601
852
|
}
|
|
602
853
|
};
|
|
603
854
|
await walk(srcDir);
|
|
604
|
-
if (!found) throw new Error('No component files
|
|
855
|
+
if (!found) throw new Error('No component files found');
|
|
605
856
|
}},
|
|
606
|
-
{ id: shortId(), name: 'Styles configured', type: 'ui', fn: async () => {
|
|
607
|
-
const
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
const entries = await fs.readdir(cwd).catch(() => []);
|
|
611
|
-
if (entries.some(f => f.includes(pat))) return;
|
|
612
|
-
}
|
|
613
|
-
throw new Error('No styling configuration found');
|
|
857
|
+
{ id: shortId(), name: 'Styles configured', type: 'ui', sev: 'P2', fn: async () => {
|
|
858
|
+
const patterns = ['tailwind.config', 'postcss.config', 'vite.config', 'styles', 'css', 'scss'];
|
|
859
|
+
const entries = await fs.readdir(process.cwd()).catch(() => []);
|
|
860
|
+
if (!entries.some(f => patterns.some(p => f.includes(p)))) throw new Error('No styling configuration found');
|
|
614
861
|
}},
|
|
615
|
-
{ id: shortId(), name: 'API client configuration', type: 'ui', fn: async () => {
|
|
862
|
+
{ id: shortId(), name: 'API client configuration', type: 'ui', sev: 'P2', fn: async () => {
|
|
616
863
|
const apiFiles = ['src/api', 'src/services', 'src/lib', 'src/utils'];
|
|
617
|
-
for (const f of apiFiles) {
|
|
618
|
-
|
|
619
|
-
}
|
|
620
|
-
throw new Error('No API client/services directory found in frontend');
|
|
621
|
-
}},
|
|
622
|
-
{ id: shortId(), name: 'Route configuration present', type: 'ui', fn: async () => {
|
|
623
|
-
const routeFiles = ['src/router', 'src/routes', 'src/pages', 'pages', 'app/routes'];
|
|
624
|
-
for (const f of routeFiles) {
|
|
625
|
-
if (await fs.pathExists(path.join(process.cwd(), f))) return;
|
|
626
|
-
}
|
|
864
|
+
for (const f of apiFiles) { if (await fs.pathExists(path.join(process.cwd(), f))) return; }
|
|
865
|
+
throw new Error('No API client/services directory found');
|
|
627
866
|
}},
|
|
628
867
|
];
|
|
629
868
|
}
|
|
630
869
|
|
|
631
|
-
// ──
|
|
632
|
-
|
|
870
|
+
// ── Helpers ────────────────────────────────────────────────────────────────
|
|
633
871
|
function buildCoverageMatrix(results) {
|
|
634
872
|
const matrix = {};
|
|
635
873
|
for (const r of results) {
|
|
@@ -653,22 +891,22 @@ function buildSummary(results) {
|
|
|
653
891
|
};
|
|
654
892
|
}
|
|
655
893
|
|
|
656
|
-
//
|
|
894
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
895
|
+
// HTML Report v10.0 — with route cards + dual-URL diff
|
|
896
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
657
897
|
|
|
658
898
|
function buildHTMLReport(runData) {
|
|
659
|
-
const { id, startedAt, duration, results, bugReports, coverage, summary } = runData;
|
|
899
|
+
const { id, startedAt, duration, results, bugReports, coverage, summary, urls = [], routeScans = [] } = runData;
|
|
660
900
|
const passRate = summary.total > 0 ? ((summary.passed / summary.total) * 100).toFixed(1) : 0;
|
|
661
901
|
const statusColor = passRate >= 90 ? '#22c55e' : passRate >= 70 ? '#f59e0b' : '#ef4444';
|
|
662
902
|
|
|
663
903
|
const typeColors = {
|
|
664
|
-
'happy-path' : ['#064e3b','#34d399'],
|
|
665
|
-
'
|
|
666
|
-
'
|
|
667
|
-
'
|
|
668
|
-
'
|
|
669
|
-
'
|
|
670
|
-
'e2e' : ['#1a3b2a','#4ade80'],
|
|
671
|
-
'ui' : ['#2a1a3b','#a78bfa'],
|
|
904
|
+
'happy-path' : ['#064e3b','#34d399'], 'validation' : ['#1e3a5f','#60a5fa'],
|
|
905
|
+
'auth' : ['#3b1f5e','#c084fc'], 'edge-case' : ['#3b2a1a','#f59e0b'],
|
|
906
|
+
'performance': ['#1a2a3b','#38bdf8'], 'security' : ['#450a0a','#f87171'],
|
|
907
|
+
'e2e' : ['#1a3b2a','#4ade80'], 'ui' : ['#2a1a3b','#a78bfa'],
|
|
908
|
+
'http' : ['#0f2a3b','#38bdf8'], 'seo' : ['#1a2e0f','#86efac'],
|
|
909
|
+
'a11y' : ['#2e1a0f','#fca5a5'], 'links' : ['#0f1a2e','#93c5fd'],
|
|
672
910
|
};
|
|
673
911
|
|
|
674
912
|
const badgeStyle = (type) => {
|
|
@@ -680,7 +918,7 @@ function buildHTMLReport(runData) {
|
|
|
680
918
|
const pct = d.total ? ((d.passed / d.total) * 100).toFixed(0) : 0;
|
|
681
919
|
const [, fg] = typeColors[type] ?? ['','#94a3b8'];
|
|
682
920
|
return `<div style="display:flex;align-items:center;gap:1rem;margin-bottom:.75rem">
|
|
683
|
-
<div style="width:
|
|
921
|
+
<div style="width:110px;font-size:.8rem;color:#94a3b8">${type}</div>
|
|
684
922
|
<div style="flex:1;background:#2d2d4e;border-radius:4px;height:8px;overflow:hidden">
|
|
685
923
|
<div style="height:100%;width:${pct}%;background:${fg};border-radius:4px"></div>
|
|
686
924
|
</div>
|
|
@@ -692,118 +930,163 @@ function buildHTMLReport(runData) {
|
|
|
692
930
|
<td>${r.name}</td>
|
|
693
931
|
<td><span style="${badgeStyle(r.type)}">${r.type}</span></td>
|
|
694
932
|
<td><span class="status status-${r.status.toLowerCase()}">${r.status}</span></td>
|
|
933
|
+
<td>${r.sev ? `<span class="sev-${(r.sev||'').toLowerCase()}">${r.sev}</span>` : '—'}</td>
|
|
695
934
|
<td>${r.duration}ms</td>
|
|
696
935
|
<td>${r.retries > 0 ? `<span style="background:#422006;color:#fb923c;padding:2px 8px;border-radius:4px;font-size:.75rem">${r.retries}x retry</span>` : '—'}</td>
|
|
697
936
|
<td class="err">${r.error ? `<code>${r.error}</code>` : '—'}</td>
|
|
698
937
|
</tr>`).join('');
|
|
699
938
|
|
|
700
939
|
const bugCards = bugReports.length ? bugReports.map(b => `
|
|
701
|
-
<div class="bug-card bug-${b.severity
|
|
702
|
-
<div class="bug-header"
|
|
940
|
+
<div class="bug-card bug-${(b.severity||'p3').toLowerCase()}">
|
|
941
|
+
<div class="bug-header">
|
|
942
|
+
<span class="bug-id">${b.id}</span>
|
|
943
|
+
<span class="bug-sev">${b.severity}</span>
|
|
944
|
+
<span class="bug-st">${b.status}</span>
|
|
945
|
+
</div>
|
|
703
946
|
<div class="bug-title">${b.title}</div>
|
|
704
947
|
${b.description ? `<div class="bug-desc">${b.description}</div>` : ''}
|
|
705
948
|
</div>`).join('') : '<p style="color:#34d399;text-align:center;padding:1rem">No bug reports 🎉</p>';
|
|
706
949
|
|
|
707
|
-
|
|
950
|
+
const urlCards = urls.length ? urls.map(u => `
|
|
951
|
+
<div style="background:#1e1e30;border:1px solid #2d2d4e;border-radius:8px;padding:1rem;margin-bottom:.75rem">
|
|
952
|
+
<div style="display:flex;justify-content:space-between;align-items:center">
|
|
953
|
+
<span style="font-size:.8rem;color:#64748b;text-transform:uppercase">${u.label}</span>
|
|
954
|
+
<a href="${u.url}" target="_blank" style="font-size:.8rem;color:#60a5fa">${u.url}</a>
|
|
955
|
+
</div>
|
|
956
|
+
</div>`).join('') : '';
|
|
957
|
+
|
|
958
|
+
const routeCards = routeScans.length ? routeScans.map(r => `
|
|
959
|
+
<div style="display:flex;align-items:center;gap:.75rem;padding:.5rem .75rem;border-bottom:1px solid #1a1a2e;font-size:.8rem">
|
|
960
|
+
<span style="font-family:monospace;color:${r.status >= 200 && r.status < 400 ? '#34d399' : r.status >= 500 ? '#f87171' : '#f59e0b'}">${r.status || 'ERR'}</span>
|
|
961
|
+
<span style="flex:1;color:#94a3b8;font-family:monospace">${r.route}</span>
|
|
962
|
+
<span style="color:#64748b">${r.duration}ms</span>
|
|
963
|
+
<span style="font-size:.7rem;padding:2px 6px;background:#1e293b;color:#64748b;border-radius:3px">${r.label}</span>
|
|
964
|
+
</div>`).join('') : '<p style="color:#64748b;font-size:.85rem;padding:.5rem">No route scans recorded.</p>';
|
|
965
|
+
|
|
708
966
|
const chartLabels = JSON.stringify(Object.keys(coverage));
|
|
709
967
|
const chartPassed = JSON.stringify(Object.values(coverage).map(d => d.passed));
|
|
710
968
|
const chartFailed = JSON.stringify(Object.values(coverage).map(d => d.failed));
|
|
711
969
|
|
|
970
|
+
const sevCounts = { P0: 0, P1: 0, P2: 0, P3: 0 };
|
|
971
|
+
bugReports.forEach(b => { if (sevCounts[b.severity] !== undefined) sevCounts[b.severity]++; });
|
|
972
|
+
|
|
712
973
|
return `<!DOCTYPE html>
|
|
713
974
|
<html lang="en">
|
|
714
975
|
<head>
|
|
715
976
|
<meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
716
|
-
<title>Backlist QA Report — ${id}</title>
|
|
977
|
+
<title>Backlist QA Report v${VERSION} — ${id}</title>
|
|
717
978
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.4.1/chart.umd.js"></script>
|
|
718
979
|
<style>
|
|
719
980
|
*{box-sizing:border-box;margin:0;padding:0}
|
|
720
981
|
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0a0a12;color:#e2e8f0;font-size:14px;line-height:1.6}
|
|
721
|
-
header{background
|
|
722
|
-
header h1{font-size:1.
|
|
982
|
+
header{background:#0f0f1e;border-bottom:1px solid #00f5ff22;padding:1.5rem 2rem;display:flex;align-items:center;justify-content:space-between}
|
|
983
|
+
header h1{font-size:1.25rem;font-weight:600;color:#00f5ff}
|
|
984
|
+
header .version{font-size:.75rem;color:#534AB7;padding:3px 10px;border:1px solid #534AB7;border-radius:20px}
|
|
985
|
+
header p{color:#64748b;font-size:.85rem;margin-top:4px}
|
|
723
986
|
.container{max-width:1200px;margin:0 auto;padding:2rem}
|
|
724
|
-
.metrics{display:grid;grid-template-columns:repeat(auto-fit,minmax(
|
|
725
|
-
.
|
|
726
|
-
.
|
|
727
|
-
.
|
|
728
|
-
.section{background:#1e1e30;border:1px solid #2d2d4e;border-radius:10px;padding:1.5rem;margin-bottom:1.
|
|
729
|
-
.section-title{font-size
|
|
730
|
-
|
|
987
|
+
.metrics{display:grid;grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:.75rem;margin-bottom:1.5rem}
|
|
988
|
+
.mc{background:#1e1e30;border:1px solid #2d2d4e;border-radius:10px;padding:1rem 1.25rem}
|
|
989
|
+
.ml{font-size:.7rem;color:#64748b;text-transform:uppercase;letter-spacing:.05em}
|
|
990
|
+
.mv{font-size:2rem;font-weight:700;margin-top:4px}
|
|
991
|
+
.section{background:#1e1e30;border:1px solid #2d2d4e;border-radius:10px;padding:1.5rem;margin-bottom:1.25rem}
|
|
992
|
+
.section-title{font-size:.95rem;font-weight:600;margin-bottom:1rem;color:#cbd5e1;border-bottom:1px solid #2d2d4e;padding-bottom:.75rem;display:flex;justify-content:space-between}
|
|
993
|
+
.grid2{display:grid;grid-template-columns:1fr 1fr;gap:1rem}
|
|
994
|
+
table{width:100%;border-collapse:collapse;font-size:.82rem}
|
|
731
995
|
th{text-align:left;color:#64748b;font-weight:500;padding:.5rem .75rem;border-bottom:1px solid #2d2d4e}
|
|
732
996
|
td{padding:.5rem .75rem;border-bottom:1px solid #1a1a2e;vertical-align:top}
|
|
733
|
-
tr.fail td{background:rgba(239,68,68,.
|
|
734
|
-
.status{display:inline-block;padding:2px 8px;border-radius:4px;font-size:.
|
|
997
|
+
tr.fail td{background:rgba(239,68,68,.04)}tr.flaky td{background:rgba(245,158,11,.04)}
|
|
998
|
+
.status{display:inline-block;padding:2px 8px;border-radius:4px;font-size:.72rem;font-weight:600}
|
|
735
999
|
.status-pass{background:#064e3b;color:#34d399}.status-fail{background:#450a0a;color:#f87171}
|
|
736
1000
|
.status-skip{background:#1e293b;color:#94a3b8}.status-flaky{background:#422006;color:#fbbf24}
|
|
737
|
-
.
|
|
1001
|
+
.sev-p0{background:#450a0a;color:#f87171;padding:2px 6px;border-radius:3px;font-size:.72rem;font-weight:700}
|
|
1002
|
+
.sev-p1{background:#422006;color:#fbbf24;padding:2px 6px;border-radius:3px;font-size:.72rem;font-weight:700}
|
|
1003
|
+
.sev-p2{background:#1e3a5f;color:#60a5fa;padding:2px 6px;border-radius:3px;font-size:.72rem}
|
|
1004
|
+
.sev-p3{background:#1e293b;color:#94a3b8;padding:2px 6px;border-radius:3px;font-size:.72rem}
|
|
1005
|
+
.err code{font-size:.72rem;color:#f87171;background:#1a0a0a;padding:2px 6px;border-radius:3px;word-break:break-all}
|
|
738
1006
|
.bug-card{border-radius:8px;padding:1rem;margin-bottom:.75rem;border-left:3px solid}
|
|
739
1007
|
.bug-p0{background:rgba(239,68,68,.08);border-color:#ef4444}
|
|
740
1008
|
.bug-p1{background:rgba(245,158,11,.08);border-color:#f59e0b}
|
|
741
1009
|
.bug-p2{background:rgba(96,165,250,.08);border-color:#60a5fa}
|
|
742
1010
|
.bug-p3{background:rgba(148,163,184,.08);border-color:#64748b}
|
|
743
1011
|
.bug-header{display:flex;gap:.75rem;align-items:center;margin-bottom:.5rem}
|
|
744
|
-
.bug-id{font-family:monospace;font-size:.
|
|
745
|
-
.bug-sev{font-size:.
|
|
746
|
-
.bug-st{font-size:.
|
|
747
|
-
.bug-title{font-weight:600;margin-bottom:.25rem
|
|
748
|
-
.
|
|
749
|
-
|
|
1012
|
+
.bug-id{font-family:monospace;font-size:.75rem;color:#64748b}
|
|
1013
|
+
.bug-sev{font-size:.72rem;font-weight:700;color:#f87171}
|
|
1014
|
+
.bug-st{font-size:.72rem;padding:2px 8px;border-radius:4px;background:#1e293b;color:#94a3b8}
|
|
1015
|
+
.bug-title{font-weight:600;margin-bottom:.25rem;font-size:.9rem}
|
|
1016
|
+
.bug-desc{font-size:.8rem;color:#94a3b8}
|
|
1017
|
+
.chart-wrap{position:relative;height:260px}
|
|
1018
|
+
footer{text-align:center;color:#334155;font-size:.72rem;padding:2rem;border-top:1px solid #1e293b;margin-top:2rem}
|
|
750
1019
|
</style>
|
|
751
1020
|
</head>
|
|
752
1021
|
<body>
|
|
753
1022
|
<header>
|
|
754
|
-
<
|
|
755
|
-
|
|
1023
|
+
<div>
|
|
1024
|
+
<h1>Backlist QA Report</h1>
|
|
1025
|
+
<p>Run ID: ${id} · ${new Date(startedAt).toLocaleString()} · ${formatDuration(duration)}</p>
|
|
1026
|
+
</div>
|
|
1027
|
+
<span class="version">v${VERSION}</span>
|
|
756
1028
|
</header>
|
|
757
1029
|
<div class="container">
|
|
1030
|
+
|
|
1031
|
+
${urlCards ? `<div class="section"><div class="section-title">Target URLs</div>${urlCards}</div>` : ''}
|
|
1032
|
+
|
|
758
1033
|
<div class="metrics">
|
|
759
|
-
<div class="
|
|
760
|
-
<div class="
|
|
761
|
-
<div class="
|
|
762
|
-
<div class="
|
|
763
|
-
<div class="
|
|
764
|
-
<div class="
|
|
1034
|
+
<div class="mc"><div class="ml">Pass Rate</div><div class="mv" style="color:${statusColor}">${passRate}%</div></div>
|
|
1035
|
+
<div class="mc"><div class="ml">Total</div><div class="mv">${summary.total}</div></div>
|
|
1036
|
+
<div class="mc"><div class="ml">Passed</div><div class="mv" style="color:#34d399">${summary.passed}</div></div>
|
|
1037
|
+
<div class="mc"><div class="ml">Failed</div><div class="mv" style="color:#f87171">${summary.failed}</div></div>
|
|
1038
|
+
<div class="mc"><div class="ml">Flaky</div><div class="mv" style="color:#fbbf24">${summary.flaky}</div></div>
|
|
1039
|
+
<div class="mc"><div class="ml">Bugs</div><div class="mv" style="color:#c084fc">${bugReports.length}</div></div>
|
|
1040
|
+
<div class="mc"><div class="ml">P0 Critical</div><div class="mv" style="color:#f87171;font-size:1.6rem">${sevCounts.P0}</div></div>
|
|
1041
|
+
<div class="mc"><div class="ml">P1 High</div><div class="mv" style="color:#fbbf24;font-size:1.6rem">${sevCounts.P1}</div></div>
|
|
765
1042
|
</div>
|
|
766
1043
|
|
|
767
|
-
<div class="
|
|
768
|
-
<div class="section
|
|
769
|
-
|
|
1044
|
+
<div class="grid2">
|
|
1045
|
+
<div class="section">
|
|
1046
|
+
<div class="section-title">Coverage by type</div>
|
|
1047
|
+
${covBars}
|
|
1048
|
+
</div>
|
|
1049
|
+
<div class="section">
|
|
1050
|
+
<div class="section-title">Pass vs Fail</div>
|
|
1051
|
+
<div class="chart-wrap"><canvas id="typeChart" role="img" aria-label="Pass vs fail by type"></canvas></div>
|
|
1052
|
+
</div>
|
|
770
1053
|
</div>
|
|
771
1054
|
|
|
772
1055
|
<div class="section">
|
|
773
|
-
<div class="section-title">
|
|
774
|
-
|
|
1056
|
+
<div class="section-title">Route Scan <span style="font-weight:400;font-size:.8rem;color:#64748b">${routeScans.length} routes probed</span></div>
|
|
1057
|
+
${routeCards}
|
|
775
1058
|
</div>
|
|
776
1059
|
|
|
777
1060
|
<div class="section">
|
|
778
|
-
<div class="section-title">Test Results
|
|
1061
|
+
<div class="section-title">Test Results <span style="font-weight:400;font-size:.8rem;color:#64748b">${results.length} tests</span></div>
|
|
779
1062
|
<table>
|
|
780
|
-
<thead><tr><th>Test</th><th>Type</th><th>Status</th><th>Duration</th><th>Retries</th><th>Error</th></tr></thead>
|
|
1063
|
+
<thead><tr><th>Test</th><th>Type</th><th>Status</th><th>Sev</th><th>Duration</th><th>Retries</th><th>Error</th></tr></thead>
|
|
781
1064
|
<tbody>${rows}</tbody>
|
|
782
1065
|
</table>
|
|
783
1066
|
</div>
|
|
784
1067
|
|
|
785
1068
|
<div class="section">
|
|
786
|
-
<div class="section-title">Bug Reports
|
|
1069
|
+
<div class="section-title">Bug Reports <span style="font-weight:400;font-size:.8rem;color:#64748b">${bugReports.length} bugs</span></div>
|
|
787
1070
|
${bugCards}
|
|
788
1071
|
</div>
|
|
789
1072
|
</div>
|
|
790
|
-
<footer>Generated by create-backlist
|
|
1073
|
+
<footer>Generated by create-backlist v${VERSION} — Backlist QA Platform · ${new Date().toLocaleString()}</footer>
|
|
791
1074
|
<script>
|
|
792
1075
|
new Chart(document.getElementById('typeChart'), {
|
|
793
1076
|
type: 'bar',
|
|
794
1077
|
data: {
|
|
795
1078
|
labels: ${chartLabels},
|
|
796
1079
|
datasets: [
|
|
797
|
-
{ label: 'Passed', data: ${chartPassed}, backgroundColor: '#34d399' },
|
|
798
|
-
{ label: 'Failed', data: ${chartFailed}, backgroundColor: '#f87171' },
|
|
1080
|
+
{ label: 'Passed', data: ${chartPassed}, backgroundColor: '#34d399', borderRadius: 4 },
|
|
1081
|
+
{ label: 'Failed', data: ${chartFailed}, backgroundColor: '#f87171', borderRadius: 4 },
|
|
799
1082
|
]
|
|
800
1083
|
},
|
|
801
1084
|
options: {
|
|
802
1085
|
responsive: true, maintainAspectRatio: false,
|
|
803
|
-
plugins: { legend: { labels: { color: '#94a3b8' } } },
|
|
1086
|
+
plugins: { legend: { labels: { color: '#94a3b8', font: { size: 12 } } } },
|
|
804
1087
|
scales: {
|
|
805
|
-
x: { ticks: { color: '#64748b' }, grid: { color: '#1e293b' } },
|
|
806
|
-
y: { ticks: { color: '#64748b', stepSize: 1 }, grid: { color: '#1e293b' } }
|
|
1088
|
+
x: { ticks: { color: '#64748b', font: { size: 11 } }, grid: { color: '#1e293b' } },
|
|
1089
|
+
y: { ticks: { color: '#64748b', stepSize: 1, font: { size: 11 } }, grid: { color: '#1e293b' } }
|
|
807
1090
|
}
|
|
808
1091
|
}
|
|
809
1092
|
});
|
|
@@ -812,8 +1095,7 @@ new Chart(document.getElementById('typeChart'), {
|
|
|
812
1095
|
</html>`;
|
|
813
1096
|
}
|
|
814
1097
|
|
|
815
|
-
// ── History helpers
|
|
816
|
-
|
|
1098
|
+
// ── History helpers ────────────────────────────────────────────────────────
|
|
817
1099
|
export async function initQASystem() {
|
|
818
1100
|
await fs.ensureDir(QA_DIR);
|
|
819
1101
|
await fs.ensureDir(REPORT_DIR);
|
|
@@ -828,7 +1110,7 @@ async function loadHistory() {
|
|
|
828
1110
|
}
|
|
829
1111
|
|
|
830
1112
|
async function saveRun(run) {
|
|
831
|
-
const hist
|
|
1113
|
+
const hist = await loadHistory();
|
|
832
1114
|
hist.runs.unshift(run);
|
|
833
1115
|
if (hist.runs.length > 50) hist.runs = hist.runs.slice(0, 50);
|
|
834
1116
|
await fs.writeJson(HISTORY_FILE, hist, { spaces: 2 });
|
|
@@ -843,7 +1125,7 @@ async function exportReport(run) {
|
|
|
843
1125
|
await fs.writeJson(jsonPath, run, { spaces: 2 });
|
|
844
1126
|
return htmlPath;
|
|
845
1127
|
} catch (err) {
|
|
846
|
-
console.error(chalk.gray(` [warn]
|
|
1128
|
+
console.error(chalk.gray(` [warn] Report write failed: ${err.message}`));
|
|
847
1129
|
return null;
|
|
848
1130
|
}
|
|
849
1131
|
}
|
|
@@ -862,8 +1144,30 @@ async function printRunDiff(currentRun) {
|
|
|
862
1144
|
} catch {}
|
|
863
1145
|
}
|
|
864
1146
|
|
|
1147
|
+
// ── Print summary ──────────────────────────────────────────────────────────
|
|
1148
|
+
function printResultsSummary(results) {
|
|
1149
|
+
const passed = results.filter(r => ['PASS','FLAKY'].includes(r.status)).length;
|
|
1150
|
+
const failed = results.filter(r => r.status === 'FAIL').length;
|
|
1151
|
+
const passRate = results.length ? Math.round((passed / results.length) * 100) : 0;
|
|
1152
|
+
|
|
1153
|
+
console.log('');
|
|
1154
|
+
console.log(chalk.hex('#00F5FF').bold(' ── Scan Results ──────────────────────────────────────'));
|
|
1155
|
+
console.log(` Pass rate: [${buildProgressBar(passRate, 24)}] ${chalk.white.bold(passRate + '%')}`);
|
|
1156
|
+
console.log(` ${chalk.green('✓')} ${passed} passed ${chalk.red('✗')} ${failed} failed (${results.length} total)`);
|
|
1157
|
+
if (failed > 0) {
|
|
1158
|
+
console.log('');
|
|
1159
|
+
console.log(chalk.red.bold(' Failures:'));
|
|
1160
|
+
results.filter(r => r.status === 'FAIL').forEach(f => {
|
|
1161
|
+
const sev = f.sev ? ` [${f.sev}]` : '';
|
|
1162
|
+
console.log(chalk.red(` ✗${sev} ${f.name}`));
|
|
1163
|
+
if (f.error) console.log(chalk.gray(` → ${f.error}`));
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1166
|
+
console.log('');
|
|
1167
|
+
}
|
|
1168
|
+
|
|
865
1169
|
// ─────────────────────────────────────────────────────────────────────────
|
|
866
|
-
// Manual QA Flow
|
|
1170
|
+
// Manual QA Flow (v9 retained + v10 URL option)
|
|
867
1171
|
// ─────────────────────────────────────────────────────────────────────────
|
|
868
1172
|
|
|
869
1173
|
export async function runManualQA() {
|
|
@@ -877,57 +1181,58 @@ export async function runManualQA() {
|
|
|
877
1181
|
const action = await p.select({
|
|
878
1182
|
message: 'Manual QA — what would you like to do?',
|
|
879
1183
|
options: [
|
|
880
|
-
{ value: '
|
|
881
|
-
{ value: '
|
|
1184
|
+
{ value: 'url-scan', label: '🌐 URL-Based Scan', hint: 'Enter URL(s) and run HTTP probe tests' },
|
|
1185
|
+
{ value: 'new-test', label: '✏️ Create & run a custom test' },
|
|
1186
|
+
{ value: 'full-scan', label: '🔬 Full system scan', hint: 'File-system + UI tests' },
|
|
882
1187
|
{ value: 'log-bug', label: '🐛 Log a bug report' },
|
|
883
|
-
{ value: 'run-suite', label: '▶️ Run saved test suite' },
|
|
884
|
-
{ value: 'ui-tests', label: '🖥️ Run UI/Frontend tests' },
|
|
885
1188
|
{ value: 'security-scan',label: '🛡️ Security scan only' },
|
|
1189
|
+
{ value: 'ui-tests', label: '🖥️ UI/Frontend tests' },
|
|
886
1190
|
],
|
|
887
1191
|
});
|
|
888
1192
|
if (p.isCancel(action)) { p.cancel('Cancelled.'); return; }
|
|
889
1193
|
|
|
890
1194
|
const dashboard = new LiveDashboard();
|
|
891
1195
|
|
|
892
|
-
if (action === '
|
|
1196
|
+
if (action === 'url-scan') {
|
|
1197
|
+
const localUrl = await p.text({ message: 'Localhost URL (leave blank to skip):', placeholder: 'http://localhost:3000' });
|
|
1198
|
+
const prodUrl = await p.text({ message: 'Production URL (leave blank to skip):', placeholder: 'https://yoursite.com' });
|
|
1199
|
+
if (p.isCancel(localUrl) || p.isCancel(prodUrl)) { p.cancel('Cancelled.'); return; }
|
|
1200
|
+
const run = await runUrlQA({
|
|
1201
|
+
localUrl : String(localUrl).trim() || undefined,
|
|
1202
|
+
prodUrl : String(prodUrl).trim() || undefined,
|
|
1203
|
+
});
|
|
1204
|
+
if (run) manualResults.push(...run.results);
|
|
1205
|
+
} else if (action === 'log-bug') {
|
|
893
1206
|
await logBugInteractive(bugs);
|
|
894
1207
|
} else if (action === 'new-test') {
|
|
895
1208
|
await createAndRunTestInteractive(runner, manualResults, dashboard);
|
|
896
1209
|
} else if (action === 'full-scan') {
|
|
897
1210
|
dashboard.start();
|
|
898
|
-
const
|
|
899
|
-
...buildFullSystemTests(),
|
|
900
|
-
...buildUITests(),
|
|
901
|
-
];
|
|
902
|
-
const results = await runner.run(allTests, dashboard);
|
|
1211
|
+
const results = await runner.run([...buildFullSystemTests(), ...buildUITests()], dashboard);
|
|
903
1212
|
manualResults.push(...results);
|
|
904
1213
|
dashboard.stop();
|
|
905
1214
|
printResultsSummary(results);
|
|
906
1215
|
} else if (action === 'ui-tests') {
|
|
907
1216
|
dashboard.start();
|
|
908
|
-
const
|
|
909
|
-
const results = await runner.run(uiTests, dashboard);
|
|
1217
|
+
const results = await runner.run(buildUITests(), dashboard);
|
|
910
1218
|
manualResults.push(...results);
|
|
911
1219
|
dashboard.stop();
|
|
912
1220
|
printResultsSummary(results);
|
|
913
1221
|
} else if (action === 'security-scan') {
|
|
914
1222
|
dashboard.start();
|
|
915
|
-
const
|
|
916
|
-
const results = await runner.run(secTests, dashboard);
|
|
1223
|
+
const results = await runner.run(buildFullSystemTests().filter(t => t.type === 'security' || t.type === 'auth'), dashboard);
|
|
917
1224
|
manualResults.push(...results);
|
|
918
1225
|
dashboard.stop();
|
|
919
1226
|
printResultsSummary(results);
|
|
920
|
-
} else if (action === 'run-suite') {
|
|
921
|
-
await runSavedSuiteInteractive(runner, manualResults, dashboard);
|
|
922
1227
|
}
|
|
923
1228
|
|
|
924
|
-
const continueLoop = await p.confirm({ message: 'Run another
|
|
1229
|
+
const continueLoop = await p.confirm({ message: 'Run another action?' });
|
|
925
1230
|
if (!p.isCancel(continueLoop) && continueLoop) return runManualQA();
|
|
926
1231
|
|
|
927
1232
|
const duration = Date.now() - new Date(startedAt).getTime();
|
|
928
1233
|
const summary = buildSummary(manualResults);
|
|
929
1234
|
const coverage = buildCoverageMatrix(manualResults);
|
|
930
|
-
const run = { id: runId, type: 'manual', startedAt, duration, results: manualResults, bugReports: bugs, summary, coverage };
|
|
1235
|
+
const run = { id: runId, type: 'manual', version: VERSION, startedAt, duration, results: manualResults, bugReports: bugs, summary, coverage };
|
|
931
1236
|
await saveRun(run);
|
|
932
1237
|
const reportFile = await exportReport(run);
|
|
933
1238
|
|
|
@@ -935,36 +1240,15 @@ export async function runManualQA() {
|
|
|
935
1240
|
if (reportFile) console.log(chalk.gray(` 📄 Report: ${reportFile}`));
|
|
936
1241
|
}
|
|
937
1242
|
|
|
938
|
-
function printResultsSummary(results) {
|
|
939
|
-
const passed = results.filter(r => ['PASS','FLAKY'].includes(r.status)).length;
|
|
940
|
-
const failed = results.filter(r => r.status === 'FAIL').length;
|
|
941
|
-
const passRate = results.length ? Math.round((passed / results.length) * 100) : 0;
|
|
942
|
-
|
|
943
|
-
console.log('');
|
|
944
|
-
console.log(chalk.hex('#00F5FF').bold(' ── Scan Results ──────────────────────────────────────'));
|
|
945
|
-
console.log(` Pass rate: [${buildProgressBar(passRate, 24)}] ${chalk.white.bold(passRate + '%')}`);
|
|
946
|
-
console.log(` ${chalk.green('✓')} ${passed} passed ${chalk.red('✗')} ${failed} failed (${results.length} total)`);
|
|
947
|
-
if (failed > 0) {
|
|
948
|
-
console.log('');
|
|
949
|
-
console.log(chalk.red.bold(' Failures:'));
|
|
950
|
-
results.filter(r => r.status === 'FAIL').forEach(f => {
|
|
951
|
-
console.log(chalk.red(` ✗ ${f.name}`));
|
|
952
|
-
if (f.error) console.log(chalk.gray(` → ${f.error}`));
|
|
953
|
-
});
|
|
954
|
-
}
|
|
955
|
-
console.log('');
|
|
956
|
-
}
|
|
957
|
-
|
|
958
1243
|
async function logBugInteractive(bugs) {
|
|
959
|
-
const title
|
|
1244
|
+
const title = await p.text({ message: 'Bug title:' });
|
|
960
1245
|
if (p.isCancel(title)) return;
|
|
961
|
-
const severity = await p.select({
|
|
962
|
-
|
|
963
|
-
options: Object.entries(SEVERITY_LEVELS).map(([k, v]) => ({ value: k, label: `${k} — ${v}` })),
|
|
964
|
-
});
|
|
1246
|
+
const severity = await p.select({ message: 'Severity:',
|
|
1247
|
+
options: Object.entries(SEVERITY_LEVELS).map(([k, v]) => ({ value: k, label: `${k} — ${v}` })) });
|
|
965
1248
|
if (p.isCancel(severity)) return;
|
|
966
1249
|
const description = await p.text({ message: 'Description (optional):', placeholder: 'Steps to reproduce…' });
|
|
967
|
-
bugs.push({ id: `BUG-${shortId()}`, title: String(title), severity: String(severity), status: 'OPEN',
|
|
1250
|
+
bugs.push({ id: `BUG-${shortId()}`, title: String(title), severity: String(severity), status: 'OPEN',
|
|
1251
|
+
description: p.isCancel(description) ? '' : description, createdAt: timestamp() });
|
|
968
1252
|
console.log(chalk.green(` ✓ Bug logged as ${colorSeverity(String(severity))}`));
|
|
969
1253
|
}
|
|
970
1254
|
|
|
@@ -976,47 +1260,31 @@ async function createAndRunTestInteractive(runner, results, dashboard) {
|
|
|
976
1260
|
const expectPass = await p.confirm({ message: 'Should this test pass?' });
|
|
977
1261
|
|
|
978
1262
|
dashboard.start();
|
|
979
|
-
const
|
|
980
|
-
id: shortId(), name: String(name), type: String(type),
|
|
1263
|
+
const [result] = await runner.run([{
|
|
1264
|
+
id: shortId(), name: String(name), type: String(type), sev: 'P3',
|
|
981
1265
|
fn: async () => {
|
|
982
1266
|
await sleep(400 + Math.random() * 300);
|
|
983
1267
|
if (!expectPass) throw new Error('Test manually marked as failure');
|
|
984
1268
|
},
|
|
985
|
-
};
|
|
986
|
-
const [result] = await runner.run([test], dashboard);
|
|
1269
|
+
}], dashboard);
|
|
987
1270
|
results.push(result);
|
|
988
1271
|
dashboard.stop();
|
|
989
1272
|
console.log(` ${colorStatus(result.status)} ${result.name} ${chalk.gray(formatDuration(result.duration))}`);
|
|
990
1273
|
}
|
|
991
1274
|
|
|
992
|
-
async function runSavedSuiteInteractive(runner, results, dashboard) {
|
|
993
|
-
const suiteFiles = await fs.readdir(QA_DIR).then(files => files.filter(f => f.endsWith('.suite.json'))).catch(() => []);
|
|
994
|
-
if (!suiteFiles.length) { console.log(chalk.yellow(' No saved suites found.')); return; }
|
|
995
|
-
const chosen = await p.select({ message: 'Select suite:', options: suiteFiles.map(f => ({ value: f, label: f })) });
|
|
996
|
-
if (p.isCancel(chosen)) return;
|
|
997
|
-
const suite = await fs.readJson(path.join(QA_DIR, String(chosen)));
|
|
998
|
-
const tests = (suite.tests ?? []).map(t => ({ ...t, fn: async () => { await sleep(200); if (t.shouldFail) throw new Error('Marked as expected failure'); } }));
|
|
999
|
-
dashboard.start();
|
|
1000
|
-
const runResults = await runner.run(tests, dashboard);
|
|
1001
|
-
results.push(...runResults);
|
|
1002
|
-
dashboard.stop();
|
|
1003
|
-
printResultsSummary(runResults);
|
|
1004
|
-
}
|
|
1005
|
-
|
|
1006
1275
|
// ─────────────────────────────────────────────────────────────────────────
|
|
1007
|
-
// Automated QA Flow
|
|
1276
|
+
// Automated QA Flow (v9 retained + v10 URL integration)
|
|
1008
1277
|
// ─────────────────────────────────────────────────────────────────────────
|
|
1009
1278
|
|
|
1010
|
-
export async function runAutomatedQA({ continuous = false } = {}) {
|
|
1279
|
+
export async function runAutomatedQA({ continuous = false, localUrl, prodUrl } = {}) {
|
|
1011
1280
|
const runOnce = async () => {
|
|
1012
1281
|
const runId = `AQA-${shortId()}`;
|
|
1013
1282
|
const startedAt = timestamp();
|
|
1014
1283
|
|
|
1015
1284
|
console.log('');
|
|
1016
|
-
console.log(chalk.hex('#BF40FF').bold(` ── 🤖 Automated QA Run ${runId} ──`));
|
|
1285
|
+
console.log(chalk.hex('#BF40FF').bold(` ── 🤖 Automated QA v${VERSION} — Run ${runId} ──`));
|
|
1017
1286
|
console.log('');
|
|
1018
1287
|
|
|
1019
|
-
// Try to get endpoints from analyzer
|
|
1020
1288
|
let endpoints = [];
|
|
1021
1289
|
try {
|
|
1022
1290
|
const { analyzeFrontend } = await import('../analyzer.js');
|
|
@@ -1029,7 +1297,7 @@ export async function runAutomatedQA({ continuous = false } = {}) {
|
|
|
1029
1297
|
...buildUITests(),
|
|
1030
1298
|
];
|
|
1031
1299
|
|
|
1032
|
-
console.log(chalk.gray(`
|
|
1300
|
+
console.log(chalk.gray(` Test suite: ${allTests.length} tests across ${new Set(allTests.map(t => t.type)).size} categories\n`));
|
|
1033
1301
|
|
|
1034
1302
|
const dashboard = new LiveDashboard();
|
|
1035
1303
|
const runner = new TestRunner();
|
|
@@ -1038,12 +1306,9 @@ export async function runAutomatedQA({ continuous = false } = {}) {
|
|
|
1038
1306
|
runner.on('result', r => {
|
|
1039
1307
|
if (r.status === 'FAIL') {
|
|
1040
1308
|
autoBugs.push({
|
|
1041
|
-
id
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
status : 'OPEN',
|
|
1045
|
-
description: r.error || '',
|
|
1046
|
-
createdAt: timestamp(),
|
|
1309
|
+
id: `AUTO-${shortId()}`, title: `Automated: ${r.name}`,
|
|
1310
|
+
severity : r.sev || (r.type === 'security' || r.type === 'auth' ? 'P0' : r.type === 'e2e' ? 'P1' : 'P2'),
|
|
1311
|
+
status : 'OPEN', description: r.error || '', createdAt: timestamp(),
|
|
1047
1312
|
});
|
|
1048
1313
|
}
|
|
1049
1314
|
});
|
|
@@ -1052,13 +1317,21 @@ export async function runAutomatedQA({ continuous = false } = {}) {
|
|
|
1052
1317
|
const results = await runner.run(allTests, dashboard);
|
|
1053
1318
|
dashboard.stop();
|
|
1054
1319
|
|
|
1320
|
+
// If URLs provided, run URL-based QA too
|
|
1321
|
+
if (localUrl || prodUrl) {
|
|
1322
|
+
const urlRun = await runUrlQA({ localUrl, prodUrl, silent: true });
|
|
1323
|
+
if (urlRun) { results.push(...urlRun.results); autoBugs.push(...urlRun.bugReports); }
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1055
1326
|
const duration = Date.now() - new Date(startedAt).getTime();
|
|
1056
1327
|
const summary = buildSummary(results);
|
|
1057
1328
|
const coverage = buildCoverageMatrix(results);
|
|
1058
1329
|
|
|
1059
1330
|
printResultsSummary(results);
|
|
1060
1331
|
|
|
1061
|
-
const run = { id: runId, type: 'automated',
|
|
1332
|
+
const run = { id: runId, type: 'automated', version: VERSION, startedAt, duration,
|
|
1333
|
+
results, bugReports: autoBugs, summary, coverage,
|
|
1334
|
+
urls: [localUrl, prodUrl].filter(Boolean).map((u, i) => ({ label: i === 0 ? 'localhost' : 'production', url: u })) };
|
|
1062
1335
|
await saveRun(run);
|
|
1063
1336
|
const reportFile = await exportReport(run);
|
|
1064
1337
|
|
|
@@ -1071,7 +1344,7 @@ export async function runAutomatedQA({ continuous = false } = {}) {
|
|
|
1071
1344
|
|
|
1072
1345
|
if (!continuous) { await runOnce(); return; }
|
|
1073
1346
|
|
|
1074
|
-
console.log(chalk.cyan(` ⚡ Continuous
|
|
1347
|
+
console.log(chalk.cyan(` ⚡ Continuous mode — reruns every ${WATCH_INTERVAL_MS / 1000}s. Ctrl+C to stop.\n`));
|
|
1075
1348
|
let iteration = 0;
|
|
1076
1349
|
while (true) {
|
|
1077
1350
|
iteration++;
|
|
@@ -1081,12 +1354,11 @@ export async function runAutomatedQA({ continuous = false } = {}) {
|
|
|
1081
1354
|
}
|
|
1082
1355
|
}
|
|
1083
1356
|
|
|
1084
|
-
// ──
|
|
1085
|
-
|
|
1357
|
+
// ── Post-gen auto-run ──────────────────────────────────────────────────────
|
|
1086
1358
|
export async function autoRunPostGeneration(options = {}) {
|
|
1087
1359
|
console.log('');
|
|
1088
|
-
console.log(chalk.hex('#00F5FF').bold(
|
|
1089
|
-
console.log(chalk.gray(`
|
|
1360
|
+
console.log(chalk.hex('#00F5FF').bold(` ── 🔬 Post-Generation QA Scan v${VERSION} ──────────────`));
|
|
1361
|
+
console.log(chalk.gray(` Validating: ${options.projectName || 'backend'}`));
|
|
1090
1362
|
console.log('');
|
|
1091
1363
|
|
|
1092
1364
|
const projectDir = options.projectDir || process.cwd();
|
|
@@ -1097,11 +1369,9 @@ export async function autoRunPostGeneration(options = {}) {
|
|
|
1097
1369
|
|
|
1098
1370
|
runner.on('result', r => {
|
|
1099
1371
|
if (r.status === 'FAIL') {
|
|
1100
|
-
autoBugs.push({
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
status: 'OPEN', description: r.error || '', createdAt: timestamp(),
|
|
1104
|
-
});
|
|
1372
|
+
autoBugs.push({ id: `POST-${shortId()}`, title: r.name,
|
|
1373
|
+
severity: r.sev || (r.type === 'security' ? 'P0' : 'P2'),
|
|
1374
|
+
status: 'OPEN', description: r.error || '', createdAt: timestamp() });
|
|
1105
1375
|
}
|
|
1106
1376
|
});
|
|
1107
1377
|
|
|
@@ -1111,23 +1381,15 @@ export async function autoRunPostGeneration(options = {}) {
|
|
|
1111
1381
|
|
|
1112
1382
|
const summary = buildSummary(results);
|
|
1113
1383
|
const coverage = buildCoverageMatrix(results);
|
|
1114
|
-
const run = {
|
|
1115
|
-
|
|
1116
|
-
type : 'post-generation',
|
|
1117
|
-
startedAt: timestamp(),
|
|
1118
|
-
duration : 0,
|
|
1119
|
-
results,
|
|
1120
|
-
bugReports: autoBugs,
|
|
1121
|
-
summary,
|
|
1122
|
-
coverage,
|
|
1123
|
-
};
|
|
1384
|
+
const run = { id: `POST-${shortId()}`, type: 'post-generation', version: VERSION,
|
|
1385
|
+
startedAt: timestamp(), duration: 0, results, bugReports: autoBugs, summary, coverage };
|
|
1124
1386
|
|
|
1125
1387
|
await saveRun(run);
|
|
1126
1388
|
const reportFile = await exportReport(run);
|
|
1127
|
-
|
|
1128
1389
|
printResultsSummary(results);
|
|
1390
|
+
|
|
1129
1391
|
if (autoBugs.length > 0) {
|
|
1130
|
-
console.log(chalk.red.bold(` ⚠ ${autoBugs.length} issue(s)
|
|
1392
|
+
console.log(chalk.red.bold(` ⚠ ${autoBugs.length} issue(s) detected:`));
|
|
1131
1393
|
autoBugs.forEach(b => console.log(chalk.red(` ${colorSeverity(b.severity)} ${b.title}`)));
|
|
1132
1394
|
console.log('');
|
|
1133
1395
|
}
|
|
@@ -1135,23 +1397,21 @@ export async function autoRunPostGeneration(options = {}) {
|
|
|
1135
1397
|
}
|
|
1136
1398
|
|
|
1137
1399
|
// ── QA History ─────────────────────────────────────────────────────────────
|
|
1138
|
-
|
|
1139
1400
|
export async function viewQAHistory() {
|
|
1140
1401
|
const hist = await loadHistory();
|
|
1141
1402
|
if (!hist.runs.length) { console.log(chalk.yellow('\n No QA history found.\n')); return; }
|
|
1142
1403
|
|
|
1143
1404
|
console.log('');
|
|
1144
1405
|
console.log(chalk.hex('#00F5FF').bold(' QA History (most recent first)'));
|
|
1145
|
-
console.log(chalk.gray('
|
|
1406
|
+
console.log(chalk.gray(' ─────────────────────────────────────────────────────'));
|
|
1146
1407
|
|
|
1147
1408
|
for (const run of hist.runs.slice(0, 10)) {
|
|
1148
|
-
const passRate
|
|
1409
|
+
const passRate = run.summary.total ? ((run.summary.passed / run.summary.total) * 100).toFixed(0) : '–';
|
|
1149
1410
|
const rateColor = Number(passRate) >= 90 ? chalk.green : Number(passRate) >= 70 ? chalk.yellow : chalk.red;
|
|
1411
|
+
const ver = run.version ? chalk.dim(`v${run.version}`) : '';
|
|
1150
1412
|
console.log(
|
|
1151
|
-
` ${chalk.gray(run.id.padEnd(18))}` +
|
|
1152
|
-
` ${chalk.gray(
|
|
1153
|
-
` ${rateColor(`${passRate}%`.padStart(5))}` +
|
|
1154
|
-
` ${chalk.gray(`${run.summary.total} tests · ${formatDuration(run.duration)}`)}`
|
|
1413
|
+
` ${chalk.gray(run.id.padEnd(18))} ${chalk.gray(new Date(run.startedAt).toLocaleString().padEnd(24))}` +
|
|
1414
|
+
` ${rateColor(`${passRate}%`.padStart(5))} ${chalk.gray(`${run.summary.total} tests`)} ${ver}`
|
|
1155
1415
|
);
|
|
1156
1416
|
}
|
|
1157
1417
|
console.log('');
|
|
@@ -1169,8 +1429,11 @@ export async function viewQAHistory() {
|
|
|
1169
1429
|
if (!run) return;
|
|
1170
1430
|
|
|
1171
1431
|
console.log('');
|
|
1172
|
-
console.log(chalk.bold(` Run: ${run.id} (${run.type})`));
|
|
1432
|
+
console.log(chalk.bold(` Run: ${run.id} (${run.type}) ${run.version ? `v${run.version}` : ''}`));
|
|
1173
1433
|
console.log(chalk.gray(` ${new Date(run.startedAt).toLocaleString()} · ${formatDuration(run.duration)}`));
|
|
1434
|
+
if (run.urls?.length) {
|
|
1435
|
+
console.log(chalk.gray(` URLs: ${run.urls.map(u => u.url).join(', ')}`));
|
|
1436
|
+
}
|
|
1174
1437
|
console.log('');
|
|
1175
1438
|
for (const r of run.results) {
|
|
1176
1439
|
console.log(` ${colorStatus(r.status)} ${r.name} ${chalk.gray(formatDuration(r.duration))}`);
|