create-backlist 7.4.0 → 9.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/index.js +629 -212
- package/bin/qa.js +157 -69
- package/package.json +25 -20
- package/src/ai-agent.js +581 -124
- package/src/analyzer.js +661 -522
- package/src/qa/qa-engine.js +1068 -790
package/src/qa/qa-engine.js
CHANGED
|
@@ -1,909 +1,1187 @@
|
|
|
1
1
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
2
|
-
// Backlist QA Engine
|
|
2
|
+
// Backlist QA Engine — qa-engine.js v9.0
|
|
3
|
+
// Full live QA runtime: manual + automated + real-time dashboard
|
|
3
4
|
// Copyright (c) W.A.H.ISHAN — MIT License
|
|
5
|
+
//
|
|
6
|
+
// NEW in v9.0:
|
|
7
|
+
// ✦ Real-time terminal dashboard with live metrics streaming
|
|
8
|
+
// ✦ End-to-end test orchestration across all modules/pages/APIs
|
|
9
|
+
// ✦ Live UI interaction simulation engine
|
|
10
|
+
// ✦ Backend validation suite (schema, auth, CORS, rate-limit)
|
|
11
|
+
// ✦ Performance benchmarking (p50/p95/p99 latency, throughput)
|
|
12
|
+
// ✦ Security scanner (JWT, SQL injection, XSS surface, OWASP top-10)
|
|
13
|
+
// ✦ Continuous watch mode with file-change-triggered reruns
|
|
14
|
+
// ✦ Bug severity classification P0–P3 with auto-triage
|
|
15
|
+
// ✦ Flaky test detector with configurable retry budget
|
|
16
|
+
// ✦ Rich HTML + JSON report with embedded charts
|
|
17
|
+
// ✦ QA run diffing (vs previous run)
|
|
18
|
+
// ✦ Post-generation auto-run hook
|
|
4
19
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
5
20
|
|
|
6
|
-
import * as p
|
|
7
|
-
import chalk
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
import
|
|
11
|
-
import
|
|
12
|
-
import { performance }
|
|
13
|
-
|
|
14
|
-
// ──
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
export async function initQASystem() {
|
|
46
|
-
await fs.ensureDir(QA_DIR);
|
|
47
|
-
await fs.ensureDir(QA_REPORTS_DIR);
|
|
48
|
-
if (!(await fs.pathExists(QA_HISTORY_PATH))) {
|
|
49
|
-
await fs.writeJson(QA_HISTORY_PATH, { runs: [] }, { spaces: 2 });
|
|
50
|
-
}
|
|
21
|
+
import * as p from '@clack/prompts';
|
|
22
|
+
import chalk from 'chalk';
|
|
23
|
+
import fs from 'fs-extra';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
import os from 'node:os';
|
|
26
|
+
import { EventEmitter } from 'node:events';
|
|
27
|
+
import { performance } from 'node:perf_hooks';
|
|
28
|
+
|
|
29
|
+
// ── Constants ─────────────────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
const QA_DIR = path.join(process.cwd(), '.backlist', 'qa');
|
|
32
|
+
const HISTORY_FILE = path.join(QA_DIR, 'history.json');
|
|
33
|
+
const REPORT_DIR = path.join(QA_DIR, 'reports');
|
|
34
|
+
|
|
35
|
+
const SEVERITY_LEVELS = { P0: 'Critical', P1: 'High', P2: 'Medium', P3: 'Low' };
|
|
36
|
+
const TEST_TYPES = ['happy-path', 'validation', 'auth', 'edge-case', 'performance', 'security', 'e2e', 'ui'];
|
|
37
|
+
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
38
|
+
const FLAKY_RETRY_COUNT = 2;
|
|
39
|
+
const WATCH_INTERVAL_MS = 30_000;
|
|
40
|
+
|
|
41
|
+
// ── ANSI escape helpers for live dashboard ────────────────────────────────
|
|
42
|
+
|
|
43
|
+
const ESC = '\x1b[';
|
|
44
|
+
const CLEAR_LINE = ESC + '2K\r';
|
|
45
|
+
const CURSOR_UP = (n) => ESC + `${n}A`;
|
|
46
|
+
const CURSOR_HIDE = ESC + '?25l';
|
|
47
|
+
const CURSOR_SHOW = ESC + '?25h';
|
|
48
|
+
const BOLD = chalk.bold;
|
|
49
|
+
const DIM = chalk.dim;
|
|
50
|
+
|
|
51
|
+
// ── Utilities ─────────────────────────────────────────────────────────────
|
|
52
|
+
|
|
53
|
+
function timestamp() { return new Date().toISOString(); }
|
|
54
|
+
function shortId() { return Math.random().toString(36).slice(2, 9); }
|
|
55
|
+
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
|
|
56
|
+
function pluralize(n, w) { return `${n} ${n === 1 ? w : w + 's'}`; }
|
|
57
|
+
|
|
58
|
+
function colorSeverity(sev) {
|
|
59
|
+
return ({ P0: chalk.red.bold, P1: chalk.yellow.bold, P2: chalk.cyan, P3: chalk.gray }[sev] ?? chalk.white)(sev);
|
|
51
60
|
}
|
|
52
61
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
62
|
+
function colorStatus(status) {
|
|
63
|
+
return ({
|
|
64
|
+
PASS : chalk.green('✓ PASS'),
|
|
65
|
+
FAIL : chalk.red('✗ FAIL'),
|
|
66
|
+
SKIP : chalk.gray('⊘ SKIP'),
|
|
67
|
+
FLAKY : chalk.yellow('⚠ FLAKY'),
|
|
68
|
+
RUN : chalk.cyan('⟳ RUN'),
|
|
69
|
+
})[status] ?? status;
|
|
60
70
|
}
|
|
61
71
|
|
|
62
|
-
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
72
|
+
function buildProgressBar(pct, width = 20) {
|
|
73
|
+
const filled = Math.min(Math.round((pct / 100) * width), width);
|
|
74
|
+
const empty = width - filled;
|
|
75
|
+
const color = pct >= 90 ? chalk.green : pct >= 70 ? chalk.yellow : chalk.red;
|
|
76
|
+
return color('█'.repeat(filled)) + chalk.gray('░'.repeat(empty));
|
|
67
77
|
}
|
|
68
78
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
export function generateBugReport({ title, steps, expected, actual, severity, logs = [], environment = {} }) {
|
|
74
|
-
const id = `BUG-${Date.now().toString(36).toUpperCase()}`;
|
|
75
|
-
const timestamp = new Date().toISOString();
|
|
76
|
-
|
|
77
|
-
const env = {
|
|
78
|
-
os: process.platform,
|
|
79
|
-
node: process.version,
|
|
80
|
-
arch: process.arch,
|
|
81
|
-
cwd: process.cwd(),
|
|
82
|
-
...environment,
|
|
83
|
-
};
|
|
79
|
+
function formatDuration(ms) {
|
|
80
|
+
if (ms < 1000) return `${ms}ms`;
|
|
81
|
+
return `${(ms / 1000).toFixed(2)}s`;
|
|
82
|
+
}
|
|
84
83
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
severity,
|
|
90
|
-
steps,
|
|
91
|
-
expected,
|
|
92
|
-
actual,
|
|
93
|
-
environment: env,
|
|
94
|
-
logs,
|
|
95
|
-
status: 'open',
|
|
96
|
-
};
|
|
84
|
+
function formatBytes(b) {
|
|
85
|
+
if (b < 1024) return `${b}B`;
|
|
86
|
+
if (b < 1024 * 1024) return `${(b / 1024).toFixed(1)}KB`;
|
|
87
|
+
return `${(b / 1024 / 1024).toFixed(1)}MB`;
|
|
97
88
|
}
|
|
98
89
|
|
|
99
|
-
|
|
100
|
-
const sevColor = SEVERITY_COLORS[report.severity] || chalk.white;
|
|
101
|
-
const sevIcon = SEVERITY_ICONS[report.severity] || '⚠️';
|
|
90
|
+
// ── System info ────────────────────────────────────────────────────────────
|
|
102
91
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
console.log(` ${chalk.dim('Timestamp:')} ${chalk.gray(report.timestamp)}`);
|
|
112
|
-
console.log('');
|
|
113
|
-
console.log(` ${chalk.hex('#00F5FF').bold('📋 Steps to Reproduce:')}`);
|
|
114
|
-
report.steps.forEach((step, i) => {
|
|
115
|
-
console.log(` ${chalk.dim(`${i + 1}.`)} ${chalk.white(step)}`);
|
|
116
|
-
});
|
|
117
|
-
console.log('');
|
|
118
|
-
console.log(` ${chalk.green.bold('✅ Expected Result:')}`);
|
|
119
|
-
console.log(` ${chalk.white(report.expected)}`);
|
|
120
|
-
console.log('');
|
|
121
|
-
console.log(` ${chalk.red.bold('❌ Actual Result:')}`);
|
|
122
|
-
console.log(` ${chalk.white(report.actual)}`);
|
|
123
|
-
console.log('');
|
|
124
|
-
console.log(` ${chalk.hex('#BF40FF').bold('🖥️ Environment:')}`);
|
|
125
|
-
Object.entries(report.environment).forEach(([k, v]) => {
|
|
126
|
-
console.log(` ${chalk.dim(k + ':')} ${chalk.white(v)}`);
|
|
127
|
-
});
|
|
92
|
+
function getSystemStats() {
|
|
93
|
+
const mem = process.memoryUsage();
|
|
94
|
+
const heapMB = (mem.heapUsed / 1024 / 1024).toFixed(1);
|
|
95
|
+
const rss = formatBytes(mem.rss);
|
|
96
|
+
const uptime = process.uptime().toFixed(1);
|
|
97
|
+
const cpuUser = process.cpuUsage().user;
|
|
98
|
+
return { heapMB, rss, uptime, cpuUser };
|
|
99
|
+
}
|
|
128
100
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
101
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
102
|
+
// Live Dashboard Renderer
|
|
103
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
104
|
+
|
|
105
|
+
class LiveDashboard {
|
|
106
|
+
#lines = 0;
|
|
107
|
+
#active = false;
|
|
108
|
+
#startTime = Date.now();
|
|
109
|
+
#lastResults = [];
|
|
110
|
+
#runningTest = null;
|
|
111
|
+
#bugs = [];
|
|
112
|
+
#log = [];
|
|
113
|
+
|
|
114
|
+
start() {
|
|
115
|
+
this.#active = true;
|
|
116
|
+
this.#startTime = Date.now();
|
|
117
|
+
process.stdout.write(CURSOR_HIDE);
|
|
118
|
+
this.render({});
|
|
135
119
|
}
|
|
136
|
-
console.log('');
|
|
137
|
-
}
|
|
138
120
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
121
|
+
stop() {
|
|
122
|
+
this.#active = false;
|
|
123
|
+
process.stdout.write(CURSOR_SHOW);
|
|
124
|
+
this.#clearLines();
|
|
125
|
+
}
|
|
142
126
|
|
|
143
|
-
|
|
144
|
-
|
|
127
|
+
updateRunning(name) { this.#runningTest = name; }
|
|
128
|
+
addResult(r) { this.#lastResults.push(r); this.#runningTest = null; }
|
|
129
|
+
addBug(b) { this.#bugs.push(b); }
|
|
130
|
+
addLog(msg) { this.#log.push(`${DIM(new Date().toLocaleTimeString())} ${msg}`); if (this.#log.length > 8) this.#log.shift(); }
|
|
145
131
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
console.log(chalk.hex('#00F5FF').bold(' ╚══════════════════════════════════════════════════════════╝'));
|
|
150
|
-
console.log('');
|
|
151
|
-
console.log(chalk.gray(' Create, execute, and track test cases interactively.'));
|
|
152
|
-
console.log(chalk.gray(' Failed tests auto-generate professional bug reports.'));
|
|
153
|
-
console.log('');
|
|
132
|
+
render(summary) {
|
|
133
|
+
if (!this.#active) return;
|
|
134
|
+
this.#clearLines();
|
|
154
135
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
136
|
+
const lines = this.#buildLines(summary);
|
|
137
|
+
this.#lines = lines.length;
|
|
138
|
+
process.stdout.write(lines.join('\n') + '\n');
|
|
139
|
+
}
|
|
159
140
|
|
|
160
|
-
|
|
141
|
+
#clearLines() {
|
|
142
|
+
if (this.#lines > 0) {
|
|
143
|
+
process.stdout.write(CURSOR_UP(this.#lines) + CLEAR_LINE);
|
|
144
|
+
for (let i = 1; i < this.#lines; i++) {
|
|
145
|
+
process.stdout.write('\n' + CLEAR_LINE);
|
|
146
|
+
}
|
|
147
|
+
process.stdout.write(CURSOR_UP(this.#lines - 1));
|
|
148
|
+
}
|
|
149
|
+
}
|
|
161
150
|
|
|
162
|
-
|
|
163
|
-
|
|
151
|
+
#buildLines(summary = {}) {
|
|
152
|
+
const elapsed = ((Date.now() - this.#startTime) / 1000).toFixed(1);
|
|
153
|
+
const sys = getSystemStats();
|
|
154
|
+
const results = this.#lastResults;
|
|
155
|
+
const total = results.length;
|
|
156
|
+
const passed = results.filter(r => ['PASS','FLAKY'].includes(r.status)).length;
|
|
157
|
+
const failed = results.filter(r => r.status === 'FAIL').length;
|
|
158
|
+
const flaky = results.filter(r => r.status === 'FLAKY').length;
|
|
159
|
+
const passRate = total > 0 ? Math.round((passed / total) * 100) : 0;
|
|
160
|
+
|
|
161
|
+
const lines = [];
|
|
162
|
+
const w = Math.min(process.stdout.columns || 80, 88);
|
|
163
|
+
const bar = '─'.repeat(w - 2);
|
|
164
|
+
|
|
165
|
+
// Header
|
|
166
|
+
lines.push(chalk.hex('#00F5FF').bold(`┌${bar}┐`));
|
|
167
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.hex('#BF40FF').bold(' ⚡ BACKLIST LIVE QA DASHBOARD v9.0'.padEnd(w - 2)) + chalk.hex('#00F5FF').bold('│'));
|
|
168
|
+
lines.push(chalk.hex('#00F5FF').bold(`├${bar}┤`));
|
|
169
|
+
|
|
170
|
+
// Metrics row
|
|
171
|
+
const metrics = [
|
|
172
|
+
`${chalk.green('✓')} ${chalk.white.bold(passed)} passed`,
|
|
173
|
+
`${chalk.red('✗')} ${chalk.white.bold(failed)} failed`,
|
|
174
|
+
`${chalk.yellow('⚠')} ${chalk.white.bold(flaky)} flaky`,
|
|
175
|
+
`${chalk.cyan('🐛')} ${chalk.white.bold(this.#bugs.length)} bugs`,
|
|
176
|
+
`${chalk.gray('⏱')} ${chalk.white(elapsed + 's')}`,
|
|
177
|
+
].map(m => m.padEnd(20)).join(' ');
|
|
178
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + ' ' + metrics.slice(0, w - 4) + chalk.hex('#00F5FF').bold('│'));
|
|
179
|
+
|
|
180
|
+
// Progress bar
|
|
181
|
+
const pBar = buildProgressBar(passRate, 30);
|
|
182
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + ` Pass rate [${pBar}] ${chalk.white.bold(passRate + '%')} (${total} tests)`.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
183
|
+
|
|
184
|
+
// System health
|
|
185
|
+
const sysLine = ` ${DIM('Heap')} ${chalk.white(sys.heapMB + 'MB')} ${DIM('RSS')} ${chalk.white(sys.rss)} ${DIM('Uptime')} ${chalk.white(sys.uptime + 's')} ${DIM('Node')} ${chalk.white(process.version)}`;
|
|
186
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + sysLine.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
187
|
+
|
|
188
|
+
lines.push(chalk.hex('#00F5FF').bold(`├${'─'.repeat(w - 2)}┤`));
|
|
189
|
+
|
|
190
|
+
// Currently running
|
|
191
|
+
const runLine = this.#runningTest
|
|
192
|
+
? ` ${chalk.cyan('⟳')} ${chalk.cyan('Running:')} ${chalk.white(this.#runningTest.slice(0, w - 16))}`
|
|
193
|
+
: ` ${chalk.gray('⊘ Idle — waiting for next test...')}`;
|
|
194
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + runLine.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
195
|
+
|
|
196
|
+
lines.push(chalk.hex('#00F5FF').bold(`├${'─'.repeat(w - 2)}┤`));
|
|
197
|
+
|
|
198
|
+
// Last 5 results
|
|
199
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.gray(' Recent results:').padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
200
|
+
const recentResults = results.slice(-5);
|
|
201
|
+
for (const r of recentResults) {
|
|
202
|
+
const type = chalk.gray(`[${(r.type || '').padEnd(11)}]`);
|
|
203
|
+
const dur = chalk.gray(formatDuration(r.duration));
|
|
204
|
+
const name = r.name.slice(0, w - 40);
|
|
205
|
+
const row = ` ${colorStatus(r.status)} ${type} ${chalk.white(name)} ${dur}`;
|
|
206
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + row.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
207
|
+
}
|
|
208
|
+
// Pad to always show 5 lines
|
|
209
|
+
for (let i = recentResults.length; i < 5; i++) {
|
|
210
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + ' '.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
211
|
+
}
|
|
164
212
|
|
|
165
|
-
//
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const steps = [];
|
|
176
|
-
let stepIndex = 1;
|
|
177
|
-
while (true) {
|
|
178
|
-
const step = await p.text({
|
|
179
|
-
message: ` Step ${stepIndex}:`,
|
|
180
|
-
placeholder: stepIndex === 1 ? 'e.g. Send POST /api/users with valid payload' : '(leave empty to finish)',
|
|
181
|
-
});
|
|
182
|
-
if (p.isCancel(step) || !step || step.trim() === '') break;
|
|
183
|
-
steps.push(step.trim());
|
|
184
|
-
stepIndex++;
|
|
213
|
+
// Bugs
|
|
214
|
+
lines.push(chalk.hex('#00F5FF').bold(`├${'─'.repeat(w - 2)}┤`));
|
|
215
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.gray(' Active bugs:').padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
216
|
+
const recentBugs = this.#bugs.slice(-3);
|
|
217
|
+
for (const b of recentBugs) {
|
|
218
|
+
const bugLine = ` ${colorSeverity(b.severity)} ${chalk.white(b.title.slice(0, w - 20))}`;
|
|
219
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + bugLine.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
220
|
+
}
|
|
221
|
+
for (let i = recentBugs.length; i < 3; i++) {
|
|
222
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + ' '.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
185
223
|
}
|
|
186
|
-
if (steps.length === 0) steps.push('No steps provided');
|
|
187
224
|
|
|
188
|
-
//
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
const execSpinner = ora({
|
|
198
|
-
text: chalk.cyan(`Executing: "${testTitle}"...`),
|
|
199
|
-
spinner: 'dots12',
|
|
200
|
-
color: 'cyan',
|
|
201
|
-
}).start();
|
|
202
|
-
await new Promise((r) => setTimeout(r, 1200));
|
|
203
|
-
|
|
204
|
-
const testResult = await p.select({
|
|
205
|
-
message: 'What was the test result?',
|
|
206
|
-
options: [
|
|
207
|
-
{ value: 'pass', label: '✅ PASS — Test passed as expected' },
|
|
208
|
-
{ value: 'fail', label: '❌ FAIL — Test did not pass' },
|
|
209
|
-
{ value: 'skip', label: '⏭️ SKIP — Not applicable/skipped' },
|
|
210
|
-
],
|
|
211
|
-
});
|
|
212
|
-
if (p.isCancel(testResult)) break;
|
|
213
|
-
|
|
214
|
-
if (testResult === 'pass') {
|
|
215
|
-
execSpinner.succeed(chalk.green(`PASS: ${testTitle}`));
|
|
216
|
-
} else if (testResult === 'skip') {
|
|
217
|
-
execSpinner.warn(chalk.yellow(`SKIP: ${testTitle}`));
|
|
218
|
-
} else {
|
|
219
|
-
execSpinner.fail(chalk.red(`FAIL: ${testTitle}`));
|
|
225
|
+
// Log
|
|
226
|
+
lines.push(chalk.hex('#00F5FF').bold(`├${'─'.repeat(w - 2)}┤`));
|
|
227
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + chalk.gray(' Event log:').padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
228
|
+
const recentLogs = this.#log.slice(-4);
|
|
229
|
+
for (const entry of recentLogs) {
|
|
230
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + (' ' + entry).slice(0, w - 2).padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
231
|
+
}
|
|
232
|
+
for (let i = recentLogs.length; i < 4; i++) {
|
|
233
|
+
lines.push(chalk.hex('#00F5FF').bold('│') + ' '.padEnd(w - 2) + chalk.hex('#00F5FF').bold('│'));
|
|
220
234
|
}
|
|
221
235
|
|
|
222
|
-
|
|
236
|
+
lines.push(chalk.hex('#00F5FF').bold(`└${bar}┘`));
|
|
237
|
+
lines.push(DIM(' Press Ctrl+C to stop live monitoring'));
|
|
223
238
|
|
|
224
|
-
|
|
225
|
-
|
|
239
|
+
return lines;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
226
242
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
let logs = [];
|
|
248
|
-
if (!p.isCancel(hasLogs) && hasLogs) {
|
|
249
|
-
const logInput = await p.text({
|
|
250
|
-
message: 'Paste logs (one line):',
|
|
251
|
-
placeholder: 'Error: ECONNREFUSED 127.0.0.1:5432',
|
|
252
|
-
});
|
|
253
|
-
if (!p.isCancel(logInput) && logInput) logs = [logInput.trim()];
|
|
243
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
244
|
+
// Test Runner
|
|
245
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
246
|
+
|
|
247
|
+
class TestRunner extends EventEmitter {
|
|
248
|
+
#results = [];
|
|
249
|
+
#running = false;
|
|
250
|
+
#aborted = false;
|
|
251
|
+
|
|
252
|
+
async run(tests, dashboard = null) {
|
|
253
|
+
this.#running = true;
|
|
254
|
+
this.#aborted = false;
|
|
255
|
+
this.#results = [];
|
|
256
|
+
|
|
257
|
+
for (const test of tests) {
|
|
258
|
+
if (this.#aborted) break;
|
|
259
|
+
if (dashboard) {
|
|
260
|
+
dashboard.updateRunning(test.name);
|
|
261
|
+
dashboard.addLog(`Starting: ${test.name}`);
|
|
262
|
+
dashboard.render({});
|
|
254
263
|
}
|
|
255
264
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
265
|
+
const result = await this.#runOne(test);
|
|
266
|
+
this.#results.push(result);
|
|
267
|
+
this.emit('result', result);
|
|
268
|
+
|
|
269
|
+
if (dashboard) {
|
|
270
|
+
dashboard.addResult(result);
|
|
271
|
+
if (result.status === 'FAIL') {
|
|
272
|
+
dashboard.addBug({
|
|
273
|
+
id : `AUTO-${shortId()}`,
|
|
274
|
+
title : `Test failure: ${test.name}`,
|
|
275
|
+
severity : this.#classifySeverity(test.type, result.error),
|
|
276
|
+
status : 'OPEN',
|
|
277
|
+
});
|
|
278
|
+
dashboard.addLog(chalk.red(`FAIL: ${test.name} — ${result.error ?? 'unknown'}`));
|
|
279
|
+
} else {
|
|
280
|
+
dashboard.addLog(chalk.green(`${result.status}: ${test.name} (${formatDuration(result.duration)})`));
|
|
281
|
+
}
|
|
282
|
+
dashboard.render({});
|
|
283
|
+
await sleep(80); // pacing for readability
|
|
284
|
+
}
|
|
267
285
|
}
|
|
268
286
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
title: testTitle,
|
|
272
|
-
steps,
|
|
273
|
-
expected: expected?.trim() || '',
|
|
274
|
-
result: testResult,
|
|
275
|
-
bugReportId: bugReport?.id || null,
|
|
276
|
-
timestamp: new Date().toISOString(),
|
|
277
|
-
});
|
|
278
|
-
|
|
279
|
-
const continueAdding = await p.confirm({
|
|
280
|
-
message: 'Add another test case?',
|
|
281
|
-
initialValue: true,
|
|
282
|
-
});
|
|
283
|
-
if (p.isCancel(continueAdding) || !continueAdding) addMore = false;
|
|
287
|
+
this.#running = false;
|
|
288
|
+
return [...this.#results];
|
|
284
289
|
}
|
|
285
290
|
|
|
286
|
-
|
|
287
|
-
const passed = testCases.filter((t) => t.result === 'pass').length;
|
|
288
|
-
const failed = testCases.filter((t) => t.result === 'fail').length;
|
|
289
|
-
const skipped = testCases.filter((t) => t.result === 'skip').length;
|
|
290
|
-
const elapsed = ((Date.now() - sessionStart) / 1000).toFixed(1);
|
|
291
|
+
abort() { this.#aborted = true; }
|
|
291
292
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
console.log(` ${chalk.dim('Session ID:')} ${chalk.white.bold(sessionId)}`);
|
|
298
|
-
console.log(` ${chalk.dim('Duration:')} ${chalk.white(elapsed + 's')}`);
|
|
299
|
-
console.log(` ${chalk.dim('Total Tests:')} ${chalk.white(testCases.length)}`);
|
|
300
|
-
console.log(` ${chalk.green('✅ Passed:')} ${chalk.green.bold(passed)}`);
|
|
301
|
-
console.log(` ${chalk.red('❌ Failed:')} ${chalk.red.bold(failed)}`);
|
|
302
|
-
console.log(` ${chalk.yellow('⏭️ Skipped:')} ${chalk.yellow.bold(skipped)}`);
|
|
303
|
-
if (bugReports.length > 0) {
|
|
304
|
-
console.log(` ${chalk.hex('#FF6B6B')('🐛 Bug Reports:')} ${chalk.hex('#FF6B6B').bold(bugReports.length)}`);
|
|
293
|
+
#classifySeverity(type, error = '') {
|
|
294
|
+
if (type === 'auth' || type === 'security') return 'P0';
|
|
295
|
+
if (type === 'e2e' || error?.includes('crash')) return 'P1';
|
|
296
|
+
if (type === 'validation' || type === 'performance') return 'P2';
|
|
297
|
+
return 'P3';
|
|
305
298
|
}
|
|
306
|
-
console.log('');
|
|
307
|
-
|
|
308
|
-
// Export option
|
|
309
|
-
if (testCases.length > 0) {
|
|
310
|
-
const exportFmt = await p.select({
|
|
311
|
-
message: 'Export QA report as:',
|
|
312
|
-
options: [
|
|
313
|
-
{ value: 'json', label: '📄 JSON' },
|
|
314
|
-
{ value: 'html', label: '🌐 HTML' },
|
|
315
|
-
{ value: 'skip', label: '⏭️ Skip export' },
|
|
316
|
-
],
|
|
317
|
-
});
|
|
318
299
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
300
|
+
async #runOne(test) {
|
|
301
|
+
const { id, name, type, fn, timeout = DEFAULT_TIMEOUT_MS } = test;
|
|
302
|
+
const start = Date.now();
|
|
303
|
+
let retries = 0;
|
|
304
|
+
let lastError = null;
|
|
305
|
+
|
|
306
|
+
for (let attempt = 0; attempt <= FLAKY_RETRY_COUNT; attempt++) {
|
|
307
|
+
try {
|
|
308
|
+
await Promise.race([
|
|
309
|
+
fn(),
|
|
310
|
+
sleep(timeout).then(() => { throw new Error(`Timed out after ${timeout}ms`); }),
|
|
311
|
+
]);
|
|
312
|
+
const status = attempt > 0 ? 'FLAKY' : 'PASS';
|
|
313
|
+
return { id, name, type, status, duration: Date.now() - start, retries: attempt, error: null };
|
|
314
|
+
} catch (err) {
|
|
315
|
+
lastError = err.message;
|
|
316
|
+
retries = attempt;
|
|
317
|
+
if (attempt < FLAKY_RETRY_COUNT) await sleep(200);
|
|
318
|
+
}
|
|
322
319
|
}
|
|
320
|
+
|
|
321
|
+
return { id, name, type, status: 'FAIL', duration: Date.now() - start, retries, error: lastError };
|
|
323
322
|
}
|
|
323
|
+
}
|
|
324
324
|
|
|
325
|
-
|
|
326
|
-
|
|
325
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
326
|
+
// End-to-End Test Suite Builder
|
|
327
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
327
328
|
|
|
328
|
-
|
|
329
|
-
|
|
329
|
+
function buildEndpointTests(endpoints) {
|
|
330
|
+
const tests = [];
|
|
330
331
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
332
|
+
for (const ep of endpoints) {
|
|
333
|
+
const label = `${ep.method} ${ep.route}`;
|
|
334
334
|
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
name: 'System Health Checks',
|
|
340
|
-
tests: [
|
|
341
|
-
{ id: 't001', name: 'Node.js version check', fn: checkNodeVersion },
|
|
342
|
-
{ id: 't002', name: 'package.json integrity', fn: checkPackageJson },
|
|
343
|
-
{ id: 't003', name: 'Required dependencies present', fn: checkDependencies },
|
|
344
|
-
{ id: 't004', name: 'No circular imports detected', fn: checkNoCircularImports },
|
|
345
|
-
],
|
|
346
|
-
},
|
|
347
|
-
{
|
|
348
|
-
id: 'suite-structure',
|
|
349
|
-
name: 'Project Structure Validation',
|
|
350
|
-
tests: [
|
|
351
|
-
{ id: 't005', name: 'bin/index.js entry point exists', fn: checkEntryPoint },
|
|
352
|
-
{ id: 't006', name: 'src/ directory structure valid', fn: checkSrcStructure },
|
|
353
|
-
{ id: 't007', name: 'Generator modules present', fn: checkGenerators },
|
|
354
|
-
{ id: 't008', name: 'Template files accessible', fn: checkTemplates },
|
|
355
|
-
],
|
|
356
|
-
},
|
|
357
|
-
{
|
|
358
|
-
id: 'suite-code-quality',
|
|
359
|
-
name: 'Code Quality Analysis',
|
|
360
|
-
tests: [
|
|
361
|
-
{ id: 't009', name: 'ES module imports consistent', fn: checkESModules },
|
|
362
|
-
{ id: 't010', name: 'No hardcoded secrets detected', fn: checkNoSecrets },
|
|
363
|
-
{ id: 't011', name: 'Error handling coverage', fn: checkErrorHandling },
|
|
364
|
-
{ id: 't012', name: 'Async/await patterns valid', fn: checkAsyncPatterns },
|
|
365
|
-
],
|
|
366
|
-
},
|
|
367
|
-
{
|
|
368
|
-
id: 'suite-perf',
|
|
369
|
-
name: 'Performance Benchmarks',
|
|
370
|
-
tests: [
|
|
371
|
-
{ id: 't013', name: 'CLI startup time benchmark', fn: benchmarkStartup },
|
|
372
|
-
{ id: 't014', name: 'File I/O performance', fn: benchmarkFileIO },
|
|
373
|
-
{ id: 't015', name: 'Memory usage check', fn: checkMemoryUsage },
|
|
374
|
-
],
|
|
375
|
-
},
|
|
376
|
-
];
|
|
335
|
+
tests.push({ id: shortId(), name: `Happy path: ${label}`, type: 'happy-path', fn: async () => {
|
|
336
|
+
await sleep(30 + Math.random() * 80);
|
|
337
|
+
if (!ep.route || !ep.method) throw new Error('Endpoint missing route or method');
|
|
338
|
+
}});
|
|
377
339
|
|
|
378
|
-
|
|
340
|
+
if (ep.schemaFields && Object.keys(ep.schemaFields).length > 0) {
|
|
341
|
+
tests.push({ id: shortId(), name: `Validation: ${label}`, type: 'validation', fn: async () => {
|
|
342
|
+
await sleep(25);
|
|
343
|
+
const missing = Object.entries(ep.schemaFields).filter(([, t]) => !t);
|
|
344
|
+
if (missing.length) throw new Error(`Fields missing types: ${missing.map(([k]) => k).join(', ')}`);
|
|
345
|
+
}});
|
|
346
|
+
}
|
|
379
347
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
}
|
|
348
|
+
if (/\/admin|\/user|\/auth|\/profile|\/dashboard|\/private/i.test(ep.route)) {
|
|
349
|
+
tests.push({ id: shortId(), name: `Auth guard: ${label}`, type: 'auth', fn: async () => {
|
|
350
|
+
await sleep(40);
|
|
351
|
+
}});
|
|
352
|
+
}
|
|
386
353
|
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
}
|
|
354
|
+
if (ep.pathParams?.length > 0) {
|
|
355
|
+
tests.push({ id: shortId(), name: `Edge case — empty param: ${label}`, type: 'edge-case', fn: async () => {
|
|
356
|
+
await sleep(20);
|
|
357
|
+
if (ep.pathParams.find(p => p.length === 0) !== undefined) throw new Error('Empty path parameter');
|
|
358
|
+
}});
|
|
359
|
+
}
|
|
360
|
+
}
|
|
395
361
|
|
|
396
|
-
|
|
397
|
-
const pkgPath = path.join(process.cwd(), 'package.json');
|
|
398
|
-
if (!(await fs.pathExists(pkgPath))) throw new Error('package.json not found.');
|
|
399
|
-
const pkg = await fs.readJson(pkgPath);
|
|
400
|
-
const required = ['chalk', 'ora', '@clack/prompts', 'fs-extra'];
|
|
401
|
-
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
402
|
-
const missing = required.filter((d) => !deps[d]);
|
|
403
|
-
if (missing.length > 0) throw new Error(`Missing dependencies: ${missing.join(', ')}`);
|
|
404
|
-
return `All ${required.length} core dependencies present`;
|
|
362
|
+
return tests;
|
|
405
363
|
}
|
|
406
364
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
365
|
+
function buildFullSystemTests(projectDir = process.cwd()) {
|
|
366
|
+
const tests = [];
|
|
367
|
+
|
|
368
|
+
// ── Module Scan ──────────────────────────────────────────────────────
|
|
369
|
+
tests.push({ id: shortId(), name: 'Project structure integrity', type: 'e2e', fn: async () => {
|
|
370
|
+
const exists = await fs.pathExists(projectDir);
|
|
371
|
+
if (!exists) throw new Error('Project directory not found');
|
|
372
|
+
}});
|
|
373
|
+
|
|
374
|
+
tests.push({ id: shortId(), name: 'Package.json valid', type: 'validation', fn: async () => {
|
|
375
|
+
const pkgPath = path.join(projectDir, 'package.json');
|
|
376
|
+
if (!(await fs.pathExists(pkgPath))) throw new Error('package.json missing');
|
|
377
|
+
const pkg = await fs.readJson(pkgPath);
|
|
378
|
+
if (!pkg.name) throw new Error('package.json has no name field');
|
|
379
|
+
}});
|
|
380
|
+
|
|
381
|
+
tests.push({ id: shortId(), name: 'Dependencies declared', type: 'validation', fn: async () => {
|
|
382
|
+
const pkgPath = path.join(projectDir, 'package.json');
|
|
383
|
+
if (!(await fs.pathExists(pkgPath))) return;
|
|
384
|
+
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
385
|
+
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
386
|
+
if (Object.keys(deps).length === 0) throw new Error('No dependencies declared');
|
|
387
|
+
}});
|
|
388
|
+
|
|
389
|
+
// ── API/Backend Tests ────────────────────────────────────────────────
|
|
390
|
+
tests.push({ id: shortId(), name: 'API routes file exists', type: 'happy-path', fn: async () => {
|
|
391
|
+
const candidates = ['src/routes', 'routes', 'src/api', 'api', 'src/controllers', 'controllers'];
|
|
392
|
+
for (const c of candidates) {
|
|
393
|
+
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
394
|
+
}
|
|
395
|
+
throw new Error('No routes/api directory found');
|
|
396
|
+
}});
|
|
412
397
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
}
|
|
398
|
+
tests.push({ id: shortId(), name: 'Entry point reachable', type: 'happy-path', fn: async () => {
|
|
399
|
+
const candidates = ['src/index.ts', 'src/index.js', 'index.ts', 'index.js', 'main.py', 'main.go', 'Program.cs'];
|
|
400
|
+
for (const c of candidates) {
|
|
401
|
+
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
402
|
+
}
|
|
403
|
+
throw new Error('No recognisable entry point found');
|
|
404
|
+
}});
|
|
420
405
|
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
406
|
+
tests.push({ id: shortId(), name: 'Environment config present', type: 'validation', fn: async () => {
|
|
407
|
+
const candidates = ['.env', '.env.example', '.env.sample', 'config.js', 'config.ts', 'appsettings.json'];
|
|
408
|
+
for (const c of candidates) {
|
|
409
|
+
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
410
|
+
}
|
|
411
|
+
throw new Error('No environment config file found');
|
|
412
|
+
}});
|
|
413
|
+
|
|
414
|
+
// ── Auth Tests ───────────────────────────────────────────────────────
|
|
415
|
+
tests.push({ id: shortId(), name: 'JWT middleware present', type: 'auth', fn: async () => {
|
|
416
|
+
const candidates = ['src/middleware', 'middleware', 'src/middlewares', 'middlewares'];
|
|
417
|
+
for (const c of candidates) {
|
|
418
|
+
if (await fs.pathExists(path.join(projectDir, c))) {
|
|
419
|
+
const files = await fs.readdir(path.join(projectDir, c)).catch(() => []);
|
|
420
|
+
const hasAuth = files.some(f => /auth|jwt|guard|verify/i.test(f));
|
|
421
|
+
if (hasAuth) return;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
// soft check — not all stacks need middleware dir
|
|
425
|
+
}});
|
|
426
|
+
|
|
427
|
+
tests.push({ id: shortId(), name: 'Password hashing config', type: 'security', fn: async () => {
|
|
428
|
+
await sleep(35);
|
|
429
|
+
// Scan for bcrypt/argon2 usage in package.json
|
|
430
|
+
const pkgPath = path.join(projectDir, 'package.json');
|
|
431
|
+
if (!(await fs.pathExists(pkgPath))) return;
|
|
432
|
+
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
433
|
+
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
434
|
+
const hasHasher = ['bcrypt', 'bcryptjs', 'argon2', 'argon2d'].some(d => deps[d]);
|
|
435
|
+
if (!hasHasher) throw new Error('No password hashing library (bcrypt/argon2) found');
|
|
436
|
+
}});
|
|
437
|
+
|
|
438
|
+
// ── Database Tests ───────────────────────────────────────────────────
|
|
439
|
+
tests.push({ id: shortId(), name: 'Database schema defined', type: 'validation', fn: async () => {
|
|
440
|
+
const candidates = ['prisma/schema.prisma', 'schema.prisma', 'src/models', 'models', 'src/entities'];
|
|
441
|
+
for (const c of candidates) {
|
|
442
|
+
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
443
|
+
}
|
|
444
|
+
throw new Error('No database schema/models directory found');
|
|
445
|
+
}});
|
|
446
|
+
|
|
447
|
+
tests.push({ id: shortId(), name: 'Migration scripts present', type: 'validation', fn: async () => {
|
|
448
|
+
await sleep(20);
|
|
449
|
+
const candidates = ['prisma/migrations', 'migrations', 'db/migrations', 'src/migrations'];
|
|
450
|
+
for (const c of candidates) {
|
|
451
|
+
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
452
|
+
}
|
|
453
|
+
// Acceptable if using ORMs that auto-migrate
|
|
454
|
+
}});
|
|
455
|
+
|
|
456
|
+
// ── Security Scan ────────────────────────────────────────────────────
|
|
457
|
+
tests.push({ id: shortId(), name: 'CORS config found', type: 'security', fn: async () => {
|
|
458
|
+
await sleep(40);
|
|
459
|
+
// Scan src/index.* for cors usage
|
|
460
|
+
const candidates = ['src/index.ts', 'src/index.js', 'src/app.ts', 'src/app.js', 'index.js', 'index.ts'];
|
|
461
|
+
for (const c of candidates) {
|
|
462
|
+
const filePath = path.join(projectDir, c);
|
|
463
|
+
if (await fs.pathExists(filePath)) {
|
|
464
|
+
const content = await fs.readFile(filePath, 'utf8').catch(() => '');
|
|
465
|
+
if (/cors|CORS/i.test(content)) return;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
throw new Error('No CORS configuration detected in app entry');
|
|
469
|
+
}});
|
|
470
|
+
|
|
471
|
+
tests.push({ id: shortId(), name: 'Rate limiting configured', type: 'security', fn: async () => {
|
|
472
|
+
await sleep(30);
|
|
473
|
+
const pkgPath = path.join(projectDir, 'package.json');
|
|
474
|
+
if (!(await fs.pathExists(pkgPath))) return;
|
|
475
|
+
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
476
|
+
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
477
|
+
const hasLimiter = ['express-rate-limit', 'rate-limiter-flexible', 'fastapi-limiter', 'throttler'].some(d => deps[d]);
|
|
478
|
+
if (!hasLimiter) throw new Error('No rate-limiting library found');
|
|
479
|
+
}});
|
|
480
|
+
|
|
481
|
+
tests.push({ id: shortId(), name: 'Secrets not hardcoded', type: 'security', fn: async () => {
|
|
482
|
+
await sleep(50);
|
|
483
|
+
const scanTargets = ['src/index.ts', 'src/index.js', 'src/app.ts', 'src/app.js'];
|
|
484
|
+
const secretPattern = /(?:password|secret|apikey|api_key)\s*=\s*['"][^'"]{6,}['"]/i;
|
|
485
|
+
for (const t of scanTargets) {
|
|
486
|
+
const filePath = path.join(projectDir, t);
|
|
487
|
+
if (await fs.pathExists(filePath)) {
|
|
488
|
+
const content = await fs.readFile(filePath, 'utf8').catch(() => '');
|
|
489
|
+
if (secretPattern.test(content)) throw new Error(`Hardcoded secret detected in ${t}`);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}});
|
|
493
|
+
|
|
494
|
+
// ── Performance Checks ───────────────────────────────────────────────
|
|
495
|
+
tests.push({ id: shortId(), name: 'Heap memory acceptable', type: 'performance', fn: async () => {
|
|
496
|
+
await sleep(20);
|
|
497
|
+
const heapMB = process.memoryUsage().heapUsed / 1024 / 1024;
|
|
498
|
+
if (heapMB > 512) throw new Error(`Heap usage too high: ${heapMB.toFixed(0)}MB (limit 512MB)`);
|
|
499
|
+
}});
|
|
500
|
+
|
|
501
|
+
tests.push({ id: shortId(), name: 'File system scan speed', type: 'performance', fn: async () => {
|
|
502
|
+
const t0 = performance.now();
|
|
503
|
+
await fs.readdir(projectDir).catch(() => []);
|
|
504
|
+
const elapsed = performance.now() - t0;
|
|
505
|
+
if (elapsed > 2000) throw new Error(`FS scan too slow: ${elapsed.toFixed(0)}ms`);
|
|
506
|
+
}});
|
|
507
|
+
|
|
508
|
+
tests.push({ id: shortId(), name: 'Node.js version check', type: 'happy-path', fn: async () => {
|
|
509
|
+
const maj = parseInt(process.version.slice(1));
|
|
510
|
+
if (maj < 18) throw new Error(`Node.js ${process.version} — requires v18+`);
|
|
511
|
+
}});
|
|
512
|
+
|
|
513
|
+
// ── Docker / Deploy ──────────────────────────────────────────────────
|
|
514
|
+
tests.push({ id: shortId(), name: 'Dockerfile present', type: 'e2e', fn: async () => {
|
|
515
|
+
const candidates = ['Dockerfile', 'Dockerfile.dev', 'docker-compose.yml', 'docker-compose.yaml'];
|
|
516
|
+
for (const c of candidates) {
|
|
517
|
+
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
518
|
+
}
|
|
519
|
+
throw new Error('No Docker configuration found');
|
|
520
|
+
}});
|
|
521
|
+
|
|
522
|
+
tests.push({ id: shortId(), name: 'CI/CD pipeline configured', type: 'e2e', fn: async () => {
|
|
523
|
+
const ciPaths = ['.github/workflows', '.gitlab-ci.yml', '.circleci', 'Jenkinsfile'];
|
|
524
|
+
for (const c of ciPaths) {
|
|
525
|
+
if (await fs.pathExists(path.join(projectDir, c))) return;
|
|
526
|
+
}
|
|
527
|
+
throw new Error('No CI/CD pipeline detected');
|
|
528
|
+
}});
|
|
529
|
+
|
|
530
|
+
// ── Test Infrastructure ──────────────────────────────────────────────
|
|
531
|
+
tests.push({ id: shortId(), name: 'Test files exist', type: 'e2e', fn: async () => {
|
|
532
|
+
const testDirs = ['tests', 'test', '__tests__', 'spec'];
|
|
533
|
+
for (const d of testDirs) {
|
|
534
|
+
if (await fs.pathExists(path.join(projectDir, d))) {
|
|
535
|
+
const files = await fs.readdir(path.join(projectDir, d)).catch(() => []);
|
|
536
|
+
if (files.length > 0) return;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
throw new Error('No test files found');
|
|
540
|
+
}});
|
|
541
|
+
|
|
542
|
+
tests.push({ id: shortId(), name: 'Test script configured', type: 'validation', fn: async () => {
|
|
543
|
+
const pkgPath = path.join(projectDir, 'package.json');
|
|
544
|
+
if (!(await fs.pathExists(pkgPath))) return;
|
|
545
|
+
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
546
|
+
if (!pkg.scripts?.test) throw new Error('No "test" script in package.json');
|
|
547
|
+
}});
|
|
548
|
+
|
|
549
|
+
// ── Swagger / Docs ───────────────────────────────────────────────────
|
|
550
|
+
tests.push({ id: shortId(), name: 'API documentation configured', type: 'happy-path', fn: async () => {
|
|
551
|
+
await sleep(20);
|
|
552
|
+
const pkgPath = path.join(projectDir, 'package.json');
|
|
553
|
+
if (!(await fs.pathExists(pkgPath))) return;
|
|
554
|
+
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
555
|
+
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
556
|
+
const hasDocs = ['swagger-ui-express', 'swagger-jsdoc', '@nestjs/swagger', 'fastapi', 'springdoc-openapi'].some(d => deps[d]);
|
|
557
|
+
if (!hasDocs) throw new Error('No API documentation library found');
|
|
558
|
+
}});
|
|
559
|
+
|
|
560
|
+
// ── Logging ──────────────────────────────────────────────────────────
|
|
561
|
+
tests.push({ id: shortId(), name: 'Logging library present', type: 'validation', fn: async () => {
|
|
562
|
+
await sleep(15);
|
|
563
|
+
const pkgPath = path.join(projectDir, 'package.json');
|
|
564
|
+
if (!(await fs.pathExists(pkgPath))) return;
|
|
565
|
+
const pkg = await fs.readJson(pkgPath).catch(() => ({}));
|
|
566
|
+
const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
567
|
+
const hasLogger = ['winston', 'pino', 'morgan', 'log4j', 'structlog'].some(d => deps[d]);
|
|
568
|
+
if (!hasLogger) throw new Error('No structured logging library found');
|
|
569
|
+
}});
|
|
570
|
+
|
|
571
|
+
// ── Baseline (always pass) ────────────────────────────────────────────
|
|
572
|
+
tests.push({ id: shortId(), name: 'QA system operational', type: 'happy-path', fn: async () => {
|
|
573
|
+
await fs.ensureDir(QA_DIR);
|
|
574
|
+
}});
|
|
575
|
+
|
|
576
|
+
tests.push({ id: shortId(), name: 'Report directory writable', type: 'happy-path', fn: async () => {
|
|
577
|
+
await fs.ensureDir(REPORT_DIR);
|
|
578
|
+
const testFile = path.join(REPORT_DIR, `.write-test-${shortId()}`);
|
|
579
|
+
await fs.writeFile(testFile, 'ok');
|
|
580
|
+
await fs.remove(testFile);
|
|
581
|
+
}});
|
|
582
|
+
|
|
583
|
+
return tests;
|
|
427
584
|
}
|
|
428
585
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
586
|
+
// ── UI Simulation tests ───────────────────────────────────────────────────
|
|
587
|
+
|
|
588
|
+
function buildUITests(srcDir = path.join(process.cwd(), 'src')) {
|
|
589
|
+
return [
|
|
590
|
+
{ id: shortId(), name: 'Frontend src directory exists', type: 'ui', fn: async () => {
|
|
591
|
+
if (!(await fs.pathExists(srcDir))) throw new Error(`src directory not found: ${srcDir}`);
|
|
592
|
+
}},
|
|
593
|
+
{ id: shortId(), name: 'Component files present', type: 'ui', fn: async () => {
|
|
594
|
+
const exts = ['.tsx', '.jsx', '.vue', '.svelte'];
|
|
595
|
+
let found = false;
|
|
596
|
+
const walk = async (dir) => {
|
|
597
|
+
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
598
|
+
for (const e of entries) {
|
|
599
|
+
if (e.isDirectory() && e.name !== 'node_modules') await walk(path.join(dir, e.name));
|
|
600
|
+
else if (exts.some(x => e.name.endsWith(x))) { found = true; return; }
|
|
601
|
+
}
|
|
602
|
+
};
|
|
603
|
+
await walk(srcDir);
|
|
604
|
+
if (!found) throw new Error('No component files (.tsx/.jsx/.vue/.svelte) found');
|
|
605
|
+
}},
|
|
606
|
+
{ id: shortId(), name: 'Styles configured', type: 'ui', fn: async () => {
|
|
607
|
+
const stylePatterns = ['tailwind.config', 'postcss.config', 'vite.config', 'styles', 'css', 'scss'];
|
|
608
|
+
for (const pat of stylePatterns) {
|
|
609
|
+
const cwd = process.cwd();
|
|
610
|
+
const entries = await fs.readdir(cwd).catch(() => []);
|
|
611
|
+
if (entries.some(f => f.includes(pat))) return;
|
|
612
|
+
}
|
|
613
|
+
throw new Error('No styling configuration found');
|
|
614
|
+
}},
|
|
615
|
+
{ id: shortId(), name: 'API client configuration', type: 'ui', fn: async () => {
|
|
616
|
+
const apiFiles = ['src/api', 'src/services', 'src/lib', 'src/utils'];
|
|
617
|
+
for (const f of apiFiles) {
|
|
618
|
+
if (await fs.pathExists(path.join(process.cwd(), f))) return;
|
|
619
|
+
}
|
|
620
|
+
throw new Error('No API client/services directory found in frontend');
|
|
621
|
+
}},
|
|
622
|
+
{ id: shortId(), name: 'Route configuration present', type: 'ui', fn: async () => {
|
|
623
|
+
const routeFiles = ['src/router', 'src/routes', 'src/pages', 'pages', 'app/routes'];
|
|
624
|
+
for (const f of routeFiles) {
|
|
625
|
+
if (await fs.pathExists(path.join(process.cwd(), f))) return;
|
|
626
|
+
}
|
|
627
|
+
}},
|
|
628
|
+
];
|
|
436
629
|
}
|
|
437
630
|
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
631
|
+
// ── Coverage matrix ────────────────────────────────────────────────────────
|
|
632
|
+
|
|
633
|
+
function buildCoverageMatrix(results) {
|
|
634
|
+
const matrix = {};
|
|
635
|
+
for (const r of results) {
|
|
636
|
+
if (!matrix[r.type]) matrix[r.type] = { total: 0, passed: 0, failed: 0, skipped: 0, flaky: 0 };
|
|
637
|
+
matrix[r.type].total++;
|
|
638
|
+
if (r.status === 'PASS') matrix[r.type].passed++;
|
|
639
|
+
if (r.status === 'FAIL') matrix[r.type].failed++;
|
|
640
|
+
if (r.status === 'SKIP') matrix[r.type].skipped++;
|
|
641
|
+
if (r.status === 'FLAKY') { matrix[r.type].flaky++; matrix[r.type].passed++; }
|
|
442
642
|
}
|
|
443
|
-
|
|
444
|
-
return `${dirs.length} template set(s) found`;
|
|
643
|
+
return matrix;
|
|
445
644
|
}
|
|
446
645
|
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
646
|
+
function buildSummary(results) {
|
|
647
|
+
return {
|
|
648
|
+
total : results.length,
|
|
649
|
+
passed : results.filter(r => ['PASS','FLAKY'].includes(r.status)).length,
|
|
650
|
+
failed : results.filter(r => r.status === 'FAIL').length,
|
|
651
|
+
skipped: results.filter(r => r.status === 'SKIP').length,
|
|
652
|
+
flaky : results.filter(r => r.status === 'FLAKY').length,
|
|
653
|
+
};
|
|
453
654
|
}
|
|
454
655
|
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
656
|
+
// ── HTML Report ────────────────────────────────────────────────────────────
|
|
657
|
+
|
|
658
|
+
function buildHTMLReport(runData) {
|
|
659
|
+
const { id, startedAt, duration, results, bugReports, coverage, summary } = runData;
|
|
660
|
+
const passRate = summary.total > 0 ? ((summary.passed / summary.total) * 100).toFixed(1) : 0;
|
|
661
|
+
const statusColor = passRate >= 90 ? '#22c55e' : passRate >= 70 ? '#f59e0b' : '#ef4444';
|
|
662
|
+
|
|
663
|
+
const typeColors = {
|
|
664
|
+
'happy-path' : ['#064e3b','#34d399'],
|
|
665
|
+
'validation' : ['#1e3a5f','#60a5fa'],
|
|
666
|
+
'auth' : ['#3b1f5e','#c084fc'],
|
|
667
|
+
'edge-case' : ['#3b2a1a','#f59e0b'],
|
|
668
|
+
'performance': ['#1a2a3b','#38bdf8'],
|
|
669
|
+
'security' : ['#450a0a','#f87171'],
|
|
670
|
+
'e2e' : ['#1a3b2a','#4ade80'],
|
|
671
|
+
'ui' : ['#2a1a3b','#a78bfa'],
|
|
672
|
+
};
|
|
673
|
+
|
|
674
|
+
const badgeStyle = (type) => {
|
|
675
|
+
const [bg, fg] = typeColors[type] ?? ['#1e293b','#94a3b8'];
|
|
676
|
+
return `background:${bg};color:${fg};padding:2px 8px;border-radius:4px;font-size:.75rem;font-weight:500`;
|
|
677
|
+
};
|
|
678
|
+
|
|
679
|
+
const covBars = Object.entries(coverage).map(([type, d]) => {
|
|
680
|
+
const pct = d.total ? ((d.passed / d.total) * 100).toFixed(0) : 0;
|
|
681
|
+
const [, fg] = typeColors[type] ?? ['','#94a3b8'];
|
|
682
|
+
return `<div style="display:flex;align-items:center;gap:1rem;margin-bottom:.75rem">
|
|
683
|
+
<div style="width:100px;font-size:.8rem;color:#94a3b8">${type}</div>
|
|
684
|
+
<div style="flex:1;background:#2d2d4e;border-radius:4px;height:8px;overflow:hidden">
|
|
685
|
+
<div style="height:100%;width:${pct}%;background:${fg};border-radius:4px"></div>
|
|
686
|
+
</div>
|
|
687
|
+
<div style="width:60px;text-align:right;font-size:.8rem;color:#64748b">${d.passed}/${d.total}</div>
|
|
688
|
+
</div>`;
|
|
689
|
+
}).join('');
|
|
690
|
+
|
|
691
|
+
const rows = results.map(r => `<tr class="${r.status.toLowerCase()}">
|
|
692
|
+
<td>${r.name}</td>
|
|
693
|
+
<td><span style="${badgeStyle(r.type)}">${r.type}</span></td>
|
|
694
|
+
<td><span class="status status-${r.status.toLowerCase()}">${r.status}</span></td>
|
|
695
|
+
<td>${r.duration}ms</td>
|
|
696
|
+
<td>${r.retries > 0 ? `<span style="background:#422006;color:#fb923c;padding:2px 8px;border-radius:4px;font-size:.75rem">${r.retries}x retry</span>` : '—'}</td>
|
|
697
|
+
<td class="err">${r.error ? `<code>${r.error}</code>` : '—'}</td>
|
|
698
|
+
</tr>`).join('');
|
|
699
|
+
|
|
700
|
+
const bugCards = bugReports.length ? bugReports.map(b => `
|
|
701
|
+
<div class="bug-card bug-${b.severity?.toLowerCase()}">
|
|
702
|
+
<div class="bug-header"><span class="bug-id">${b.id}</span><span class="bug-sev">${b.severity}</span><span class="bug-st">${b.status}</span></div>
|
|
703
|
+
<div class="bug-title">${b.title}</div>
|
|
704
|
+
${b.description ? `<div class="bug-desc">${b.description}</div>` : ''}
|
|
705
|
+
</div>`).join('') : '<p style="color:#34d399;text-align:center;padding:1rem">No bug reports 🎉</p>';
|
|
706
|
+
|
|
707
|
+
// Chart data for pass/fail by type
|
|
708
|
+
const chartLabels = JSON.stringify(Object.keys(coverage));
|
|
709
|
+
const chartPassed = JSON.stringify(Object.values(coverage).map(d => d.passed));
|
|
710
|
+
const chartFailed = JSON.stringify(Object.values(coverage).map(d => d.failed));
|
|
711
|
+
|
|
712
|
+
return `<!DOCTYPE html>
|
|
713
|
+
<html lang="en">
|
|
714
|
+
<head>
|
|
715
|
+
<meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
716
|
+
<title>Backlist QA Report — ${id}</title>
|
|
717
|
+
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.4.1/chart.umd.js"></script>
|
|
718
|
+
<style>
|
|
719
|
+
*{box-sizing:border-box;margin:0;padding:0}
|
|
720
|
+
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;background:#0a0a12;color:#e2e8f0;font-size:14px;line-height:1.6}
|
|
721
|
+
header{background:linear-gradient(135deg,#1a1a2e,#16213e);border-bottom:1px solid #00f5ff33;padding:1.5rem 2rem}
|
|
722
|
+
header h1{font-size:1.4rem;font-weight:600;color:#00f5ff}header p{color:#64748b;font-size:.85rem;margin-top:4px}
|
|
723
|
+
.container{max-width:1200px;margin:0 auto;padding:2rem}
|
|
724
|
+
.metrics{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:1rem;margin-bottom:2rem}
|
|
725
|
+
.metric-card{background:#1e1e30;border:1px solid #2d2d4e;border-radius:10px;padding:1rem 1.25rem}
|
|
726
|
+
.metric-label{font-size:.75rem;color:#64748b;text-transform:uppercase;letter-spacing:.05em}
|
|
727
|
+
.metric-value{font-size:2rem;font-weight:700;margin-top:4px}
|
|
728
|
+
.section{background:#1e1e30;border:1px solid #2d2d4e;border-radius:10px;padding:1.5rem;margin-bottom:1.5rem}
|
|
729
|
+
.section-title{font-size:1rem;font-weight:600;margin-bottom:1rem;color:#cbd5e1;border-bottom:1px solid #2d2d4e;padding-bottom:.75rem}
|
|
730
|
+
table{width:100%;border-collapse:collapse;font-size:.85rem}
|
|
731
|
+
th{text-align:left;color:#64748b;font-weight:500;padding:.5rem .75rem;border-bottom:1px solid #2d2d4e}
|
|
732
|
+
td{padding:.5rem .75rem;border-bottom:1px solid #1a1a2e;vertical-align:top}
|
|
733
|
+
tr.fail td{background:rgba(239,68,68,.05)}tr.flaky td{background:rgba(245,158,11,.05)}
|
|
734
|
+
.status{display:inline-block;padding:2px 8px;border-radius:4px;font-size:.75rem;font-weight:600}
|
|
735
|
+
.status-pass{background:#064e3b;color:#34d399}.status-fail{background:#450a0a;color:#f87171}
|
|
736
|
+
.status-skip{background:#1e293b;color:#94a3b8}.status-flaky{background:#422006;color:#fbbf24}
|
|
737
|
+
.err code{font-size:.75rem;color:#f87171;background:#1a0a0a;padding:2px 6px;border-radius:3px}
|
|
738
|
+
.bug-card{border-radius:8px;padding:1rem;margin-bottom:.75rem;border-left:3px solid}
|
|
739
|
+
.bug-p0{background:rgba(239,68,68,.08);border-color:#ef4444}
|
|
740
|
+
.bug-p1{background:rgba(245,158,11,.08);border-color:#f59e0b}
|
|
741
|
+
.bug-p2{background:rgba(96,165,250,.08);border-color:#60a5fa}
|
|
742
|
+
.bug-p3{background:rgba(148,163,184,.08);border-color:#64748b}
|
|
743
|
+
.bug-header{display:flex;gap:.75rem;align-items:center;margin-bottom:.5rem}
|
|
744
|
+
.bug-id{font-family:monospace;font-size:.8rem;color:#64748b}
|
|
745
|
+
.bug-sev{font-size:.75rem;font-weight:700;color:#f87171}
|
|
746
|
+
.bug-st{font-size:.75rem;padding:2px 8px;border-radius:4px;background:#1e293b;color:#94a3b8}
|
|
747
|
+
.bug-title{font-weight:600;margin-bottom:.25rem}.bug-desc{font-size:.8rem;color:#94a3b8}
|
|
748
|
+
.chart-wrap{position:relative;height:280px}
|
|
749
|
+
footer{text-align:center;color:#334155;font-size:.75rem;padding:2rem;border-top:1px solid #1e293b;margin-top:2rem}
|
|
750
|
+
</style>
|
|
751
|
+
</head>
|
|
752
|
+
<body>
|
|
753
|
+
<header>
|
|
754
|
+
<h1>🧪 Backlist QA Report — v9.0</h1>
|
|
755
|
+
<p>Run ID: ${id} · ${new Date(startedAt).toLocaleString()} · Duration: ${formatDuration(duration)}</p>
|
|
756
|
+
</header>
|
|
757
|
+
<div class="container">
|
|
758
|
+
<div class="metrics">
|
|
759
|
+
<div class="metric-card"><div class="metric-label">Pass Rate</div><div class="metric-value" style="color:${statusColor}">${passRate}%</div></div>
|
|
760
|
+
<div class="metric-card"><div class="metric-label">Total Tests</div><div class="metric-value">${summary.total}</div></div>
|
|
761
|
+
<div class="metric-card"><div class="metric-label">Passed</div><div class="metric-value" style="color:#34d399">${summary.passed}</div></div>
|
|
762
|
+
<div class="metric-card"><div class="metric-label">Failed</div><div class="metric-value" style="color:#f87171">${summary.failed}</div></div>
|
|
763
|
+
<div class="metric-card"><div class="metric-label">Flaky</div><div class="metric-value" style="color:#fbbf24">${summary.flaky}</div></div>
|
|
764
|
+
<div class="metric-card"><div class="metric-label">Bug Reports</div><div class="metric-value" style="color:#c084fc">${bugReports.length}</div></div>
|
|
765
|
+
</div>
|
|
766
|
+
|
|
767
|
+
<div class="section">
|
|
768
|
+
<div class="section-title">Coverage by Test Type</div>
|
|
769
|
+
${covBars}
|
|
770
|
+
</div>
|
|
771
|
+
|
|
772
|
+
<div class="section">
|
|
773
|
+
<div class="section-title">Pass vs Fail by Type</div>
|
|
774
|
+
<div class="chart-wrap"><canvas id="typeChart" role="img" aria-label="Grouped bar chart showing pass and fail counts by test type"></canvas></div>
|
|
775
|
+
</div>
|
|
776
|
+
|
|
777
|
+
<div class="section">
|
|
778
|
+
<div class="section-title">Test Results (${results.length})</div>
|
|
779
|
+
<table>
|
|
780
|
+
<thead><tr><th>Test</th><th>Type</th><th>Status</th><th>Duration</th><th>Retries</th><th>Error</th></tr></thead>
|
|
781
|
+
<tbody>${rows}</tbody>
|
|
782
|
+
</table>
|
|
783
|
+
</div>
|
|
784
|
+
|
|
785
|
+
<div class="section">
|
|
786
|
+
<div class="section-title">Bug Reports (${bugReports.length})</div>
|
|
787
|
+
${bugCards}
|
|
788
|
+
</div>
|
|
789
|
+
</div>
|
|
790
|
+
<footer>Generated by create-backlist v9.0 — Backlist Live QA System · ${new Date().toLocaleString()}</footer>
|
|
791
|
+
<script>
|
|
792
|
+
new Chart(document.getElementById('typeChart'), {
|
|
793
|
+
type: 'bar',
|
|
794
|
+
data: {
|
|
795
|
+
labels: ${chartLabels},
|
|
796
|
+
datasets: [
|
|
797
|
+
{ label: 'Passed', data: ${chartPassed}, backgroundColor: '#34d399' },
|
|
798
|
+
{ label: 'Failed', data: ${chartFailed}, backgroundColor: '#f87171' },
|
|
799
|
+
]
|
|
800
|
+
},
|
|
801
|
+
options: {
|
|
802
|
+
responsive: true, maintainAspectRatio: false,
|
|
803
|
+
plugins: { legend: { labels: { color: '#94a3b8' } } },
|
|
804
|
+
scales: {
|
|
805
|
+
x: { ticks: { color: '#64748b' }, grid: { color: '#1e293b' } },
|
|
806
|
+
y: { ticks: { color: '#64748b', stepSize: 1 }, grid: { color: '#1e293b' } }
|
|
467
807
|
}
|
|
468
808
|
}
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
const entryPath = path.join(process.cwd(), 'bin', 'index.js');
|
|
474
|
-
if (!(await fs.pathExists(entryPath))) throw new Error('bin/index.js not found.');
|
|
475
|
-
const content = await fs.readFile(entryPath, 'utf-8');
|
|
476
|
-
const tryCatchCount = (content.match(/try\s*{/g) || []).length;
|
|
477
|
-
if (tryCatchCount < 2) throw new Error(`Only ${tryCatchCount} try/catch block(s) found. Insufficient error handling.`);
|
|
478
|
-
return `${tryCatchCount} try/catch block(s) found — adequate coverage`;
|
|
809
|
+
});
|
|
810
|
+
</script>
|
|
811
|
+
</body>
|
|
812
|
+
</html>`;
|
|
479
813
|
}
|
|
480
814
|
|
|
481
|
-
|
|
482
|
-
const entryPath = path.join(process.cwd(), 'bin', 'index.js');
|
|
483
|
-
if (!(await fs.pathExists(entryPath))) throw new Error('bin/index.js not found.');
|
|
484
|
-
const content = await fs.readFile(entryPath, 'utf-8');
|
|
485
|
-
const asyncFns = (content.match(/async\s+function/g) || []).length + (content.match(/async\s*\(/g) || []).length;
|
|
486
|
-
const awaitUsage = (content.match(/\bawait\s+/g) || []).length;
|
|
487
|
-
if (asyncFns === 0) throw new Error('No async functions found.');
|
|
488
|
-
return `${asyncFns} async function(s), ${awaitUsage} await usage(s) — valid`;
|
|
489
|
-
}
|
|
815
|
+
// ── History helpers ───────────────────────────────────────────────────────
|
|
490
816
|
|
|
491
|
-
async function
|
|
492
|
-
|
|
493
|
-
await fs.
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
817
|
+
export async function initQASystem() {
|
|
818
|
+
await fs.ensureDir(QA_DIR);
|
|
819
|
+
await fs.ensureDir(REPORT_DIR);
|
|
820
|
+
if (!await fs.pathExists(HISTORY_FILE)) {
|
|
821
|
+
await fs.writeJson(HISTORY_FILE, { runs: [] }, { spaces: 2 });
|
|
822
|
+
}
|
|
497
823
|
}
|
|
498
824
|
|
|
499
|
-
async function
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
await fs.writeFile(tmpFile, 'x'.repeat(10000));
|
|
503
|
-
await fs.readFile(tmpFile, 'utf-8');
|
|
504
|
-
await fs.remove(tmpFile);
|
|
505
|
-
const elapsed = (performance.now() - start).toFixed(2);
|
|
506
|
-
if (parseFloat(elapsed) > 1000) throw new Error(`File I/O too slow: ${elapsed}ms`);
|
|
507
|
-
return `10KB read/write: ${elapsed}ms`;
|
|
825
|
+
async function loadHistory() {
|
|
826
|
+
try { return await fs.readJson(HISTORY_FILE); }
|
|
827
|
+
catch { return { runs: [] }; }
|
|
508
828
|
}
|
|
509
829
|
|
|
510
|
-
async function
|
|
511
|
-
const
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
return `Heap: ${heapMB}MB | RSS: ${rssMB}MB`;
|
|
830
|
+
async function saveRun(run) {
|
|
831
|
+
const hist = await loadHistory();
|
|
832
|
+
hist.runs.unshift(run);
|
|
833
|
+
if (hist.runs.length > 50) hist.runs = hist.runs.slice(0, 50);
|
|
834
|
+
await fs.writeJson(HISTORY_FILE, hist, { spaces: 2 });
|
|
516
835
|
}
|
|
517
836
|
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
results.push({ id: test.id, name: test.name, status: 'pass', message, duration: parseFloat(duration) });
|
|
530
|
-
} catch (err) {
|
|
531
|
-
const duration = (performance.now() - testStart).toFixed(1);
|
|
532
|
-
results.push({ id: test.id, name: test.name, status: 'fail', error: err.message, duration: parseFloat(duration) });
|
|
533
|
-
}
|
|
837
|
+
async function exportReport(run) {
|
|
838
|
+
try {
|
|
839
|
+
const slug = run.id.toLowerCase();
|
|
840
|
+
const htmlPath = path.join(REPORT_DIR, `${slug}.html`);
|
|
841
|
+
const jsonPath = path.join(REPORT_DIR, `${slug}.json`);
|
|
842
|
+
await fs.writeFile(htmlPath, buildHTMLReport(run));
|
|
843
|
+
await fs.writeJson(jsonPath, run, { spaces: 2 });
|
|
844
|
+
return htmlPath;
|
|
845
|
+
} catch (err) {
|
|
846
|
+
console.error(chalk.gray(` [warn] Could not write report: ${err.message}`));
|
|
847
|
+
return null;
|
|
534
848
|
}
|
|
535
|
-
|
|
536
|
-
const duration = (performance.now() - suiteStart).toFixed(1);
|
|
537
|
-
return { suiteId: suite.id, suiteName: suite.name, results, duration: parseFloat(duration) };
|
|
538
849
|
}
|
|
539
850
|
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
851
|
+
async function printRunDiff(currentRun) {
|
|
852
|
+
try {
|
|
853
|
+
const hist = await loadHistory();
|
|
854
|
+
const previous = hist.runs.find(r => r.id !== currentRun.id && r.type === currentRun.type);
|
|
855
|
+
if (!previous) return;
|
|
856
|
+
const prevRate = previous.summary.total ? (previous.summary.passed / previous.summary.total * 100).toFixed(0) : 0;
|
|
857
|
+
const currRate = currentRun.summary.total ? (currentRun.summary.passed / currentRun.summary.total * 100).toFixed(0) : 0;
|
|
858
|
+
const delta = Number(currRate) - Number(prevRate);
|
|
859
|
+
if (delta === 0) return;
|
|
860
|
+
const arrow = delta > 0 ? chalk.green(`↑ +${delta}%`) : chalk.red(`↓ ${delta}%`);
|
|
861
|
+
console.log(chalk.gray(` vs previous run (${previous.id}): ${arrow} pass rate`));
|
|
862
|
+
} catch {}
|
|
863
|
+
}
|
|
543
864
|
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
console.log(chalk.hex('#BF40FF').bold(' ║ 🤖 AUTOMATED QA TESTING ENGINE ║'));
|
|
548
|
-
console.log(chalk.hex('#BF40FF').bold(' ╚══════════════════════════════════════════════════════════╝'));
|
|
549
|
-
console.log('');
|
|
550
|
-
console.log(chalk.gray(' Running test suites against your Backlist project.'));
|
|
551
|
-
if (continuous) console.log(chalk.yellow(' ⚡ Continuous mode — press Ctrl+C to stop.'));
|
|
552
|
-
console.log('');
|
|
553
|
-
};
|
|
865
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
866
|
+
// Manual QA Flow
|
|
867
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
554
868
|
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
869
|
+
export async function runManualQA() {
|
|
870
|
+
const runId = `MQA-${shortId()}`;
|
|
871
|
+
const startedAt = timestamp();
|
|
872
|
+
const runner = new TestRunner();
|
|
873
|
+
const bugs = [];
|
|
874
|
+
const manualResults = [];
|
|
559
875
|
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
876
|
+
console.log('');
|
|
877
|
+
const action = await p.select({
|
|
878
|
+
message: 'Manual QA — what would you like to do?',
|
|
879
|
+
options: [
|
|
880
|
+
{ value: 'new-test', label: '✏️ Create & run a custom test case' },
|
|
881
|
+
{ value: 'full-scan', label: '🔬 Full system scan (all modules)', hint: 'Scans entire project' },
|
|
882
|
+
{ value: 'log-bug', label: '🐛 Log a bug report' },
|
|
883
|
+
{ value: 'run-suite', label: '▶️ Run saved test suite' },
|
|
884
|
+
{ value: 'ui-tests', label: '🖥️ Run UI/Frontend tests' },
|
|
885
|
+
{ value: 'security-scan',label: '🛡️ Security scan only' },
|
|
886
|
+
],
|
|
887
|
+
});
|
|
888
|
+
if (p.isCancel(action)) { p.cancel('Cancelled.'); return; }
|
|
889
|
+
|
|
890
|
+
const dashboard = new LiveDashboard();
|
|
891
|
+
|
|
892
|
+
if (action === 'log-bug') {
|
|
893
|
+
await logBugInteractive(bugs);
|
|
894
|
+
} else if (action === 'new-test') {
|
|
895
|
+
await createAndRunTestInteractive(runner, manualResults, dashboard);
|
|
896
|
+
} else if (action === 'full-scan') {
|
|
897
|
+
dashboard.start();
|
|
898
|
+
const allTests = [
|
|
899
|
+
...buildFullSystemTests(),
|
|
900
|
+
...buildUITests(),
|
|
901
|
+
];
|
|
902
|
+
const results = await runner.run(allTests, dashboard);
|
|
903
|
+
manualResults.push(...results);
|
|
904
|
+
dashboard.stop();
|
|
905
|
+
printResultsSummary(results);
|
|
906
|
+
} else if (action === 'ui-tests') {
|
|
907
|
+
dashboard.start();
|
|
908
|
+
const uiTests = buildUITests();
|
|
909
|
+
const results = await runner.run(uiTests, dashboard);
|
|
910
|
+
manualResults.push(...results);
|
|
911
|
+
dashboard.stop();
|
|
912
|
+
printResultsSummary(results);
|
|
913
|
+
} else if (action === 'security-scan') {
|
|
914
|
+
dashboard.start();
|
|
915
|
+
const secTests = buildFullSystemTests().filter(t => t.type === 'security' || t.type === 'auth');
|
|
916
|
+
const results = await runner.run(secTests, dashboard);
|
|
917
|
+
manualResults.push(...results);
|
|
918
|
+
dashboard.stop();
|
|
919
|
+
printResultsSummary(results);
|
|
920
|
+
} else if (action === 'run-suite') {
|
|
921
|
+
await runSavedSuiteInteractive(runner, manualResults, dashboard);
|
|
922
|
+
}
|
|
589
923
|
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
const passRate = totalTests > 0 ? ((totalPassed / totalTests) * 100).toFixed(1) : '0.0';
|
|
924
|
+
const continueLoop = await p.confirm({ message: 'Run another test/action?' });
|
|
925
|
+
if (!p.isCancel(continueLoop) && continueLoop) return runManualQA();
|
|
593
926
|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
console.log(` ${chalk.dim('Execution Time:')} ${chalk.white(totalDuration + 'ms')}`);
|
|
601
|
-
console.log(` ${chalk.dim('Total Tests:')} ${chalk.white(totalTests)}`);
|
|
602
|
-
console.log(` ${chalk.green('✅ Passed:')} ${chalk.green.bold(totalPassed)}`);
|
|
603
|
-
console.log(` ${chalk.red('❌ Failed:')} ${chalk.red.bold(totalFailed)}`);
|
|
604
|
-
console.log(` ${chalk.hex('#00F5FF')('📈 Pass Rate:')} ${parseFloat(passRate) >= 80 ? chalk.green.bold(passRate + '%') : chalk.red.bold(passRate + '%')}`);
|
|
605
|
-
console.log('');
|
|
927
|
+
const duration = Date.now() - new Date(startedAt).getTime();
|
|
928
|
+
const summary = buildSummary(manualResults);
|
|
929
|
+
const coverage = buildCoverageMatrix(manualResults);
|
|
930
|
+
const run = { id: runId, type: 'manual', startedAt, duration, results: manualResults, bugReports: bugs, summary, coverage };
|
|
931
|
+
await saveRun(run);
|
|
932
|
+
const reportFile = await exportReport(run);
|
|
606
933
|
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
const sp = sr.results.filter((r) => r.status === 'pass').length;
|
|
611
|
-
const sf = sr.results.filter((r) => r.status === 'fail').length;
|
|
612
|
-
const icon = sf === 0 ? chalk.green('✅') : chalk.red('❌');
|
|
613
|
-
console.log(` ${icon} ${chalk.white(sr.suiteName)} — ${chalk.dim(sp + '/' + sr.results.length + ' | ' + sr.duration + 'ms')}`);
|
|
614
|
-
});
|
|
934
|
+
p.outro(chalk.hex('#00F5FF').bold(`✓ Session saved — ${pluralize(manualResults.length, 'test')}, ${pluralize(bugs.length, 'bug')}`));
|
|
935
|
+
if (reportFile) console.log(chalk.gray(` 📄 Report: ${reportFile}`));
|
|
936
|
+
}
|
|
615
937
|
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
if (perfSuite) {
|
|
621
|
-
perfSuite.results.forEach((r) => {
|
|
622
|
-
const icon = r.status === 'pass' ? chalk.green('✓') : chalk.red('✗');
|
|
623
|
-
console.log(` ${icon} ${chalk.dim(r.name + ':')} ${chalk.white(r.message || r.error || '')}`);
|
|
624
|
-
});
|
|
625
|
-
}
|
|
938
|
+
function printResultsSummary(results) {
|
|
939
|
+
const passed = results.filter(r => ['PASS','FLAKY'].includes(r.status)).length;
|
|
940
|
+
const failed = results.filter(r => r.status === 'FAIL').length;
|
|
941
|
+
const passRate = results.length ? Math.round((passed / results.length) * 100) : 0;
|
|
626
942
|
|
|
627
|
-
|
|
943
|
+
console.log('');
|
|
944
|
+
console.log(chalk.hex('#00F5FF').bold(' ── Scan Results ──────────────────────────────────────'));
|
|
945
|
+
console.log(` Pass rate: [${buildProgressBar(passRate, 24)}] ${chalk.white.bold(passRate + '%')}`);
|
|
946
|
+
console.log(` ${chalk.green('✓')} ${passed} passed ${chalk.red('✗')} ${failed} failed (${results.length} total)`);
|
|
947
|
+
if (failed > 0) {
|
|
628
948
|
console.log('');
|
|
629
|
-
console.log(
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
console.log(` ${chalk.dim(sr.suiteName.padEnd(30))} ${bar} ${chalk.white(cov + '%')}`);
|
|
949
|
+
console.log(chalk.red.bold(' Failures:'));
|
|
950
|
+
results.filter(r => r.status === 'FAIL').forEach(f => {
|
|
951
|
+
console.log(chalk.red(` ✗ ${f.name}`));
|
|
952
|
+
if (f.error) console.log(chalk.gray(` → ${f.error}`));
|
|
634
953
|
});
|
|
954
|
+
}
|
|
955
|
+
console.log('');
|
|
956
|
+
}
|
|
635
957
|
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
console.log('');
|
|
958
|
+
async function logBugInteractive(bugs) {
|
|
959
|
+
const title = await p.text({ message: 'Bug title:' });
|
|
960
|
+
if (p.isCancel(title)) return;
|
|
961
|
+
const severity = await p.select({
|
|
962
|
+
message: 'Severity:',
|
|
963
|
+
options: Object.entries(SEVERITY_LEVELS).map(([k, v]) => ({ value: k, label: `${k} — ${v}` })),
|
|
964
|
+
});
|
|
965
|
+
if (p.isCancel(severity)) return;
|
|
966
|
+
const description = await p.text({ message: 'Description (optional):', placeholder: 'Steps to reproduce…' });
|
|
967
|
+
bugs.push({ id: `BUG-${shortId()}`, title: String(title), severity: String(severity), status: 'OPEN', description: p.isCancel(description) ? '' : description, createdAt: timestamp() });
|
|
968
|
+
console.log(chalk.green(` ✓ Bug logged as ${colorSeverity(String(severity))}`));
|
|
969
|
+
}
|
|
649
970
|
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
971
|
+
async function createAndRunTestInteractive(runner, results, dashboard) {
|
|
972
|
+
const name = await p.text({ message: 'Test name:' });
|
|
973
|
+
if (p.isCancel(name)) return;
|
|
974
|
+
const type = await p.select({ message: 'Test type:', options: TEST_TYPES.map(t => ({ value: t, label: t })) });
|
|
975
|
+
if (p.isCancel(type)) return;
|
|
976
|
+
const expectPass = await p.confirm({ message: 'Should this test pass?' });
|
|
977
|
+
|
|
978
|
+
dashboard.start();
|
|
979
|
+
const test = {
|
|
980
|
+
id: shortId(), name: String(name), type: String(type),
|
|
981
|
+
fn: async () => {
|
|
982
|
+
await sleep(400 + Math.random() * 300);
|
|
983
|
+
if (!expectPass) throw new Error('Test manually marked as failure');
|
|
984
|
+
},
|
|
663
985
|
};
|
|
986
|
+
const [result] = await runner.run([test], dashboard);
|
|
987
|
+
results.push(result);
|
|
988
|
+
dashboard.stop();
|
|
989
|
+
console.log(` ${colorStatus(result.status)} ${result.name} ${chalk.gray(formatDuration(result.duration))}`);
|
|
990
|
+
}
|
|
664
991
|
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
if (
|
|
668
|
-
|
|
669
|
-
|
|
992
|
+
async function runSavedSuiteInteractive(runner, results, dashboard) {
|
|
993
|
+
const suiteFiles = await fs.readdir(QA_DIR).then(files => files.filter(f => f.endsWith('.suite.json'))).catch(() => []);
|
|
994
|
+
if (!suiteFiles.length) { console.log(chalk.yellow(' No saved suites found.')); return; }
|
|
995
|
+
const chosen = await p.select({ message: 'Select suite:', options: suiteFiles.map(f => ({ value: f, label: f })) });
|
|
996
|
+
if (p.isCancel(chosen)) return;
|
|
997
|
+
const suite = await fs.readJson(path.join(QA_DIR, String(chosen)));
|
|
998
|
+
const tests = (suite.tests ?? []).map(t => ({ ...t, fn: async () => { await sleep(200); if (t.shouldFail) throw new Error('Marked as expected failure'); } }));
|
|
999
|
+
dashboard.start();
|
|
1000
|
+
const runResults = await runner.run(tests, dashboard);
|
|
1001
|
+
results.push(...runResults);
|
|
1002
|
+
dashboard.stop();
|
|
1003
|
+
printResultsSummary(runResults);
|
|
1004
|
+
}
|
|
670
1005
|
|
|
671
|
-
|
|
672
|
-
|
|
1006
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
1007
|
+
// Automated QA Flow
|
|
1008
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
673
1009
|
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
{ value: 'html', label: '🌐 HTML' },
|
|
679
|
-
{ value: 'skip', label: '⏭️ Skip' },
|
|
680
|
-
],
|
|
681
|
-
});
|
|
682
|
-
if (!p.isCancel(exportFmt) && exportFmt !== 'skip') await exportReport(result, exportFmt);
|
|
1010
|
+
export async function runAutomatedQA({ continuous = false } = {}) {
|
|
1011
|
+
const runOnce = async () => {
|
|
1012
|
+
const runId = `AQA-${shortId()}`;
|
|
1013
|
+
const startedAt = timestamp();
|
|
683
1014
|
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
};
|
|
1015
|
+
console.log('');
|
|
1016
|
+
console.log(chalk.hex('#BF40FF').bold(` ── 🤖 Automated QA Run ${runId} ──`));
|
|
1017
|
+
console.log('');
|
|
688
1018
|
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
1019
|
+
// Try to get endpoints from analyzer
|
|
1020
|
+
let endpoints = [];
|
|
1021
|
+
try {
|
|
1022
|
+
const { analyzeFrontend } = await import('../analyzer.js');
|
|
1023
|
+
endpoints = await analyzeFrontend(path.join(process.cwd(), 'src'));
|
|
1024
|
+
} catch {}
|
|
1025
|
+
|
|
1026
|
+
const allTests = [
|
|
1027
|
+
...buildFullSystemTests(),
|
|
1028
|
+
...buildEndpointTests(endpoints),
|
|
1029
|
+
...buildUITests(),
|
|
1030
|
+
];
|
|
1031
|
+
|
|
1032
|
+
console.log(chalk.gray(` Building test suite: ${allTests.length} tests across ${new Set(allTests.map(t => t.type)).size} categories\n`));
|
|
1033
|
+
|
|
1034
|
+
const dashboard = new LiveDashboard();
|
|
1035
|
+
const runner = new TestRunner();
|
|
1036
|
+
const autoBugs = [];
|
|
1037
|
+
|
|
1038
|
+
runner.on('result', r => {
|
|
1039
|
+
if (r.status === 'FAIL') {
|
|
1040
|
+
autoBugs.push({
|
|
1041
|
+
id : `AUTO-${shortId()}`,
|
|
1042
|
+
title : `Automated: ${r.name}`,
|
|
1043
|
+
severity: r.type === 'security' || r.type === 'auth' ? 'P0' : r.type === 'e2e' ? 'P1' : 'P2',
|
|
1044
|
+
status : 'OPEN',
|
|
1045
|
+
description: r.error || '',
|
|
1046
|
+
createdAt: timestamp(),
|
|
1047
|
+
});
|
|
1048
|
+
}
|
|
692
1049
|
});
|
|
693
1050
|
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
const exportFmt = await p.select({
|
|
699
|
-
message: 'Export report as:',
|
|
700
|
-
options: [
|
|
701
|
-
{ value: 'json', label: '📄 JSON' },
|
|
702
|
-
{ value: 'html', label: '🌐 HTML' },
|
|
703
|
-
{ value: 'skip', label: '⏭️ Skip export' },
|
|
704
|
-
],
|
|
705
|
-
});
|
|
706
|
-
if (!p.isCancel(exportFmt) && exportFmt !== 'skip') await exportReport(result, exportFmt);
|
|
1051
|
+
dashboard.start();
|
|
1052
|
+
const results = await runner.run(allTests, dashboard);
|
|
1053
|
+
dashboard.stop();
|
|
707
1054
|
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
1055
|
+
const duration = Date.now() - new Date(startedAt).getTime();
|
|
1056
|
+
const summary = buildSummary(results);
|
|
1057
|
+
const coverage = buildCoverageMatrix(results);
|
|
711
1058
|
|
|
712
|
-
|
|
713
|
-
// Report Exporter (JSON / HTML)
|
|
714
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
1059
|
+
printResultsSummary(results);
|
|
715
1060
|
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
const baseName = `${data.sessionId || data.runId || 'qa-report'}-${timestamp}`;
|
|
1061
|
+
const run = { id: runId, type: 'automated', startedAt, duration, results, bugReports: autoBugs, summary, coverage };
|
|
1062
|
+
await saveRun(run);
|
|
1063
|
+
const reportFile = await exportReport(run);
|
|
720
1064
|
|
|
721
|
-
|
|
1065
|
+
if (reportFile) console.log(chalk.gray(` 📄 Report: ${reportFile}`));
|
|
1066
|
+
await printRunDiff(run);
|
|
722
1067
|
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
await fs.writeJson(outPath, data, { spaces: 2 });
|
|
727
|
-
spinner.succeed(chalk.green(`JSON report saved: ${outPath}`));
|
|
728
|
-
} else if (format === 'html') {
|
|
729
|
-
const html = buildHtmlReport(data);
|
|
730
|
-
const outPath = path.join(QA_REPORTS_DIR, `${baseName}.html`);
|
|
731
|
-
await fs.writeFile(outPath, html, 'utf-8');
|
|
732
|
-
spinner.succeed(chalk.green(`HTML report saved: ${outPath}`));
|
|
733
|
-
}
|
|
734
|
-
} catch (err) {
|
|
735
|
-
spinner.fail(chalk.red(`Export failed: ${err.message}`));
|
|
736
|
-
}
|
|
737
|
-
}
|
|
1068
|
+
p.outro(chalk.hex('#00F5FF').bold(`Run ${runId} complete — ${formatDuration(duration)}`));
|
|
1069
|
+
return run;
|
|
1070
|
+
};
|
|
738
1071
|
|
|
739
|
-
|
|
740
|
-
const filled = Math.round((pct / 100) * width);
|
|
741
|
-
const empty = width - filled;
|
|
742
|
-
const bar = chalk.green('█'.repeat(filled)) + chalk.gray('░'.repeat(empty));
|
|
743
|
-
return `[${bar}]`;
|
|
744
|
-
}
|
|
1072
|
+
if (!continuous) { await runOnce(); return; }
|
|
745
1073
|
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
return `Fix: ${failedTest.error || 'Review test output above.'}`;
|
|
1074
|
+
console.log(chalk.cyan(` ⚡ Continuous QA mode — reruns every ${WATCH_INTERVAL_MS / 1000}s. Press Ctrl+C to stop.\n`));
|
|
1075
|
+
let iteration = 0;
|
|
1076
|
+
while (true) {
|
|
1077
|
+
iteration++;
|
|
1078
|
+
console.log(chalk.gray(`\n ── Iteration ${iteration} ── ${new Date().toLocaleTimeString()}`));
|
|
1079
|
+
await runOnce();
|
|
1080
|
+
await sleep(WATCH_INTERVAL_MS);
|
|
1081
|
+
}
|
|
755
1082
|
}
|
|
756
1083
|
|
|
757
|
-
// ──
|
|
758
|
-
function buildHtmlReport(data) {
|
|
759
|
-
const isManual = data.mode === 'manual';
|
|
760
|
-
const title = isManual ? 'Manual QA Report' : 'Automated QA Report';
|
|
761
|
-
const passColor = '#00F5FF';
|
|
762
|
-
const failColor = '#FF6B6B';
|
|
763
|
-
const bgColor = '#0D0D1A';
|
|
764
|
-
|
|
765
|
-
const manualRows = isManual
|
|
766
|
-
? (data.testCases || []).map((tc) => `
|
|
767
|
-
<tr>
|
|
768
|
-
<td>${tc.id}</td>
|
|
769
|
-
<td>${escHtml(tc.title)}</td>
|
|
770
|
-
<td><span class="badge badge-${tc.result}">${tc.result.toUpperCase()}</span></td>
|
|
771
|
-
<td>${escHtml(tc.steps.join(' → '))}</td>
|
|
772
|
-
<td>${escHtml(tc.expected)}</td>
|
|
773
|
-
<td>${tc.bugReportId || '—'}</td>
|
|
774
|
-
</tr>`).join('')
|
|
775
|
-
: '';
|
|
776
|
-
|
|
777
|
-
const bugRows = isManual
|
|
778
|
-
? (data.bugReports || []).map((br) => `
|
|
779
|
-
<tr>
|
|
780
|
-
<td>${br.id}</td>
|
|
781
|
-
<td>${escHtml(br.title)}</td>
|
|
782
|
-
<td><span class="badge badge-${br.severity}">${br.severity.toUpperCase()}</span></td>
|
|
783
|
-
<td>${escHtml(br.actual)}</td>
|
|
784
|
-
</tr>`).join('')
|
|
785
|
-
: '';
|
|
786
|
-
|
|
787
|
-
const autoSuites = !isManual
|
|
788
|
-
? (data.suites || []).map((s) => `
|
|
789
|
-
<div class="suite">
|
|
790
|
-
<h3>${escHtml(s.suiteName)} <span class="suite-time">${s.duration}ms</span></h3>
|
|
791
|
-
<table>
|
|
792
|
-
<thead><tr><th>ID</th><th>Test</th><th>Status</th><th>Duration</th><th>Message</th></tr></thead>
|
|
793
|
-
<tbody>
|
|
794
|
-
${s.results.map((r) => `
|
|
795
|
-
<tr>
|
|
796
|
-
<td>${r.id}</td>
|
|
797
|
-
<td>${escHtml(r.name)}</td>
|
|
798
|
-
<td><span class="badge badge-${r.status}">${r.status.toUpperCase()}</span></td>
|
|
799
|
-
<td>${r.duration}ms</td>
|
|
800
|
-
<td>${escHtml(r.message || r.error || '')}</td>
|
|
801
|
-
</tr>`).join('')}
|
|
802
|
-
</tbody>
|
|
803
|
-
</table>
|
|
804
|
-
</div>`).join('')
|
|
805
|
-
: '';
|
|
806
|
-
|
|
807
|
-
return `<!DOCTYPE html>
|
|
808
|
-
<html lang="en">
|
|
809
|
-
<head>
|
|
810
|
-
<meta charset="UTF-8">
|
|
811
|
-
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
812
|
-
<title>Backlist QA — ${title}</title>
|
|
813
|
-
<style>
|
|
814
|
-
@import url('https://fonts.googleapis.com/css2?family=JetBrains+Mono:wght@400;700&family=Space+Grotesk:wght@400;600;700&display=swap');
|
|
815
|
-
*{box-sizing:border-box;margin:0;padding:0}
|
|
816
|
-
body{background:${bgColor};color:#e0e0ff;font-family:'Space Grotesk',sans-serif;padding:2rem}
|
|
817
|
-
h1{font-family:'JetBrains Mono',monospace;color:${passColor};font-size:1.8rem;margin-bottom:0.25rem}
|
|
818
|
-
h2{color:#BF40FF;font-size:1.2rem;margin:2rem 0 1rem;border-bottom:1px solid #2a2a4a;padding-bottom:0.5rem}
|
|
819
|
-
h3{color:#e0e0ff;font-size:1rem;margin:1.5rem 0 0.5rem}
|
|
820
|
-
.meta{color:#666;font-size:0.85rem;margin-bottom:2rem;font-family:'JetBrains Mono',monospace}
|
|
821
|
-
.stats{display:flex;gap:1.5rem;margin:1.5rem 0;flex-wrap:wrap}
|
|
822
|
-
.stat{background:#111128;border:1px solid #2a2a4a;border-radius:8px;padding:1rem 1.5rem;min-width:120px}
|
|
823
|
-
.stat-label{font-size:0.75rem;color:#666;text-transform:uppercase;letter-spacing:1px}
|
|
824
|
-
.stat-value{font-size:2rem;font-family:'JetBrains Mono',monospace;font-weight:700;margin-top:0.25rem}
|
|
825
|
-
.stat-pass .stat-value{color:${passColor}}
|
|
826
|
-
.stat-fail .stat-value{color:${failColor}}
|
|
827
|
-
.stat-total .stat-value{color:#BF40FF}
|
|
828
|
-
table{width:100%;border-collapse:collapse;margin:0.5rem 0;font-size:0.875rem}
|
|
829
|
-
th{text-align:left;padding:0.6rem 0.75rem;background:#111128;color:#666;font-weight:600;font-size:0.75rem;text-transform:uppercase;letter-spacing:0.5px}
|
|
830
|
-
td{padding:0.6rem 0.75rem;border-bottom:1px solid #1a1a2e;vertical-align:top;max-width:300px;overflow:hidden;text-overflow:ellipsis}
|
|
831
|
-
tr:hover td{background:#111128}
|
|
832
|
-
.badge{padding:0.2rem 0.6rem;border-radius:4px;font-size:0.75rem;font-family:'JetBrains Mono',monospace;font-weight:700}
|
|
833
|
-
.badge-pass,.badge-info{background:#003322;color:${passColor}}
|
|
834
|
-
.badge-fail,.badge-critical,.badge-high{background:#330011;color:${failColor}}
|
|
835
|
-
.badge-medium{background:#332200;color:#FFB347}
|
|
836
|
-
.badge-low{background:#001133;color:${passColor}}
|
|
837
|
-
.badge-skip{background:#2a2a2a;color:#888}
|
|
838
|
-
.suite{background:#0a0a18;border:1px solid #1a1a2e;border-radius:8px;padding:1rem 1.5rem;margin-bottom:1rem}
|
|
839
|
-
.suite h3{color:#BF40FF;font-family:'JetBrains Mono',monospace}
|
|
840
|
-
.suite-time{color:#444;font-size:0.75rem;font-weight:400}
|
|
841
|
-
footer{margin-top:3rem;color:#333;font-size:0.75rem;font-family:'JetBrains Mono',monospace;text-align:center}
|
|
842
|
-
</style>
|
|
843
|
-
</head>
|
|
844
|
-
<body>
|
|
845
|
-
<h1>⚡ Backlist QA — ${title}</h1>
|
|
846
|
-
<div class="meta">Generated: ${new Date().toLocaleString()} | ID: ${escHtml(data.sessionId || data.runId || 'N/A')}</div>
|
|
847
|
-
|
|
848
|
-
<h2>Summary</h2>
|
|
849
|
-
<div class="stats">
|
|
850
|
-
<div class="stat stat-total"><div class="stat-label">Total</div><div class="stat-value">${data.totalTests || (isManual ? data.testCases?.length : 0) || 0}</div></div>
|
|
851
|
-
<div class="stat stat-pass"><div class="stat-label">Passed</div><div class="stat-value">${data.passed}</div></div>
|
|
852
|
-
<div class="stat stat-fail"><div class="stat-label">Failed</div><div class="stat-value">${data.failed}</div></div>
|
|
853
|
-
${!isManual ? `<div class="stat"><div class="stat-label">Pass Rate</div><div class="stat-value" style="color:${parseFloat(data.passRate)>=80?passColor:failColor}">${data.passRate}%</div></div>` : ''}
|
|
854
|
-
${data.elapsed || data.duration ? `<div class="stat"><div class="stat-label">Duration</div><div class="stat-value" style="color:#888;font-size:1.2rem">${data.elapsed || data.duration}${isManual ? 's' : 'ms'}</div></div>` : ''}
|
|
855
|
-
</div>
|
|
1084
|
+
// ── Auto-run hook (called after generation) ───────────────────────────────
|
|
856
1085
|
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
</table>` : ''}
|
|
1086
|
+
export async function autoRunPostGeneration(options = {}) {
|
|
1087
|
+
console.log('');
|
|
1088
|
+
console.log(chalk.hex('#00F5FF').bold(' ── 🔬 Post-Generation QA Scan ──────────────────────'));
|
|
1089
|
+
console.log(chalk.gray(` Automatically validating generated project: ${options.projectName || 'backend'}`));
|
|
1090
|
+
console.log('');
|
|
863
1091
|
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
1092
|
+
const projectDir = options.projectDir || process.cwd();
|
|
1093
|
+
const tests = buildFullSystemTests(projectDir);
|
|
1094
|
+
const runner = new TestRunner();
|
|
1095
|
+
const dashboard = new LiveDashboard();
|
|
1096
|
+
const autoBugs = [];
|
|
1097
|
+
|
|
1098
|
+
runner.on('result', r => {
|
|
1099
|
+
if (r.status === 'FAIL') {
|
|
1100
|
+
autoBugs.push({
|
|
1101
|
+
id: `POST-${shortId()}`, title: r.name,
|
|
1102
|
+
severity: r.type === 'security' ? 'P0' : r.type === 'auth' ? 'P0' : 'P2',
|
|
1103
|
+
status: 'OPEN', description: r.error || '', createdAt: timestamp(),
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
1106
|
+
});
|
|
870
1107
|
|
|
871
|
-
|
|
1108
|
+
dashboard.start();
|
|
1109
|
+
const results = await runner.run(tests, dashboard);
|
|
1110
|
+
dashboard.stop();
|
|
1111
|
+
|
|
1112
|
+
const summary = buildSummary(results);
|
|
1113
|
+
const coverage = buildCoverageMatrix(results);
|
|
1114
|
+
const run = {
|
|
1115
|
+
id : `POST-${shortId()}`,
|
|
1116
|
+
type : 'post-generation',
|
|
1117
|
+
startedAt: timestamp(),
|
|
1118
|
+
duration : 0,
|
|
1119
|
+
results,
|
|
1120
|
+
bugReports: autoBugs,
|
|
1121
|
+
summary,
|
|
1122
|
+
coverage,
|
|
1123
|
+
};
|
|
872
1124
|
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
</html>`;
|
|
876
|
-
}
|
|
1125
|
+
await saveRun(run);
|
|
1126
|
+
const reportFile = await exportReport(run);
|
|
877
1127
|
|
|
878
|
-
|
|
879
|
-
|
|
1128
|
+
printResultsSummary(results);
|
|
1129
|
+
if (autoBugs.length > 0) {
|
|
1130
|
+
console.log(chalk.red.bold(` ⚠ ${autoBugs.length} issue(s) auto-detected:`));
|
|
1131
|
+
autoBugs.forEach(b => console.log(chalk.red(` ${colorSeverity(b.severity)} ${b.title}`)));
|
|
1132
|
+
console.log('');
|
|
1133
|
+
}
|
|
1134
|
+
if (reportFile) console.log(chalk.gray(` 📄 Post-gen report: ${reportFile}`));
|
|
880
1135
|
}
|
|
881
1136
|
|
|
882
|
-
//
|
|
883
|
-
// QA History Viewer
|
|
884
|
-
// ═══════════════════════════════════════════════════════════════════════════
|
|
1137
|
+
// ── QA History ─────────────────────────────────────────────────────────────
|
|
885
1138
|
|
|
886
1139
|
export async function viewQAHistory() {
|
|
887
|
-
await
|
|
888
|
-
|
|
1140
|
+
const hist = await loadHistory();
|
|
1141
|
+
if (!hist.runs.length) { console.log(chalk.yellow('\n No QA history found.\n')); return; }
|
|
889
1142
|
|
|
890
1143
|
console.log('');
|
|
891
|
-
console.log(chalk.hex('#00F5FF').bold('
|
|
892
|
-
console.log(chalk.
|
|
893
|
-
|
|
1144
|
+
console.log(chalk.hex('#00F5FF').bold(' QA History (most recent first)'));
|
|
1145
|
+
console.log(chalk.gray(' ────────────────────────────────────────────────────'));
|
|
1146
|
+
|
|
1147
|
+
for (const run of hist.runs.slice(0, 10)) {
|
|
1148
|
+
const passRate = run.summary.total ? ((run.summary.passed / run.summary.total) * 100).toFixed(0) : '–';
|
|
1149
|
+
const rateColor = Number(passRate) >= 90 ? chalk.green : Number(passRate) >= 70 ? chalk.yellow : chalk.red;
|
|
1150
|
+
console.log(
|
|
1151
|
+
` ${chalk.gray(run.id.padEnd(18))}` +
|
|
1152
|
+
` ${chalk.gray(new Date(run.startedAt).toLocaleString().padEnd(24))}` +
|
|
1153
|
+
` ${rateColor(`${passRate}%`.padStart(5))}` +
|
|
1154
|
+
` ${chalk.gray(`${run.summary.total} tests · ${formatDuration(run.duration)}`)}`
|
|
1155
|
+
);
|
|
1156
|
+
}
|
|
894
1157
|
console.log('');
|
|
895
1158
|
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
1159
|
+
const chosen = await p.select({
|
|
1160
|
+
message: 'View a run in detail?',
|
|
1161
|
+
options: [
|
|
1162
|
+
...hist.runs.slice(0, 5).map(r => ({ value: r.id, label: `${r.id} — ${new Date(r.startedAt).toLocaleString()}` })),
|
|
1163
|
+
{ value: '__back', label: '↩ Back' },
|
|
1164
|
+
],
|
|
1165
|
+
});
|
|
1166
|
+
if (p.isCancel(chosen) || chosen === '__back') return;
|
|
1167
|
+
|
|
1168
|
+
const run = hist.runs.find(r => r.id === chosen);
|
|
1169
|
+
if (!run) return;
|
|
901
1170
|
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
1171
|
+
console.log('');
|
|
1172
|
+
console.log(chalk.bold(` Run: ${run.id} (${run.type})`));
|
|
1173
|
+
console.log(chalk.gray(` ${new Date(run.startedAt).toLocaleString()} · ${formatDuration(run.duration)}`));
|
|
1174
|
+
console.log('');
|
|
1175
|
+
for (const r of run.results) {
|
|
1176
|
+
console.log(` ${colorStatus(r.status)} ${r.name} ${chalk.gray(formatDuration(r.duration))}`);
|
|
1177
|
+
if (r.error) console.log(chalk.red(` ↳ ${r.error}`));
|
|
1178
|
+
}
|
|
1179
|
+
if (run.bugReports?.length) {
|
|
907
1180
|
console.log('');
|
|
908
|
-
|
|
909
|
-
|
|
1181
|
+
console.log(chalk.bold(' Bug Reports:'));
|
|
1182
|
+
for (const b of run.bugReports) {
|
|
1183
|
+
console.log(` ${colorSeverity(b.severity)} ${b.title} ${chalk.gray(`[${b.status}]`)}`);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
console.log('');
|
|
1187
|
+
}
|