@ai-qa/workflow 2.0.11 → 2.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/agents/playwright-test-generator.agent.md +49 -0
- package/.github/agents/playwright-test-healer.agent.md +32 -3
- package/.github/agents/playwright-test-planner.agent.md +26 -0
- package/.github/copilot-instructions.md +44 -2
- package/.opencode/agents/qa-generator.md +16 -0
- package/.opencode/agents/qa-healer.md +18 -0
- package/.opencode/agents/qa-planner.md +17 -0
- package/.opencode/rules.md +66 -2
- package/.qa-context/auth.json +29 -0
- package/.qa-context/heal-history.json +40 -0
- package/.qa-context/pipeline.json +34 -0
- package/.qa-context/selectors.json +64 -0
- package/.qa-context/traceability.json +30 -0
- package/README.md +399 -196
- package/ai-qa-workflow.js +82 -104
- package/install.js +7 -12
- package/package.json +5 -6
- package/prompting_template.md +283 -0
- package/qa-dashboard/app.js +1 -0
- package/qa-dashboard/routes/review.js +114 -0
- package/qa-dashboard/views/layouts/main.ejs +1 -0
- package/qa-dashboard/views/review.ejs +201 -0
- package/router.md +109 -29
- package/scripts/auth-manager.js +186 -0
- package/scripts/context-manager.js +226 -0
- package/scripts/executor.js +18 -7
- package/scripts/generator.js +18 -124
- package/scripts/healer.js +78 -157
- package/scripts/planner.js +18 -136
- package/scripts/reporter.js +21 -1
- package/scripts/utils.js +2 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
const express = require('express');
|
|
2
|
+
const router = express.Router();
|
|
3
|
+
const pm = require('../services/project-manager');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const fs = require('fs');
|
|
6
|
+
|
|
7
|
+
router.get('/', (req, res) => {
|
|
8
|
+
const projectId = req.query.project;
|
|
9
|
+
const project = projectId ? pm.get(projectId) : pm.getAll()[0];
|
|
10
|
+
if (!project) return res.redirect('/projects');
|
|
11
|
+
|
|
12
|
+
const bridge = pm.getBridge(project.id);
|
|
13
|
+
const allFiles = bridge.listAll();
|
|
14
|
+
const runDirs = bridge.getRunDirs();
|
|
15
|
+
|
|
16
|
+
const runs = runDirs.map(d => {
|
|
17
|
+
const result = bridge.getRunResult(d.name);
|
|
18
|
+
const healing = bridge.getHealingReport(d.name);
|
|
19
|
+
return {
|
|
20
|
+
id: d.name,
|
|
21
|
+
timestamp: d.name.replace('run-', '').replace(/-/g, ':').slice(0, 19),
|
|
22
|
+
success: result ? result.success : null,
|
|
23
|
+
duration: result ? result.duration : null,
|
|
24
|
+
testName: result ? result.test : 'Unknown',
|
|
25
|
+
failedCount: result && Array.isArray(result.failedTests) ? result.failedTests.length : 0,
|
|
26
|
+
passedCount: result && Array.isArray(result.passedTests) ? result.passedTests.length : 0,
|
|
27
|
+
healedCount: healing ? healing.totalHealed || 0 : 0,
|
|
28
|
+
hasReport: !!bridge.getFinalReport(d.name),
|
|
29
|
+
};
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const latestRun = runs.length > 0 ? runs[0] : null;
|
|
33
|
+
|
|
34
|
+
const allDefects = [];
|
|
35
|
+
for (const d of runDirs) {
|
|
36
|
+
const result = bridge.getRunResult(d.name);
|
|
37
|
+
if (result && Array.isArray(result.failedTests)) {
|
|
38
|
+
for (const ft of result.failedTests) {
|
|
39
|
+
allDefects.push({
|
|
40
|
+
runId: d.name,
|
|
41
|
+
testName: typeof ft === 'string' ? ft : (ft.name || ft.test || 'Unknown'),
|
|
42
|
+
error: typeof ft === 'string' ? '' : (ft.error || ft.message || ''),
|
|
43
|
+
runTimestamp: d.name.replace('run-', '').replace(/-/g, ':').slice(0, 19),
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const totalPassed = runs.reduce((s, r) => s + r.passedCount, 0);
|
|
50
|
+
const totalFailed = runs.reduce((s, r) => s + r.failedCount, 0);
|
|
51
|
+
const totalHealed = runs.reduce((s, r) => s + r.healedCount, 0);
|
|
52
|
+
const totalRuns = runs.length;
|
|
53
|
+
|
|
54
|
+
res.render('review', {
|
|
55
|
+
project,
|
|
56
|
+
stats: {
|
|
57
|
+
stories: allFiles.stories.length,
|
|
58
|
+
plans: allFiles.plans.length,
|
|
59
|
+
specs: allFiles.specs.length,
|
|
60
|
+
runs: totalRuns,
|
|
61
|
+
passed: totalPassed,
|
|
62
|
+
failed: totalFailed,
|
|
63
|
+
healed: totalHealed,
|
|
64
|
+
},
|
|
65
|
+
latestRun,
|
|
66
|
+
runs,
|
|
67
|
+
defects: allDefects,
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
router.get('/:runId', (req, res) => {
|
|
72
|
+
const projectId = req.query.project;
|
|
73
|
+
const project = projectId ? pm.get(projectId) : pm.getAll()[0];
|
|
74
|
+
if (!project) return res.redirect('/projects');
|
|
75
|
+
|
|
76
|
+
const bridge = pm.getBridge(project.id);
|
|
77
|
+
const runDirs = bridge.getRunDirs();
|
|
78
|
+
const runExists = runDirs.some(d => d.name === req.params.runId);
|
|
79
|
+
if (!runExists) return res.status(404).render('error', { message: 'Review: run not found' });
|
|
80
|
+
|
|
81
|
+
let result = bridge.getRunResult(req.params.runId);
|
|
82
|
+
if (!result) {
|
|
83
|
+
const outputPath = path.join(bridge.projectPath, 'test-results', req.params.runId, 'execution-output.json');
|
|
84
|
+
let outputText = 'No console output captured yet.';
|
|
85
|
+
if (fs.existsSync(outputPath)) {
|
|
86
|
+
try { outputText = fs.readFileSync(outputPath, 'utf-8'); } catch (err) {}
|
|
87
|
+
}
|
|
88
|
+
result = {
|
|
89
|
+
runId: req.params.runId,
|
|
90
|
+
success: null,
|
|
91
|
+
duration: null,
|
|
92
|
+
test: 'Running / In Progress...',
|
|
93
|
+
timestamp: new Date().toISOString(),
|
|
94
|
+
failedTests: [],
|
|
95
|
+
passedTests: [],
|
|
96
|
+
output: outputText,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const healing = bridge.getHealingReport(req.params.runId);
|
|
101
|
+
const report = bridge.getFinalReport(req.params.runId);
|
|
102
|
+
|
|
103
|
+
res.render('review', {
|
|
104
|
+
project,
|
|
105
|
+
run: {
|
|
106
|
+
id: req.params.runId,
|
|
107
|
+
result,
|
|
108
|
+
healing,
|
|
109
|
+
report,
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
module.exports = router;
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
<a href="/projects">Projects</a>
|
|
18
18
|
<a href="/stories">Stories</a>
|
|
19
19
|
<a href="/runs">Runs</a>
|
|
20
|
+
<a href="/review">Review</a>
|
|
20
21
|
<a href="/analytics">Analytics</a>
|
|
21
22
|
<a href="/data">Test Data</a>
|
|
22
23
|
<button onclick="location.reload()" class="btn-refresh" title="Refresh">↻</button>
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
<% if (typeof run !== 'undefined') { %>
|
|
2
|
+
<%# ——— SINGLE RUN REVIEW ——— %>
|
|
3
|
+
<div class="page-header">
|
|
4
|
+
<div class="page-header-left">
|
|
5
|
+
<h1><%= project.name %> — Review</h1>
|
|
6
|
+
<p class="subtitle">Run: <%= run.id %></p>
|
|
7
|
+
</div>
|
|
8
|
+
<div class="page-header-right">
|
|
9
|
+
<a href="/review?project=<%= project.id %>" class="btn">← Back to Overview</a>
|
|
10
|
+
</div>
|
|
11
|
+
</div>
|
|
12
|
+
|
|
13
|
+
<%# Run Summary %>
|
|
14
|
+
<div class="card">
|
|
15
|
+
<h3>Run Summary</h3>
|
|
16
|
+
<table class="table">
|
|
17
|
+
<tr><td>Status</td><td>
|
|
18
|
+
<% if (run.result.success === true) { %><span class="badge badge-success">Passed</span>
|
|
19
|
+
<% } else if (run.result.success === false) { %><span class="badge badge-danger">Failed</span>
|
|
20
|
+
<% } else { %><span class="badge badge-warning">In Progress</span><% } %>
|
|
21
|
+
</td></tr>
|
|
22
|
+
<tr><td>Duration</td><td><%= run.result.duration || '—' %></td></tr>
|
|
23
|
+
<tr><td>Test Suite</td><td><%= run.result.test || '—' %></td></tr>
|
|
24
|
+
<tr><td>Timestamp</td><td><%= run.result.timestamp ? new Date(run.result.timestamp).toLocaleString() : '—' %></td></tr>
|
|
25
|
+
<tr><td>Failures</td><td><%= Array.isArray(run.result.failedTests) ? run.result.failedTests.length : 0 %></td></tr>
|
|
26
|
+
</table>
|
|
27
|
+
</div>
|
|
28
|
+
|
|
29
|
+
<%# Test Results %>
|
|
30
|
+
<div class="card">
|
|
31
|
+
<h3>Test Results</h3>
|
|
32
|
+
<% if (Array.isArray(run.result.passedTests) && run.result.passedTests.length > 0) { %>
|
|
33
|
+
<h4>Passed (<%= run.result.passedTests.length %>)</h4>
|
|
34
|
+
<ul>
|
|
35
|
+
<% run.result.passedTests.forEach(function(t) { %>
|
|
36
|
+
<li style="color: var(--success)"><%= typeof t === 'string' ? t : (t.name || t.test || JSON.stringify(t)) %></li>
|
|
37
|
+
<% }); %>
|
|
38
|
+
</ul>
|
|
39
|
+
<% } %>
|
|
40
|
+
<% if (Array.isArray(run.result.failedTests) && run.result.failedTests.length > 0) { %>
|
|
41
|
+
<h4>Failed (<%= run.result.failedTests.length %>)</h4>
|
|
42
|
+
<% run.result.failedTests.forEach(function(ft) { %>
|
|
43
|
+
<div class="alert alert-error" style="margin-bottom: 8px;">
|
|
44
|
+
<strong><%= typeof ft === 'string' ? ft : (ft.name || ft.test || 'Unknown') %></strong>
|
|
45
|
+
<% if (typeof ft === 'object' && ft.error) { %>
|
|
46
|
+
<pre style="margin: 4px 0 0; font-size: 13px;"><%= ft.error %></pre>
|
|
47
|
+
<% } %>
|
|
48
|
+
</div>
|
|
49
|
+
<% }); %>
|
|
50
|
+
<% } %>
|
|
51
|
+
<% if ((!run.result.passedTests || run.result.passedTests.length === 0) && (!run.result.failedTests || run.result.failedTests.length === 0)) { %>
|
|
52
|
+
<p>No test results available.</p>
|
|
53
|
+
<% } %>
|
|
54
|
+
</div>
|
|
55
|
+
|
|
56
|
+
<%# Healing Report %>
|
|
57
|
+
<% if (run.healing) { %>
|
|
58
|
+
<div class="card">
|
|
59
|
+
<h3>Self-Healing Report</h3>
|
|
60
|
+
<table class="table">
|
|
61
|
+
<tr><td>Total Healed</td><td><%= run.healing.totalHealed || 0 %></td></tr>
|
|
62
|
+
<tr><td>Total Attempts</td><td><%= run.healing.totalAttempts || 0 %></td></tr>
|
|
63
|
+
<tr><td>Remaining Failures</td><td><%= run.healing.remainingFailures || 0 %></td></tr>
|
|
64
|
+
</table>
|
|
65
|
+
<% if (Array.isArray(run.healing.healedTests) && run.healing.healedTests.length > 0) { %>
|
|
66
|
+
<h4>Healed Tests</h4>
|
|
67
|
+
<% run.healing.healedTests.forEach(function(ht) { %>
|
|
68
|
+
<div class="card" style="margin-bottom: 8px; padding: 12px;">
|
|
69
|
+
<strong><%= ht.test || ht.name || 'Unknown' %></strong>
|
|
70
|
+
<% if (ht.originalSelector) { %><p>Original: <code><%= ht.originalSelector %></code></p><% } %>
|
|
71
|
+
<% if (ht.newSelector) { %><p>Healed: <code><%= ht.newSelector %></code></p><% } %>
|
|
72
|
+
<% if (ht.status) { %><span class="badge badge-success"><%= ht.status %></span><% } %>
|
|
73
|
+
</div>
|
|
74
|
+
<% }); %>
|
|
75
|
+
<% } %>
|
|
76
|
+
</div>
|
|
77
|
+
<% } %>
|
|
78
|
+
|
|
79
|
+
<%# Final Report %>
|
|
80
|
+
<% if (run.report) { %>
|
|
81
|
+
<div class="card">
|
|
82
|
+
<h3>Final Test Report</h3>
|
|
83
|
+
<div class="markdown-content"><%= run.report %></div>
|
|
84
|
+
</div>
|
|
85
|
+
<% } %>
|
|
86
|
+
|
|
87
|
+
<%# Console Output %>
|
|
88
|
+
<% if (run.result.output) { %>
|
|
89
|
+
<div class="card">
|
|
90
|
+
<h3>Console Output</h3>
|
|
91
|
+
<pre class="output-terminal"><%= run.result.output %></pre>
|
|
92
|
+
</div>
|
|
93
|
+
<% } %>
|
|
94
|
+
|
|
95
|
+
<% } else { %>
|
|
96
|
+
<%# ——— PROJECT REVIEW OVERVIEW ——— %>
|
|
97
|
+
<div class="page-header">
|
|
98
|
+
<div class="page-header-left">
|
|
99
|
+
<h1><%= project.name %> — AI Review</h1>
|
|
100
|
+
<p class="subtitle">Read-only overview of AI-generated artifacts</p>
|
|
101
|
+
</div>
|
|
102
|
+
<div class="page-header-right">
|
|
103
|
+
<a href="/stories?project=<%= project.id %>" class="btn">View Stories</a>
|
|
104
|
+
<a href="/runs?project=<%= project.id %>" class="btn">View Runs</a>
|
|
105
|
+
</div>
|
|
106
|
+
</div>
|
|
107
|
+
|
|
108
|
+
<%# AI Activity Summary %>
|
|
109
|
+
<div class="stats-row">
|
|
110
|
+
<div class="stat-card"><span class="stat-number"><%= stats.stories %></span><span class="stat-label">Stories</span></div>
|
|
111
|
+
<div class="stat-card"><span class="stat-number"><%= stats.plans %></span><span class="stat-label">Test Plans</span></div>
|
|
112
|
+
<div class="stat-card"><span class="stat-number"><%= stats.specs %></span><span class="stat-label">Test Specs</span></div>
|
|
113
|
+
<div class="stat-card"><span class="stat-number"><%= stats.runs %></span><span class="stat-label">Runs</span></div>
|
|
114
|
+
<div class="stat-card"><span class="stat-number"><%= stats.passed %></span><span class="stat-label">Passed</span></div>
|
|
115
|
+
<div class="stat-card"><span class="stat-number"><%= stats.failed %></span><span class="stat-label">Failed</span></div>
|
|
116
|
+
<div class="stat-card"><span class="stat-number"><%= stats.healed %></span><span class="stat-label">Healed</span></div>
|
|
117
|
+
</div>
|
|
118
|
+
|
|
119
|
+
<%# Latest Run %>
|
|
120
|
+
<% if (latestRun) { %>
|
|
121
|
+
<div class="card">
|
|
122
|
+
<h3>Latest Run</h3>
|
|
123
|
+
<table class="table">
|
|
124
|
+
<tr>
|
|
125
|
+
<td>Status</td>
|
|
126
|
+
<td>
|
|
127
|
+
<% if (latestRun.success === true) { %><span class="badge badge-success">Passed</span>
|
|
128
|
+
<% } else if (latestRun.success === false) { %><span class="badge badge-danger">Failed</span>
|
|
129
|
+
<% } else { %><span class="badge badge-warning">Pending</span><% } %>
|
|
130
|
+
</td>
|
|
131
|
+
</tr>
|
|
132
|
+
<tr><td>Test Suite</td><td><%= latestRun.testName %></td></tr>
|
|
133
|
+
<tr><td>Duration</td><td><%= latestRun.duration || '—' %></td></tr>
|
|
134
|
+
<tr><td>Passed / Failed / Healed</td><td><%= latestRun.passedCount %> / <%= latestRun.failedCount %> / <%= latestRun.healedCount %></td></tr>
|
|
135
|
+
<tr><td>Timestamp</td><td><%= latestRun.timestamp %></td></tr>
|
|
136
|
+
<tr><td><a href="/review/<%= latestRun.id %>?project=<%= project.id %>" class="btn">View Details</a></td></tr>
|
|
137
|
+
</table>
|
|
138
|
+
</div>
|
|
139
|
+
<% } else { %>
|
|
140
|
+
<div class="alert alert-info">No runs executed yet.</div>
|
|
141
|
+
<% } %>
|
|
142
|
+
|
|
143
|
+
<%# Defects Found %>
|
|
144
|
+
<% if (defects.length > 0) { %>
|
|
145
|
+
<div class="card">
|
|
146
|
+
<h3>Defects Found (<%= defects.length %>)</h3>
|
|
147
|
+
<table class="table">
|
|
148
|
+
<thead><tr><th>Test</th><th>Run</th><th>Error</th></tr></thead>
|
|
149
|
+
<tbody>
|
|
150
|
+
<% defects.forEach(function(d) { %>
|
|
151
|
+
<tr>
|
|
152
|
+
<td><%= d.testName %></td>
|
|
153
|
+
<td><a href="/review/<%= d.runId %>?project=<%= project.id %>"><%= d.runTimestamp %></a></td>
|
|
154
|
+
<td><code><%= d.error ? d.error.slice(0, 120) : '—' %></code></td>
|
|
155
|
+
</tr>
|
|
156
|
+
<% }); %>
|
|
157
|
+
</tbody>
|
|
158
|
+
</table>
|
|
159
|
+
</div>
|
|
160
|
+
<% } else { %>
|
|
161
|
+
<div class="card">
|
|
162
|
+
<h3>Defects Found</h3>
|
|
163
|
+
<div class="alert alert-info">No defects found.</div>
|
|
164
|
+
</div>
|
|
165
|
+
<% } %>
|
|
166
|
+
|
|
167
|
+
<%# AI Timeline %>
|
|
168
|
+
<% if (runs.length > 0) { %>
|
|
169
|
+
<div class="card">
|
|
170
|
+
<h3>AI Activity Timeline</h3>
|
|
171
|
+
<table class="table">
|
|
172
|
+
<thead><tr><th>Run</th><th>Timestamp</th><th>Suite</th><th>Status</th><th>Duration</th><th>Passed</th><th>Failed</th><th>Healed</th></tr></thead>
|
|
173
|
+
<tbody>
|
|
174
|
+
<% runs.forEach(function(r) { %>
|
|
175
|
+
<tr>
|
|
176
|
+
<td><a href="/review/<%= r.id %>?project=<%= project.id %>"><%= r.id.slice(0, 20) %>…</a></td>
|
|
177
|
+
<td><%= r.timestamp %></td>
|
|
178
|
+
<td><%= r.testName %></td>
|
|
179
|
+
<td>
|
|
180
|
+
<% if (r.success === true) { %><span class="badge badge-success">Passed</span>
|
|
181
|
+
<% } else if (r.success === false) { %><span class="badge badge-danger">Failed</span>
|
|
182
|
+
<% } else { %><span class="badge badge-warning">Pending</span><% } %>
|
|
183
|
+
</td>
|
|
184
|
+
<td><%= r.duration || '—' %></td>
|
|
185
|
+
<td><%= r.passedCount %></td>
|
|
186
|
+
<td><%= r.failedCount %></td>
|
|
187
|
+
<td><%= r.healedCount %></td>
|
|
188
|
+
</tr>
|
|
189
|
+
<% }); %>
|
|
190
|
+
</tbody>
|
|
191
|
+
</table>
|
|
192
|
+
</div>
|
|
193
|
+
<% } %>
|
|
194
|
+
|
|
195
|
+
<% } %>
|
|
196
|
+
|
|
197
|
+
<style>
|
|
198
|
+
.stat-card { cursor: default; }
|
|
199
|
+
.table a { color: var(--brand); text-decoration: none; }
|
|
200
|
+
.table a:hover { text-decoration: underline; }
|
|
201
|
+
</style>
|
package/router.md
CHANGED
|
@@ -1,29 +1,109 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
1
|
+
# AI QA Engineer — Agent Router
|
|
2
|
+
|
|
3
|
+
You are an **AI QA Engineer** assigned to this project.
|
|
4
|
+
Your mission: transform user stories into tested, reliable software through structured QA.
|
|
5
|
+
|
|
6
|
+
You have **Playwright MCP** (browser automation), **Applitools MCP** (visual testing), and **GitHub MCP** (version control) tools.
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## Phase 0: Environment Check (Do This FIRST)
|
|
11
|
+
|
|
12
|
+
When you first open this project, **automatically run this check and report to the user**:
|
|
13
|
+
|
|
14
|
+
1. ✅ **Config**: Does `.qa-workflow.json` exist and have project name + URL configured?
|
|
15
|
+
2. ✅ **Context**: Does `docs/application-context.md` exist with content?
|
|
16
|
+
3. ✅ **Memory**: Does `.qa-context/` exist with pipeline.json, selectors.json, heal-history.json, traceability.json?
|
|
17
|
+
4. ✅ **Auth**: Does `.auth/credentials.json` exist with login credentials? Is `.auth/storage-state.json` valid?
|
|
18
|
+
4. ✅ **Playwright**: Is `@playwright/test` installed? (`npx playwright --version`)
|
|
19
|
+
5. ✅ **Chromium**: Is Chromium installed? (`npx playwright install --dry-run chromium`)
|
|
20
|
+
6. ✅ **MCP**: Is `@playwright/mcp` installed?
|
|
21
|
+
7. ✅ **Directories**: Do `user-story/`, `specs/`, `tests/`, `test-results/` exist?
|
|
22
|
+
8. ✅ **Dev server**: Is the app accessible at the configured URL?
|
|
23
|
+
9. ✅ **Applitools**: Is `APPLITOOLS_API_KEY` env var set? (needed for visual testing)
|
|
24
|
+
10. ✅ **GitHub**: Is `GITHUB_TOKEN` env var set? (needed for GitHub MCP)
|
|
25
|
+
|
|
26
|
+
Present a clean summary:
|
|
27
|
+
- **What's ready** ✅
|
|
28
|
+
- **What's missing** ❌ (with commands to fix)
|
|
29
|
+
- **What you need from the user** 📋 (user story, credentials, etc.)
|
|
30
|
+
- Then **stop and wait** for the user to give you a user story or direction.
|
|
31
|
+
|
|
32
|
+
**Do not skip this check.** The user needs to know what's missing before anything else.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## Phase 1-5: Workflow
|
|
37
|
+
|
|
38
|
+
Once the user provides a user story or testing request, follow this workflow. **Every phase requires human approval before proceeding.**
|
|
39
|
+
|
|
40
|
+
### Phase 1: Plan & Explore
|
|
41
|
+
User says: *"Plan tests for this user story"*
|
|
42
|
+
1. Read `.github/agents/playwright-test-planner.agent.md`
|
|
43
|
+
2. Explore the app with Playwright MCP
|
|
44
|
+
3. Write test plan to `specs/`
|
|
45
|
+
4. **STOP** — present the plan to the user
|
|
46
|
+
5. Wait for approval before continuing
|
|
47
|
+
|
|
48
|
+
### Phase 2: Generate Tests
|
|
49
|
+
User says: *"Generate tests from the plan"*
|
|
50
|
+
1. Read `.github/agents/playwright-test-generator.agent.md`
|
|
51
|
+
2. Read `prompts/QAe2eprompt.md` for conventions
|
|
52
|
+
3. Write Playwright test files to `tests/`
|
|
53
|
+
4. **STOP** — present the generated tests to the user
|
|
54
|
+
5. Wait for approval before executing
|
|
55
|
+
|
|
56
|
+
### Phase 3: Execute
|
|
57
|
+
User says: *"Execute the tests"*
|
|
58
|
+
1. User runs `npm run qa:execute` (or you suggest it)
|
|
59
|
+
2. Check results — did they pass or fail?
|
|
60
|
+
|
|
61
|
+
### Phase 4: Heal Failures
|
|
62
|
+
If tests fail, user says: *"Fix the failing tests"*
|
|
63
|
+
1. Read `.github/agents/playwright-test-healer.agent.md`
|
|
64
|
+
2. Debug with Playwright MCP (`test_debug`, `browser_snapshot`, etc.)
|
|
65
|
+
3. Present diagnosis to the user (root cause + proposed 1-3 line fix)
|
|
66
|
+
4. **STOP** — wait for user to approve the fix
|
|
67
|
+
5. Apply fix and re-run
|
|
68
|
+
6. If still failing, mark `test.fixme()` and classify as defect
|
|
69
|
+
|
|
70
|
+
### Phase 5: Report
|
|
71
|
+
User says: *"Generate the report"*
|
|
72
|
+
1. User runs `npm run qa:report` (or you suggest it)
|
|
73
|
+
2. Present the report summary to the user
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## ⛔ Human Supervision (Mandatory)
|
|
78
|
+
|
|
79
|
+
| Phase | You do this | Then wait for approval |
|
|
80
|
+
|-------|------------|----------------------|
|
|
81
|
+
| Plan | Explore app, write plan | User reviews plan |
|
|
82
|
+
| Generate | Write Playwright tests | User reviews code |
|
|
83
|
+
| Execute | Run tests (or tell user to) | User checks results |
|
|
84
|
+
| Heal | Diagnose failure, propose fix | User approves fix |
|
|
85
|
+
| Report | Generate and present | User reviews |
|
|
86
|
+
|
|
87
|
+
**You never proceed to the next phase without explicit user approval.**
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## 🧭 Routing Quick Reference
|
|
92
|
+
|
|
93
|
+
| User intent | Read this |
|
|
94
|
+
|------------|-----------|
|
|
95
|
+
| "Plan tests / Explore the app / Write a test plan" | `.github/agents/playwright-test-planner.agent.md` |
|
|
96
|
+
| "Generate tests / Write test code / Create spec files" | `.github/agents/playwright-test-generator.agent.md` + `prompts/QAe2eprompt.md` |
|
|
97
|
+
| "Fix failing tests / Debug / Heal" | `.github/agents/playwright-test-healer.agent.md` |
|
|
98
|
+
| General QA requests | `prompts/general_prompt.md` |
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## ⚠️ Technical Rules
|
|
103
|
+
|
|
104
|
+
- Never hallucinate CSS selectors — use Playwright MCP to find real ones
|
|
105
|
+
- Priority: `data-testid` > `aria-label` > `role` > `text` > `xpath` (last resort)
|
|
106
|
+
- 1 fix attempt max per test. If still failing, it's a defect.
|
|
107
|
+
- 1-3 line targeted edits only. No full file rewrites.
|
|
108
|
+
- If the dev server isn't running, warn the user before trying to connect.
|
|
109
|
+
- Token efficiency: keep responses focused and minimal.
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
const path = require('path');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const { DIRS, ensureDir } = require('./utils');
|
|
4
|
+
|
|
5
|
+
const AUTH_CONFIG_PATH = path.join(DIRS.qaContext, 'auth.json');
|
|
6
|
+
const CREDENTIALS_PATH = path.join(DIRS.auth, 'credentials.json');
|
|
7
|
+
const STORAGE_PATH = path.join(DIRS.auth, 'storage-state.json');
|
|
8
|
+
|
|
9
|
+
function readJSON(filePath) {
|
|
10
|
+
if (!fs.existsSync(filePath)) return null;
|
|
11
|
+
try { return JSON.parse(fs.readFileSync(filePath, 'utf-8')); } catch { return null; }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function writeJSON(filePath, data) {
|
|
15
|
+
ensureDir(path.dirname(filePath));
|
|
16
|
+
fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const auth = {
|
|
20
|
+
|
|
21
|
+
// ── Auth Config (.qa-context/auth.json) ──
|
|
22
|
+
|
|
23
|
+
getConfig() {
|
|
24
|
+
return readJSON(AUTH_CONFIG_PATH) || {
|
|
25
|
+
methods: [],
|
|
26
|
+
session: { storage_path: STORAGE_PATH, last_refreshed: null },
|
|
27
|
+
};
|
|
28
|
+
},
|
|
29
|
+
|
|
30
|
+
saveConfig(config) {
|
|
31
|
+
writeJSON(AUTH_CONFIG_PATH, config);
|
|
32
|
+
},
|
|
33
|
+
|
|
34
|
+
updateMethod(method) {
|
|
35
|
+
const config = this.getConfig();
|
|
36
|
+
const idx = config.methods.findIndex(m => m.id === method.id);
|
|
37
|
+
if (idx >= 0) config.methods[idx] = method;
|
|
38
|
+
else config.methods.push(method);
|
|
39
|
+
this.saveConfig(config);
|
|
40
|
+
return config;
|
|
41
|
+
},
|
|
42
|
+
|
|
43
|
+
hasMethod() {
|
|
44
|
+
const config = this.getConfig();
|
|
45
|
+
return config.methods.length > 0;
|
|
46
|
+
},
|
|
47
|
+
|
|
48
|
+
// ── Credentials (.auth/credentials.json) ──
|
|
49
|
+
|
|
50
|
+
saveCredentials(username, password, options = {}) {
|
|
51
|
+
const creds = { username, password, url: options.url || '', updated_at: new Date().toISOString() };
|
|
52
|
+
writeJSON(CREDENTIALS_PATH, creds);
|
|
53
|
+
|
|
54
|
+
const config = this.getConfig();
|
|
55
|
+
if (!config.methods.find(m => m.id === 'main-login')) {
|
|
56
|
+
this.updateMethod({
|
|
57
|
+
id: 'main-login',
|
|
58
|
+
type: 'form',
|
|
59
|
+
detected_at: new Date().toISOString(),
|
|
60
|
+
login_url: options.url || '/login',
|
|
61
|
+
fields: [
|
|
62
|
+
{ name: 'username', selector: '', type: 'text' },
|
|
63
|
+
{ name: 'password', selector: '', type: 'password' },
|
|
64
|
+
],
|
|
65
|
+
submit: '',
|
|
66
|
+
success_indicator: '',
|
|
67
|
+
last_verified: new Date().toISOString(),
|
|
68
|
+
status: 'configured',
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return creds;
|
|
73
|
+
},
|
|
74
|
+
|
|
75
|
+
getCredentials() {
|
|
76
|
+
return readJSON(CREDENTIALS_PATH) || null;
|
|
77
|
+
},
|
|
78
|
+
|
|
79
|
+
hasCredentials() {
|
|
80
|
+
return this.getCredentials() !== null;
|
|
81
|
+
},
|
|
82
|
+
|
|
83
|
+
clearCredentials() {
|
|
84
|
+
if (fs.existsSync(CREDENTIALS_PATH)) fs.unlinkSync(CREDENTIALS_PATH);
|
|
85
|
+
if (fs.existsSync(STORAGE_PATH)) fs.unlinkSync(STORAGE_PATH);
|
|
86
|
+
},
|
|
87
|
+
|
|
88
|
+
// ── Session (.auth/storage-state.json) ──
|
|
89
|
+
|
|
90
|
+
saveSession(page) {
|
|
91
|
+
throw new Error('saveSession requires Playwright page — use from AI agent or Playwright context');
|
|
92
|
+
},
|
|
93
|
+
|
|
94
|
+
getStorageStatePath() {
|
|
95
|
+
return STORAGE_PATH;
|
|
96
|
+
},
|
|
97
|
+
|
|
98
|
+
hasSession() {
|
|
99
|
+
return fs.existsSync(STORAGE_PATH);
|
|
100
|
+
},
|
|
101
|
+
|
|
102
|
+
clearSession() {
|
|
103
|
+
if (fs.existsSync(STORAGE_PATH)) fs.unlinkSync(STORAGE_PATH);
|
|
104
|
+
},
|
|
105
|
+
|
|
106
|
+
// ── Login Steps Generator ──
|
|
107
|
+
|
|
108
|
+
getLoginSteps(methodId) {
|
|
109
|
+
const config = this.getConfig();
|
|
110
|
+
const method = config.methods.find(m => m.id === (methodId || 'main-login'));
|
|
111
|
+
if (!method) return null;
|
|
112
|
+
|
|
113
|
+
const creds = this.getCredentials();
|
|
114
|
+
if (!creds) return null;
|
|
115
|
+
|
|
116
|
+
return {
|
|
117
|
+
url: method.login_url || creds.url || '/login',
|
|
118
|
+
fields: method.fields.map(f => ({
|
|
119
|
+
selector: f.selector,
|
|
120
|
+
value: f.name === 'username' || f.name === 'email' ? creds.username : creds.password,
|
|
121
|
+
})),
|
|
122
|
+
submit: method.submit,
|
|
123
|
+
success: method.success_indicator,
|
|
124
|
+
};
|
|
125
|
+
},
|
|
126
|
+
|
|
127
|
+
generateSetupCode() {
|
|
128
|
+
const config = this.getConfig();
|
|
129
|
+
const method = config.methods.find(m => m.id === 'main-login');
|
|
130
|
+
const creds = this.getCredentials();
|
|
131
|
+
if (!method || !creds) return null;
|
|
132
|
+
|
|
133
|
+
const storagePath = STORAGE_PATH.replace(/\\/g, '/');
|
|
134
|
+
const fields = method.fields.map(f => {
|
|
135
|
+
const val = f.name === 'username' || f.name === 'email' ? 'credentials.username' : 'credentials.password';
|
|
136
|
+
return ` await page.locator('${f.selector || ''}').fill(${val});`;
|
|
137
|
+
}).join('\n');
|
|
138
|
+
|
|
139
|
+
return `import { test as setup, expect } from '@playwright/test';
|
|
140
|
+
import path from 'path';
|
|
141
|
+
|
|
142
|
+
const AUTH_FILE = path.resolve(__dirname, '..', '${storagePath}');
|
|
143
|
+
const CREDENTIALS = require('${CREDENTIALS_PATH.replace(/\\/g, '/')}');
|
|
144
|
+
|
|
145
|
+
setup('authenticate', async ({ page }) => {
|
|
146
|
+
await page.goto('${method.login_url || creds.url || '/'}');
|
|
147
|
+
${fields}
|
|
148
|
+
await page.locator('${method.submit || ''}').click();
|
|
149
|
+
await page.waitForURL('**${method.success_indicator || ''}**');
|
|
150
|
+
await page.context().storageState({ path: AUTH_FILE });
|
|
151
|
+
});`;
|
|
152
|
+
},
|
|
153
|
+
|
|
154
|
+
// ── Auth Failure Detection ──
|
|
155
|
+
|
|
156
|
+
isAuthFailure(error) {
|
|
157
|
+
if (!error) return false;
|
|
158
|
+
const msg = typeof error === 'string' ? error : (error.message || error.error || '');
|
|
159
|
+
const patterns = [
|
|
160
|
+
'login', 'unauthorized', 'unauthenticated', '401', '403',
|
|
161
|
+
'session expired', 'invalid token', 'redirected to login',
|
|
162
|
+
'sign in', 'authentication required', 'credentials',
|
|
163
|
+
];
|
|
164
|
+
return patterns.some(p => msg.toLowerCase().includes(p));
|
|
165
|
+
},
|
|
166
|
+
|
|
167
|
+
// ── Init ──
|
|
168
|
+
|
|
169
|
+
init() {
|
|
170
|
+
ensureDir(DIRS.auth);
|
|
171
|
+
|
|
172
|
+
if (!fs.existsSync(AUTH_CONFIG_PATH)) {
|
|
173
|
+
this.saveConfig({
|
|
174
|
+
methods: [],
|
|
175
|
+
session: { storage_path: STORAGE_PATH, last_refreshed: null },
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const gitignorePath = path.join(DIRS.auth, '.gitignore');
|
|
180
|
+
if (!fs.existsSync(gitignorePath)) {
|
|
181
|
+
fs.writeFileSync(gitignorePath, '# Auth credentials and Playwright storage state\n*\n', 'utf-8');
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
module.exports = auth;
|