@ai-qa/workflow 2.0.11 → 2.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/agents/playwright-test-generator.agent.md +49 -0
- package/.github/agents/playwright-test-healer.agent.md +32 -3
- package/.github/agents/playwright-test-planner.agent.md +26 -0
- package/.github/copilot-instructions.md +44 -2
- package/.opencode/agents/qa-generator.md +16 -0
- package/.opencode/agents/qa-healer.md +18 -0
- package/.opencode/agents/qa-planner.md +17 -0
- package/.opencode/rules.md +66 -2
- package/.qa-context/auth.json +29 -0
- package/.qa-context/heal-history.json +40 -0
- package/.qa-context/pipeline.json +34 -0
- package/.qa-context/selectors.json +64 -0
- package/.qa-context/traceability.json +30 -0
- package/README.md +399 -196
- package/ai-qa-workflow.js +82 -104
- package/install.js +7 -12
- package/package.json +5 -6
- package/prompting_template.md +283 -0
- package/qa-dashboard/app.js +1 -0
- package/qa-dashboard/routes/review.js +114 -0
- package/qa-dashboard/views/layouts/main.ejs +1 -0
- package/qa-dashboard/views/review.ejs +201 -0
- package/router.md +109 -29
- package/scripts/auth-manager.js +186 -0
- package/scripts/context-manager.js +226 -0
- package/scripts/executor.js +18 -7
- package/scripts/generator.js +18 -124
- package/scripts/healer.js +78 -157
- package/scripts/planner.js +18 -136
- package/scripts/reporter.js +21 -1
- package/scripts/utils.js +2 -0
package/ai-qa-workflow.js
CHANGED
|
@@ -1,33 +1,33 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
const { DIRS, ROOT, CONFIG, ensureDir, listUserStories, listTestPlans, listTestSpecs, timestamp, log, reloadConfig, autoDetectConfig } = require('./scripts/utils');
|
|
4
|
-
const
|
|
5
|
-
const { generateTestSpec } = require('./scripts/generator');
|
|
4
|
+
const context = require('./scripts/context-manager');
|
|
6
5
|
const { executeTests } = require('./scripts/executor');
|
|
7
6
|
const { selfHeal } = require('./scripts/healer');
|
|
8
7
|
const { generateReport } = require('./scripts/reporter');
|
|
9
8
|
const path = require('path');
|
|
10
9
|
const fs = require('fs');
|
|
11
10
|
|
|
12
|
-
const PROJECT_NAME = CONFIG.project.name;
|
|
13
|
-
|
|
14
11
|
const COMMANDS = {
|
|
15
12
|
init: { desc: 'Create required directories and config', fn: cmdInit },
|
|
16
|
-
plan: { desc: '
|
|
17
|
-
generate:{ desc: '
|
|
13
|
+
plan: { desc: '(handled by AI agent — see playwright-test-planner.agent.md)', fn: cmdPlan },
|
|
14
|
+
generate:{ desc: '(handled by AI agent — see playwright-test-generator.agent.md)', fn: cmdGenerate },
|
|
18
15
|
execute: { desc: '[test-name] Run tests with Playwright', fn: cmdExecute },
|
|
19
|
-
heal: { desc: '[run-id]
|
|
16
|
+
heal: { desc: '[run-id] Retry failed tests (AI handles deeper diagnosis)', fn: cmdHeal },
|
|
20
17
|
report: { desc: '[run-id] Generate execution report', fn: cmdReport },
|
|
21
18
|
'report:allure': { desc: '[run-id] Generate Allure report from test results', fn: cmdAllure },
|
|
22
|
-
run: { desc: '<story-name.md> Full pipeline: plan → generate → execute → heal → report', fn: cmdRun },
|
|
23
19
|
status: { desc: 'Show workflow state', fn: cmdStatus },
|
|
24
20
|
list: { desc: 'List user stories, test plans, and test specs', fn: cmdList },
|
|
21
|
+
context: { desc: '[phase] [story] Mark a pipeline phase complete for a story', fn: cmdContext },
|
|
25
22
|
};
|
|
26
23
|
|
|
27
24
|
function cmdInit() {
|
|
28
25
|
log('INIT', 'Creating required directories...');
|
|
29
26
|
Object.values(DIRS).forEach(dir => ensureDir(dir));
|
|
30
27
|
|
|
28
|
+
const created = ['user-story', 'specs', 'tests', 'test-results', 'docs', '.qa-context', '.auth'];
|
|
29
|
+
log('INIT', `Directories: ${created.join(', ')}`);
|
|
30
|
+
|
|
31
31
|
const configPath = path.join(ROOT, '.qa-workflow.json');
|
|
32
32
|
if (!fs.existsSync(configPath)) {
|
|
33
33
|
log('INIT', 'Scanning project to auto-detect configuration...');
|
|
@@ -65,36 +65,38 @@ function cmdInit() {
|
|
|
65
65
|
log('INIT', 'docs/application-context.md created — AI agents will use this for context');
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
|
|
69
|
-
log('INIT', '
|
|
68
|
+
context.init();
|
|
69
|
+
log('INIT', '.qa-context/ initialized (pipeline.json, selectors.json, heal-history.json, traceability.json)');
|
|
70
|
+
|
|
71
|
+
const authGitignore = path.join(DIRS.auth, '.gitignore');
|
|
72
|
+
if (!fs.existsSync(authGitignore)) {
|
|
73
|
+
fs.writeFileSync(authGitignore, '# Auth credentials and Playwright storage state — never commit\n*\n', 'utf-8');
|
|
74
|
+
log('INIT', '.auth/ created (credentials.json, storage-state.json) — gitignored');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
log('INIT', 'Ready. Edit .qa-workflow.json to override auto-detected values');
|
|
70
78
|
}
|
|
71
79
|
|
|
72
80
|
function cmdPlan() {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
console.log(`\n ✓ Test plan generated: specs/${path.basename(result.planPath)}`);
|
|
82
|
-
console.log(`\n Next step: Review and edit the test plan, then run:`);
|
|
83
|
-
console.log(` node ai-qa-workflow.js generate ${path.basename(result.planPath)}`);
|
|
81
|
+
console.log('');
|
|
82
|
+
console.log('╔══════════════════════════════════════════════════════════╗');
|
|
83
|
+
console.log('║ PLANNING is handled by the AI agent ║');
|
|
84
|
+
console.log('║ ║');
|
|
85
|
+
console.log('║ Tell your AI agent: ║');
|
|
86
|
+
console.log('║ "Read router.md and plan tests for my-story.md" ║');
|
|
87
|
+
console.log('╚══════════════════════════════════════════════════════════╝');
|
|
88
|
+
console.log('');
|
|
84
89
|
}
|
|
85
90
|
|
|
86
91
|
function cmdGenerate() {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
console.log(`\n ✓ Test spec generated: tests/${result.specName}.spec.ts`);
|
|
96
|
-
console.log(`\n Next step: Implement test logic in the spec file, then run:`);
|
|
97
|
-
console.log(` node ai-qa-workflow.js execute ${result.specName}`);
|
|
92
|
+
console.log('');
|
|
93
|
+
console.log('╔══════════════════════════════════════════════════════════╗');
|
|
94
|
+
console.log('║ TEST GENERATION is handled by the AI agent ║');
|
|
95
|
+
console.log('║ ║');
|
|
96
|
+
console.log('║ Tell your AI agent: ║');
|
|
97
|
+
console.log('║ "Generate tests for the plan in specs/" ║');
|
|
98
|
+
console.log('╚══════════════════════════════════════════════════════════╝');
|
|
99
|
+
console.log('');
|
|
98
100
|
}
|
|
99
101
|
|
|
100
102
|
function cmdExecute() {
|
|
@@ -123,22 +125,18 @@ function cmdExecute() {
|
|
|
123
125
|
|
|
124
126
|
function cmdHeal() {
|
|
125
127
|
const runId = process.argv[3];
|
|
126
|
-
const result = selfHeal(runId
|
|
128
|
+
const result = selfHeal(runId);
|
|
127
129
|
|
|
128
|
-
if (result.
|
|
129
|
-
console.log(`\n
|
|
130
|
-
result.healed.forEach(h => console.log(` - ${h.test || h.file} (${h.strategy})`));
|
|
130
|
+
if (result.rerun.length > 0) {
|
|
131
|
+
console.log(`\n Re-ran ${result.rerun.length} test(s) with longer timeout`);
|
|
131
132
|
}
|
|
132
|
-
if (result.
|
|
133
|
-
console.log(`\n
|
|
134
|
-
|
|
133
|
+
if (result.stillFailing.length > 0) {
|
|
134
|
+
console.log(`\n ${result.stillFailing.length} test(s) still failing — AI agent should diagnose`);
|
|
135
|
+
console.log(` Tell your AI agent: "Debug the failing tests in test-results/"`);
|
|
135
136
|
}
|
|
136
|
-
if (result.
|
|
137
|
-
console.log('\n No failures to
|
|
137
|
+
if (result.rerun.length === 0 && result.stillFailing.length === 0) {
|
|
138
|
+
console.log('\n No failures to retry.');
|
|
138
139
|
}
|
|
139
|
-
|
|
140
|
-
console.log(`\n Next step: Generate report:`);
|
|
141
|
-
console.log(` node ai-qa-workflow.js report ${result.reportPath ? path.basename(path.dirname(result.reportPath)) : runId || '(latest)'}`);
|
|
142
140
|
}
|
|
143
141
|
|
|
144
142
|
function cmdReport() {
|
|
@@ -187,60 +185,6 @@ function cmdAllure() {
|
|
|
187
185
|
}
|
|
188
186
|
}
|
|
189
187
|
|
|
190
|
-
function cmdRun() {
|
|
191
|
-
const storyName = process.argv[3];
|
|
192
|
-
if (!storyName) {
|
|
193
|
-
console.log(' Usage: node ai-qa-workflow.js run <user-story-file.md>');
|
|
194
|
-
console.log(` Available: ${listUserStories().join(', ')}`);
|
|
195
|
-
process.exit(1);
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
console.log('\n╔════════════════════════════════════════╗');
|
|
199
|
-
console.log('║ AI QA WORKFLOW - FULL PIPELINE ║');
|
|
200
|
-
console.log('╚════════════════════════════════════════╝\n');
|
|
201
|
-
|
|
202
|
-
// Step 1: Plan
|
|
203
|
-
log('RUN', 'Step 1/5: Generating test plan...');
|
|
204
|
-
ensureDir(DIRS.specs);
|
|
205
|
-
const plan = generateTestPlan(storyName);
|
|
206
|
-
console.log(` → specs/${path.basename(plan.planPath)}\n`);
|
|
207
|
-
|
|
208
|
-
// Step 2: Generate
|
|
209
|
-
log('RUN', 'Step 2/5: Generating test spec...');
|
|
210
|
-
ensureDir(DIRS.tests);
|
|
211
|
-
const spec = generateTestSpec(path.basename(plan.planPath));
|
|
212
|
-
console.log(` → tests/${spec.specName}.spec.ts\n`);
|
|
213
|
-
|
|
214
|
-
// Step 3: Execute
|
|
215
|
-
log('RUN', 'Step 3/5: Executing tests...');
|
|
216
|
-
ensureDir(DIRS.testResults);
|
|
217
|
-
const execResult = executeTests(spec.specName);
|
|
218
|
-
console.log(` → ${execResult.success ? '✓ PASSED' : '✗ FAILED'} (${(execResult.duration / 1000).toFixed(1)}s)\n`);
|
|
219
|
-
|
|
220
|
-
// Step 4: Heal (if needed)
|
|
221
|
-
if (!execResult.success) {
|
|
222
|
-
log('RUN', 'Step 4/5: Self-healing...');
|
|
223
|
-
const healResult = selfHeal(execResult.runId, PROJECT_NAME);
|
|
224
|
-
console.log(` → Healed: ${healResult.healed.length} | Remaining: ${healResult.remaining.length}\n`);
|
|
225
|
-
} else {
|
|
226
|
-
log('RUN', 'Step 4/5: Skipped (all tests passed)');
|
|
227
|
-
console.log('');
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// Step 5: Report
|
|
231
|
-
log('RUN', 'Step 5/5: Generating report...');
|
|
232
|
-
const report = generateReport(execResult.runId);
|
|
233
|
-
console.log(` → ${report ? report.reportPath : 'N/A'}\n`);
|
|
234
|
-
|
|
235
|
-
console.log('╔════════════════════════════════════════╗');
|
|
236
|
-
console.log('║ WORKFLOW COMPLETE ║');
|
|
237
|
-
console.log('╚════════════════════════════════════════╝');
|
|
238
|
-
if (report) {
|
|
239
|
-
console.log(`\n Report: test-results/${execResult.runId}/final-test-report.md`);
|
|
240
|
-
console.log(` Passed: ${report.stats.passed} | Failed: ${report.stats.failed} | Healed: ${report.stats.healed}`);
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
|
|
244
188
|
function cmdStatus() {
|
|
245
189
|
console.log('\n📊 AI QA Workflow Status\n');
|
|
246
190
|
|
|
@@ -298,6 +242,35 @@ function cmdList() {
|
|
|
298
242
|
console.log('');
|
|
299
243
|
}
|
|
300
244
|
|
|
245
|
+
function cmdContext() {
|
|
246
|
+
const phase = process.argv[3];
|
|
247
|
+
const story = process.argv[4];
|
|
248
|
+
|
|
249
|
+
if (!phase || !story) {
|
|
250
|
+
console.log('\nUsage: node ai-qa-workflow.js context <phase> <story>');
|
|
251
|
+
console.log('Phases: plan, generate, execute, heal, report');
|
|
252
|
+
console.log('Example: node ai-qa-workflow.js context plan login\n');
|
|
253
|
+
process.exit(1);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const validPhases = ['plan', 'generate', 'execute', 'heal', 'report'];
|
|
257
|
+
if (!validPhases.includes(phase)) {
|
|
258
|
+
console.log(`\nInvalid phase: ${phase}`);
|
|
259
|
+
console.log(`Valid: ${validPhases.join(', ')}\n`);
|
|
260
|
+
process.exit(1);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
context.phaseComplete(phase, story);
|
|
264
|
+
context.setCurrentStory(story);
|
|
265
|
+
|
|
266
|
+
const pipeline = context.getPipeline();
|
|
267
|
+
const storyData = pipeline.phases[phase];
|
|
268
|
+
const completed = storyData ? storyData.completed.join(', ') : '—';
|
|
269
|
+
|
|
270
|
+
console.log(`\n ✓ Phase "${phase}" marked complete for story "${story}"`);
|
|
271
|
+
console.log(` Completed for this phase: ${completed}\n`);
|
|
272
|
+
}
|
|
273
|
+
|
|
301
274
|
const cmd = process.argv[2];
|
|
302
275
|
|
|
303
276
|
if (!cmd || cmd === '--help' || cmd === '-h') {
|
|
@@ -307,14 +280,19 @@ if (!cmd || cmd === '--help' || cmd === '-h') {
|
|
|
307
280
|
for (const [name, { desc }] of Object.entries(COMMANDS)) {
|
|
308
281
|
console.log(` ${name.padEnd(12)} ${desc}`);
|
|
309
282
|
}
|
|
310
|
-
console.log('\
|
|
283
|
+
console.log('\nNote: Planning and test generation are done by the AI agent, not CLI commands.');
|
|
284
|
+
console.log('See: .github/agents/ and prompts/ for AI instructions.\n');
|
|
285
|
+
console.log('Examples:');
|
|
311
286
|
console.log(' node ai-qa-workflow.js init');
|
|
312
|
-
console.log(' node ai-qa-workflow.js
|
|
313
|
-
console.log(' node ai-qa-workflow.js
|
|
314
|
-
console.log(' node ai-qa-workflow.js
|
|
315
|
-
console.log(' node ai-qa-workflow.js run
|
|
287
|
+
console.log(' node ai-qa-workflow.js execute');
|
|
288
|
+
console.log(' node ai-qa-workflow.js execute my-feature');
|
|
289
|
+
console.log(' node ai-qa-workflow.js heal');
|
|
290
|
+
console.log(' node ai-qa-workflow.js heal run-2026-05-24');
|
|
291
|
+
console.log(' node ai-qa-workflow.js report');
|
|
316
292
|
console.log(' node ai-qa-workflow.js report:allure');
|
|
317
|
-
console.log(' node ai-qa-workflow.js status
|
|
293
|
+
console.log(' node ai-qa-workflow.js status');
|
|
294
|
+
console.log(' node ai-qa-workflow.js list');
|
|
295
|
+
console.log(' node ai-qa-workflow.js context plan login\n');
|
|
318
296
|
process.exit(0);
|
|
319
297
|
}
|
|
320
298
|
|
package/install.js
CHANGED
|
@@ -22,11 +22,8 @@ const USER_DIRS = new Set([
|
|
|
22
22
|
'specs',
|
|
23
23
|
'tests',
|
|
24
24
|
'test-results',
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
'.qa-context',
|
|
26
|
+
'.auth',
|
|
30
27
|
]);
|
|
31
28
|
|
|
32
29
|
// Template files to copy/update
|
|
@@ -39,13 +36,11 @@ const QA_ITEMS = [
|
|
|
39
36
|
{ src: '.opencode', dest: '.opencode', dir: true },
|
|
40
37
|
{ src: 'README.md', dest: 'README.md' },
|
|
41
38
|
{ src: 'PROJECT_GUIDE.md', dest: 'PROJECT_GUIDE.md' },
|
|
39
|
+
{ src: 'prompting_template.md', dest: 'prompting_template.md' },
|
|
42
40
|
{ src: '.cursorrules', dest: '.cursorrules' },
|
|
43
41
|
{ src: '.geminirules', dest: '.geminirules' },
|
|
44
42
|
{ src: '.github/copilot-instructions.md', dest: '.github/copilot-instructions.md' },
|
|
45
43
|
{ src: 'router.md', dest: 'router.md' },
|
|
46
|
-
{ src: 'user-story', dest: 'user-story', dir: true },
|
|
47
|
-
{ src: 'specs', dest: 'specs', dir: true },
|
|
48
|
-
{ src: 'tests', dest: 'tests', dir: true },
|
|
49
44
|
{ src: '.opencode.json', dest: '.opencode.json' },
|
|
50
45
|
|
|
51
46
|
];
|
|
@@ -56,7 +51,7 @@ const UPDATE_ITEMS = QA_ITEMS.filter(item => {
|
|
|
56
51
|
return !USER_FILES.has(item.dest);
|
|
57
52
|
});
|
|
58
53
|
|
|
59
|
-
const DIRS_TO_CREATE = ['user-story', 'specs', 'tests', 'test-results'];
|
|
54
|
+
const DIRS_TO_CREATE = ['user-story', 'specs', 'tests', 'test-results', '.qa-context', '.auth', 'templates'];
|
|
60
55
|
|
|
61
56
|
const NPM_SCRIPTS = {
|
|
62
57
|
'qa': 'node ai-qa-workflow.js',
|
|
@@ -65,9 +60,9 @@ const NPM_SCRIPTS = {
|
|
|
65
60
|
'qa:generate': 'node ai-qa-workflow.js generate',
|
|
66
61
|
'qa:execute': 'node ai-qa-workflow.js execute',
|
|
67
62
|
'qa:heal': 'node ai-qa-workflow.js heal',
|
|
63
|
+
'qa:retry': 'node ai-qa-workflow.js heal',
|
|
68
64
|
'qa:report': 'node ai-qa-workflow.js report',
|
|
69
65
|
'qa:report:allure': 'node ai-qa-workflow.js report:allure',
|
|
70
|
-
'qa:run': 'node ai-qa-workflow.js run',
|
|
71
66
|
'qa:status': 'node ai-qa-workflow.js status',
|
|
72
67
|
'qa:list': 'node ai-qa-workflow.js list',
|
|
73
68
|
'dashboard': 'cd qa-dashboard && npm start',
|
|
@@ -239,8 +234,8 @@ async function install(targetPath, mode) {
|
|
|
239
234
|
} else {
|
|
240
235
|
console.log(` Files: ~${totalFiles} scripts + ${dashboardCount} dashboard files`);
|
|
241
236
|
console.log(` Next:\n`);
|
|
242
|
-
console.log(` npm run qa:init Initialize pipeline (
|
|
243
|
-
console.log(` npm run qa:
|
|
237
|
+
console.log(` npm run qa:init Initialize pipeline (config + dirs + auth)`);
|
|
238
|
+
console.log(` npm run qa:execute Run Playwright tests`);
|
|
244
239
|
console.log(` npm run qa:status Check pipeline state`);
|
|
245
240
|
console.log(` npm run dashboard Start dashboard (port 4000)\n`);
|
|
246
241
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ai-qa/workflow",
|
|
3
|
-
"version": "2.0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "2.0.13",
|
|
4
|
+
"description": "AI QA Workflow Template — transforms any AI agent into an autonomous QA engineer. AI explores, plans, generates tests, and heals. Scripts execute and report.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"qa",
|
|
7
7
|
"testing",
|
|
@@ -16,17 +16,16 @@
|
|
|
16
16
|
"prompts/",
|
|
17
17
|
"scripts/",
|
|
18
18
|
"qa-dashboard/",
|
|
19
|
-
"user-story/",
|
|
20
|
-
"specs/",
|
|
21
|
-
"tests/",
|
|
22
19
|
"test-results/",
|
|
23
20
|
"docs/",
|
|
21
|
+
".qa-context/",
|
|
24
22
|
"templates/",
|
|
25
23
|
"ai-qa-workflow.js",
|
|
26
24
|
"cli.js",
|
|
27
25
|
"install.js",
|
|
28
26
|
"README.md",
|
|
29
27
|
"PROJECT_GUIDE.md",
|
|
28
|
+
"prompting_template.md",
|
|
30
29
|
".cursorrules",
|
|
31
30
|
".geminirules",
|
|
32
31
|
".opencode/",
|
|
@@ -50,9 +49,9 @@
|
|
|
50
49
|
"qa:generate": "node ai-qa-workflow.js generate",
|
|
51
50
|
"qa:execute": "node ai-qa-workflow.js execute",
|
|
52
51
|
"qa:heal": "node ai-qa-workflow.js heal",
|
|
52
|
+
"qa:retry": "node ai-qa-workflow.js heal",
|
|
53
53
|
"qa:report": "node ai-qa-workflow.js report",
|
|
54
54
|
"qa:report:allure": "node ai-qa-workflow.js report:allure",
|
|
55
|
-
"qa:run": "node ai-qa-workflow.js run",
|
|
56
55
|
"qa:status": "node ai-qa-workflow.js status",
|
|
57
56
|
"qa:list": "node ai-qa-workflow.js list",
|
|
58
57
|
"dashboard": "cd qa-dashboard && npm start",
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
# Prompting Template — How to Talk to the AI QA Agent
|
|
2
|
+
|
|
3
|
+
This guide covers every prompt you need — from installation to final report. Use it as your conversation script.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Quick Start (Copy-Paste Ready)
|
|
8
|
+
|
|
9
|
+
After installing the template and writing a user story, open your AI editor and send:
|
|
10
|
+
|
|
11
|
+
> **"Read router.md and follow the QA workflow for my-story.md"**
|
|
12
|
+
|
|
13
|
+
That's it. The AI handles everything from there, stopping at each phase for your approval.
|
|
14
|
+
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
## Phase-by-Phase Prompts
|
|
18
|
+
|
|
19
|
+
### Phase 0: First Contact (Environment Check)
|
|
20
|
+
|
|
21
|
+
When you open the project for the **first time**, the AI automatically runs an environment check. If it doesn't, or if you want to re-check:
|
|
22
|
+
|
|
23
|
+
> **"Run the environment check and tell me what's ready and what's missing"**
|
|
24
|
+
|
|
25
|
+
The AI will report:
|
|
26
|
+
- ✅ What's installed and configured
|
|
27
|
+
- ✅ Pipeline state from `.qa-context/pipeline.json` (phases completed, last run)
|
|
28
|
+
- ✅ Auth status from `.auth/credentials.json` (credentials found or missing)
|
|
29
|
+
- ❌ What's missing (with commands to fix)
|
|
30
|
+
- 📋 What it needs from you (user story, credentials, etc.)
|
|
31
|
+
|
|
32
|
+
The AI **automatically reads `.qa-context/pipeline.json`, `.qa-context/selectors.json`, and `.qa-context/heal-history.json`** to understand the current state before starting any phase.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
### Phase 1: Test Planning
|
|
37
|
+
|
|
38
|
+
You have a user story. You want the AI to explore the app and write a test plan.
|
|
39
|
+
|
|
40
|
+
> **"Read router.md and plan tests for my-story.md"**
|
|
41
|
+
|
|
42
|
+
The AI will:
|
|
43
|
+
1. Read `router.md` → routes to `playwright-test-planner.agent.md`
|
|
44
|
+
2. Open your app with Playwright MCP
|
|
45
|
+
3. Explore navigation, flows, and UI components
|
|
46
|
+
4. Map critical paths and edge cases
|
|
47
|
+
5. Write a test plan to `specs/my-story-test-plan.md`
|
|
48
|
+
6. **STOP and present the plan to you**
|
|
49
|
+
|
|
50
|
+
#### What you should review in the plan:
|
|
51
|
+
- Are all acceptance criteria covered?
|
|
52
|
+
- Are edge cases and negative scenarios included?
|
|
53
|
+
- Are the preconditions accurate?
|
|
54
|
+
- Does the environment URL match your running app?
|
|
55
|
+
|
|
56
|
+
#### Approval responses:
|
|
57
|
+
| You say | Result |
|
|
58
|
+
|---------|--------|
|
|
59
|
+
| "Looks good, continue" | AI proceeds to test generation |
|
|
60
|
+
| "Approved" | Same |
|
|
61
|
+
| "Add a scenario for X" | AI updates the plan, then waits again |
|
|
62
|
+
| "I'll review later, proceed" | AI proceeds (use with caution) |
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
### Phase 2: Test Generation
|
|
67
|
+
|
|
68
|
+
The plan is approved. Now you want the AI to write real Playwright tests.
|
|
69
|
+
|
|
70
|
+
> **"Generate tests from the plan in specs/my-story-test-plan.md"**
|
|
71
|
+
|
|
72
|
+
Or simply (if already in context):
|
|
73
|
+
|
|
74
|
+
> **"Generate the tests"**
|
|
75
|
+
|
|
76
|
+
The AI will:
|
|
77
|
+
1. Read `playwright-test-generator.agent.md`
|
|
78
|
+
2. Read `prompts/QAe2eprompt.md` for conventions
|
|
79
|
+
3. Use Playwright MCP to manually execute each scenario
|
|
80
|
+
4. Capture real selectors from the actual page
|
|
81
|
+
5. Add visual checkpoints via Applitools on critical pages (if `APPLITOOLS_API_KEY` is set)
|
|
82
|
+
6. Write complete Playwright `.spec.ts` files to `tests/`
|
|
83
|
+
7. **STOP and present the code to you**
|
|
84
|
+
|
|
85
|
+
#### What you should review in the test code:
|
|
86
|
+
- Are selectors using real elements? (no hallucinated CSS classes)
|
|
87
|
+
- Are assertions meaningful?
|
|
88
|
+
- Are visual checkpoints added on critical pages? (login, dashboard, checkout)
|
|
89
|
+
- Does the test handle auth/state correctly?
|
|
90
|
+
- Are there timeouts or waits that might flake?
|
|
91
|
+
|
|
92
|
+
#### Approval responses:
|
|
93
|
+
| You say | Result |
|
|
94
|
+
|---------|--------|
|
|
95
|
+
| "Code looks good, execute" | User runs tests, or AI suggests running |
|
|
96
|
+
| "Approved" | Same |
|
|
97
|
+
| "Fix the selector on line 23" | AI corrects it, presents again |
|
|
98
|
+
| "Add a test for X edge case" | AI adds the scenario, presents again |
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
### Phase 3: Execution
|
|
103
|
+
|
|
104
|
+
Tests are written and approved. Time to run them.
|
|
105
|
+
|
|
106
|
+
> **User runs:** `npm run qa:execute [test-name]`
|
|
107
|
+
|
|
108
|
+
Or, if you want the AI to trigger execution (if it has shell access):
|
|
109
|
+
|
|
110
|
+
> **"Execute the tests and report back"**
|
|
111
|
+
|
|
112
|
+
The AI runs Playwright and reports:
|
|
113
|
+
- ✅ Tests passed → ready for report
|
|
114
|
+
- ❌ Tests failed → moves to healing phase
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
### Phase 4: Healing (Debugging Failures)
|
|
119
|
+
|
|
120
|
+
Tests failed. You want the AI to fix them.
|
|
121
|
+
|
|
122
|
+
> **"Debug and fix the failing tests"**
|
|
123
|
+
|
|
124
|
+
Or specifically:
|
|
125
|
+
|
|
126
|
+
> **"Read playwright-test-healer.agent.md and debug the failures in test-results/latest-run"**
|
|
127
|
+
|
|
128
|
+
The AI will:
|
|
129
|
+
1. Read `playwright-test-healer.agent.md`
|
|
130
|
+
2. Run the failing tests with `test_run`
|
|
131
|
+
3. Debug with `test_debug` — examine the actual UI state
|
|
132
|
+
4. Classify the failure:
|
|
133
|
+
- **Selector broken** → proposes 1-3 line fix
|
|
134
|
+
- **Timing issue** → proposes adding a wait
|
|
135
|
+
- **App bug** → marks `test.fixme()`, logs defect
|
|
136
|
+
5. **STOP and present diagnosis + proposed fix to you**
|
|
137
|
+
|
|
138
|
+
#### What you should review in the diagnosis:
|
|
139
|
+
- What exactly failed and why
|
|
140
|
+
- What the AI wants to change (which file, which lines)
|
|
141
|
+
- Is it a test issue or a real bug in your app?
|
|
142
|
+
|
|
143
|
+
#### Approval responses:
|
|
144
|
+
| You say | Result |
|
|
145
|
+
|---------|--------|
|
|
146
|
+
| "Fix it" | AI applies the 1-3 line fix and re-runs |
|
|
147
|
+
| "Approved, apply the fix" | Same |
|
|
148
|
+
| "That's a real bug, file it" | AI marks as defect, moves on |
|
|
149
|
+
| "Try a different approach" | AI suggests alternative fix |
|
|
150
|
+
| "Skip it, I'll investigate later" | AI marks as `test.fixme()` |
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
### Phase 5: Report
|
|
155
|
+
|
|
156
|
+
Tests are done. You want a summary.
|
|
157
|
+
|
|
158
|
+
> **"Generate the execution report"**
|
|
159
|
+
|
|
160
|
+
Or manually:
|
|
161
|
+
|
|
162
|
+
> **User runs:** `npm run qa:report [run-id]`
|
|
163
|
+
|
|
164
|
+
The AI will:
|
|
165
|
+
1. Run `npm run qa:report` (or read the results directly)
|
|
166
|
+
2. Present a summary: passed, failed, healed, defects found
|
|
167
|
+
3. Update `docs/application-context.md` with learned selectors and flaky areas
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## Quick Reference — All Prompts in One Table
|
|
172
|
+
|
|
173
|
+
| Phase | Your prompt | AI does | You do |
|
|
174
|
+
|-------|------------|---------|--------|
|
|
175
|
+
| **0** | *(automatic on first open)* | Environment check | Read status report |
|
|
176
|
+
| **0** | "Run the environment check" | Re-check setup | Fix any missing items |
|
|
177
|
+
| **1** | "Read router.md and plan tests for my-story.md" | Explore app, write plan | **Review + approve** |
|
|
178
|
+
| **2** | "Generate tests from the plan" | Write Playwright code | **Review + approve** |
|
|
179
|
+
| **3** | `npm run qa:execute` | Run tests | Check results |
|
|
180
|
+
| **4** | "Debug and fix the failing tests" | Diagnose, propose fix | **Review + approve** |
|
|
181
|
+
| **5** | "Generate the report" | Summarize results | Review final report |
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## Best Practices
|
|
186
|
+
|
|
187
|
+
### For Best Results
|
|
188
|
+
|
|
189
|
+
1. **Write structured user stories** — Use the format in the README. Clear acceptance criteria produce better test plans.
|
|
190
|
+
|
|
191
|
+
2. **Review the plan first** — The AI's understanding of your app is only as good as its exploration. Catch incorrect assumptions early.
|
|
192
|
+
|
|
193
|
+
3. **Don't skip the approval gates** — Each gate is a chance to redirect the AI before it wastes time on the wrong approach.
|
|
194
|
+
|
|
195
|
+
4. **Be specific in your feedback** — Instead of "this looks wrong", say "the selector on line 23 should use `data-testid` instead of class name".
|
|
196
|
+
|
|
197
|
+
5. **Provide context upfront** — Populate `docs/application-context.md` with stable selectors, auth details, and tech stack notes before starting. The AI also reads `.qa-context/` automatically — you don't need to tell it.
|
|
198
|
+
|
|
199
|
+
6. **Memory is persistent** — The AI stores its learning in `.qa-context/` (selectors, healing history, pipeline state). This means each session starts smarter than the last. Don't delete this folder unless you want to reset the AI's memory.
|
|
200
|
+
|
|
201
|
+
7. **Credentials are stored once** — The AI saves your login to `.auth/credentials.json` after first use. If your password changes, just update that file or delete it and the AI will ask again.
|
|
202
|
+
|
|
203
|
+
6. **Use `npm run qa:retry` for simple flakes** — If a test fails intermittently, try the mechanical retry first before asking the AI to debug.
|
|
204
|
+
|
|
205
|
+
7. **Token efficiency works both ways** — Keep your feedback concise. The AI is designed for 1-3 line fixes and targeted edits.
|
|
206
|
+
|
|
207
|
+
### Common Mistakes to Avoid
|
|
208
|
+
|
|
209
|
+
| Mistake | Better approach |
|
|
210
|
+
|---------|----------------|
|
|
211
|
+
| Sending a vague "test my app" prompt | "Read router.md and plan tests for user-story/login.md" |
|
|
212
|
+
| Approving without reviewing the plan | Always check: does it cover the acceptance criteria? |
|
|
213
|
+
| Letting the AI generate tests without a plan | Always do Phase 1 before Phase 2 |
|
|
214
|
+
| Running tests before reviewing the code | AI can hallucinate selectors — verify them first |
|
|
215
|
+
| Asking the AI to fix everything at once | One test at a time: "Debug the login test failure" |
|
|
216
|
+
| Skipping the approval gate | You're the supervisor for a reason |
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## Example Full Session
|
|
221
|
+
|
|
222
|
+
```
|
|
223
|
+
You: "Read router.md and plan tests for login-story.md"
|
|
224
|
+
|
|
225
|
+
AI: [Explores app, writes plan to specs/login-story-test-plan.md]
|
|
226
|
+
"Plan is ready. Here's what I found:
|
|
227
|
+
- 3 scenarios: login success, wrong password, locked account
|
|
228
|
+
- 1 concern: no visible error toast for locked accounts
|
|
229
|
+
Do you approve?"
|
|
230
|
+
|
|
231
|
+
You: "Looks good, but add a scenario for empty email field."
|
|
232
|
+
|
|
233
|
+
AI: [Updates plan]
|
|
234
|
+
"Added. 4 scenarios now. Approve?"
|
|
235
|
+
|
|
236
|
+
You: "Approved, generate the tests."
|
|
237
|
+
|
|
238
|
+
AI: [Writes tests/login-story.spec.ts]
|
|
239
|
+
"Tests generated. I used getByRole for buttons and getByLabel for inputs.
|
|
240
|
+
Do you approve the code?"
|
|
241
|
+
|
|
242
|
+
You: "Looks good, execute."
|
|
243
|
+
|
|
244
|
+
AI: "Run: npm run qa:execute login-story"
|
|
245
|
+
|
|
246
|
+
You: [Runs it] "Two tests failed — wrong password and locked account."
|
|
247
|
+
|
|
248
|
+
AI: [Debugs]
|
|
249
|
+
"Diagnosis:
|
|
250
|
+
- Wrong password: error message class changed, needs new selector
|
|
251
|
+
- Locked account: the lock icon isn't rendering (app bug)
|
|
252
|
+
Fix the selector on line 42? Approve?"
|
|
253
|
+
|
|
254
|
+
You: "Fix the selector, mark the lock icon as a bug."
|
|
255
|
+
|
|
256
|
+
AI: [Applies fix, re-runs, marks test.fixme()]
|
|
257
|
+
"Fixed: wrong password now passes. Locked account marked as defect.
|
|
258
|
+
Generate the report?"
|
|
259
|
+
|
|
260
|
+
You: "Yes, generate it."
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
---
|
|
264
|
+
|
|
265
|
+
## Quick Copy-Paste
|
|
266
|
+
|
|
267
|
+
### Minimal workflow (5 prompts total)
|
|
268
|
+
|
|
269
|
+
```
|
|
270
|
+
1. "Read router.md and plan tests for my-story.md"
|
|
271
|
+
2. "Approved" (after reviewing plan)
|
|
272
|
+
3. "Approved" (after reviewing tests)
|
|
273
|
+
(run npm run qa:execute)
|
|
274
|
+
4. "Debug and fix the failing tests"
|
|
275
|
+
5. "Approved, apply the fix" (after reviewing diagnosis)
|
|
276
|
+
(run npm run qa:report)
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
### Single command to start everything
|
|
280
|
+
|
|
281
|
+
```
|
|
282
|
+
"Read router.md and follow the QA workflow for my-story.md"
|
|
283
|
+
```
|
package/qa-dashboard/app.js
CHANGED
|
@@ -86,6 +86,7 @@ app.use('/', require('./routes/index'));
|
|
|
86
86
|
app.use('/projects', require('./routes/projects'));
|
|
87
87
|
app.use('/stories', require('./routes/stories'));
|
|
88
88
|
app.use('/runs', require('./routes/runs'));
|
|
89
|
+
app.use('/review', require('./routes/review'));
|
|
89
90
|
app.use('/analytics', require('./routes/analytics'));
|
|
90
91
|
app.use('/export', require('./routes/export'));
|
|
91
92
|
app.use('/data', require('./routes/test-data'));
|