@keber/qa-framework 1.9.1 → 1.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +103 -16
- package/README.md +1 -1
- package/docs/folder-structure-guide.md +6 -0
- package/integrations/ado-powershell/pipelines/azure-pipeline-qa.yml +277 -65
- package/integrations/playwright/README.md +124 -0
- package/integrations/playwright-azure-reporter/README.md +75 -24
- package/package.json +3 -4
- package/qa-framework.config.json +1 -1
- package/scripts/init.js +53 -38
- package/scripts/upgrade.js +105 -10
- package/templates/automation-scaffold/.env.example +20 -1
- package/templates/automation-scaffold/fixtures/base.ts +108 -0
- package/templates/automation-scaffold/global-setup.ts +119 -29
- package/templates/automation-scaffold/playwright.config.ts +24 -0
- package/templates/defect-report.md +1 -1
- package/templates/integration-scaffold/.env.example +40 -0
- package/templates/integration-scaffold/README.md +77 -0
- package/templates/integration-scaffold/global-setup.ts +99 -0
- package/templates/integration-scaffold/package.json +15 -0
- package/templates/integration-scaffold/playwright.config.ts +78 -0
- package/templates/integration-scaffold/tests/example.spec.ts +43 -0
- package/templates/memory/ci-pipeline-findings.md +36 -0
- package/templates/memory/data-volatility-strategies.md +70 -0
- package/templates/memory/e2e-stabilization-patterns.md +54 -0
- package/templates/qa-framework.instructions.md +9 -1
- package/templates/qa-readme.md +21 -0
|
@@ -66,3 +66,127 @@ test.describe('[TC-M-S-NNN] Module > Submodule @P0', () => {
|
|
|
66
66
|
});
|
|
67
67
|
});
|
|
68
68
|
```
|
|
69
|
+
|
|
70
|
+
## Visual AI helper (optional advanced pattern)
|
|
71
|
+
|
|
72
|
+
> **Advanced / optional.** This pattern is not part of the default scaffold and requires
|
|
73
|
+
> an AI vision API key. Use it when a selector-based assertion is too brittle or the
|
|
74
|
+
> element structure changes frequently (e.g. PDF previews, dashboard charts, export outputs).
|
|
75
|
+
|
|
76
|
+
Instead of asserting specific selectors, take a screenshot and ask an AI vision model to
|
|
77
|
+
verify that the expected content is visually present. This decouples the assertion from DOM
|
|
78
|
+
structure and is useful for generated documents or complex layouts.
|
|
79
|
+
|
|
80
|
+
```typescript
|
|
81
|
+
// fixtures/vision-helper.ts
|
|
82
|
+
import * as https from 'https';
|
|
83
|
+
|
|
84
|
+
export interface VisionResult {
|
|
85
|
+
passes: boolean;
|
|
86
|
+
reason: string;
|
|
87
|
+
raw?: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Sends a screenshot to an AI vision model and asks whether the described
|
|
92
|
+
* content is visually present. Degrades gracefully when the API key is absent.
|
|
93
|
+
*
|
|
94
|
+
* Requires an API key in the environment (e.g. OPENAI_API_KEY for GPT-4o,
|
|
95
|
+
* or adapt the request body for any other vision-capable model).
|
|
96
|
+
*/
|
|
97
|
+
export async function assertScreenshotShowsContent(
|
|
98
|
+
screenshotBuffer: Buffer,
|
|
99
|
+
contentDescription: string,
|
|
100
|
+
options: { model?: string; degradeOnMissingKey?: boolean } = {}
|
|
101
|
+
): Promise<VisionResult> {
|
|
102
|
+
const apiKey = process.env.AI_VISION_API_KEY;
|
|
103
|
+
const degrade = options.degradeOnMissingKey ?? true;
|
|
104
|
+
|
|
105
|
+
if (!apiKey) {
|
|
106
|
+
if (degrade) {
|
|
107
|
+
console.warn('[vision-helper] AI_VISION_API_KEY not set. Skipping visual assertion.');
|
|
108
|
+
return { passes: true, reason: 'SKIPPED: AI_VISION_API_KEY not configured' };
|
|
109
|
+
}
|
|
110
|
+
return { passes: false, reason: 'AI_VISION_API_KEY not set in environment' };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const base64Image = screenshotBuffer.toString('base64');
|
|
114
|
+
const model = options.model ?? 'gpt-4o';
|
|
115
|
+
|
|
116
|
+
const prompt = `You are a QA assistant verifying screenshots of a web application.
|
|
117
|
+
Answer YES or NO: Does the screenshot clearly show ${contentDescription}?
|
|
118
|
+
Reply in exactly this format:
|
|
119
|
+
VERDICT: YES
|
|
120
|
+
REASON: <one short sentence>`;
|
|
121
|
+
|
|
122
|
+
const body = JSON.stringify({
|
|
123
|
+
model,
|
|
124
|
+
max_tokens: 150,
|
|
125
|
+
messages: [{
|
|
126
|
+
role: 'user',
|
|
127
|
+
content: [
|
|
128
|
+
{ type: 'text', text: prompt },
|
|
129
|
+
{ type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}`, detail: 'low' } },
|
|
130
|
+
],
|
|
131
|
+
}],
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
const raw = await new Promise<string>((resolve, reject) => {
|
|
135
|
+
const req = https.request({
|
|
136
|
+
hostname: 'api.openai.com',
|
|
137
|
+
path: '/v1/chat/completions',
|
|
138
|
+
method: 'POST',
|
|
139
|
+
headers: {
|
|
140
|
+
'Content-Type': 'application/json',
|
|
141
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
142
|
+
'Content-Length': Buffer.byteLength(body),
|
|
143
|
+
},
|
|
144
|
+
}, (res) => {
|
|
145
|
+
let data = '';
|
|
146
|
+
res.on('data', (chunk) => { data += chunk; });
|
|
147
|
+
res.on('end', () => resolve(data));
|
|
148
|
+
});
|
|
149
|
+
req.on('error', reject);
|
|
150
|
+
req.write(body);
|
|
151
|
+
req.end();
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
const parsed = JSON.parse(raw);
|
|
155
|
+
const text: string = parsed.choices?.[0]?.message?.content ?? '';
|
|
156
|
+
const verdict = text.match(/VERDICT:\s*(YES|NO)/i);
|
|
157
|
+
const reason = text.match(/REASON:\s*(.+)/i);
|
|
158
|
+
|
|
159
|
+
if (!verdict) return { passes: false, reason: `Unexpected response: ${text}`, raw: text };
|
|
160
|
+
return {
|
|
161
|
+
passes: verdict[1].toUpperCase() === 'YES',
|
|
162
|
+
reason: reason?.[1]?.trim() ?? text,
|
|
163
|
+
raw: text,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
**Usage in a test:**
|
|
169
|
+
|
|
170
|
+
```typescript
|
|
171
|
+
import { assertScreenshotShowsContent } from '../fixtures/vision-helper';
|
|
172
|
+
|
|
173
|
+
test('[TC-MOD-001] Export PDF contains expected content', async ({ page }) => {
|
|
174
|
+
// ... trigger export ...
|
|
175
|
+
const screenshot = await page.screenshot();
|
|
176
|
+
const result = await assertScreenshotShowsContent(
|
|
177
|
+
screenshot,
|
|
178
|
+
'a completed export confirmation message with a download link'
|
|
179
|
+
);
|
|
180
|
+
expect(result.passes, result.reason).toBe(true);
|
|
181
|
+
});
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
**Setup notes:**
|
|
185
|
+
|
|
186
|
+
- Copy `vision-helper.ts` into your `fixtures/` directory.
|
|
187
|
+
- Set `AI_VISION_API_KEY` in your `.env` file (not committed). The example uses the OpenAI
|
|
188
|
+
API; adapt the request body for any other vision model.
|
|
189
|
+
- By default, the helper degrades to a skip (passes: true) when the key is absent, so CI
|
|
190
|
+
runs without the key do not fail on this assertion.
|
|
191
|
+
- `pdf-parse` (v2+) exports a `PDFParse` class rather than a function. Use
|
|
192
|
+
`new PDFParse({ data: buffer })` and call `.getText()` on the instance.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Integration
|
|
1
|
+
# Integration - playwright-azure-reporter
|
|
2
2
|
|
|
3
3
|
Connects Playwright test results to Azure DevOps Test Plans automatically.
|
|
4
4
|
|
|
@@ -8,42 +8,98 @@ Connects Playwright test results to Azure DevOps Test Plans automatically.
|
|
|
8
8
|
npm install --save-dev @alex_neo/playwright-azure-reporter
|
|
9
9
|
```
|
|
10
10
|
|
|
11
|
-
##
|
|
11
|
+
## Environment variables
|
|
12
|
+
|
|
13
|
+
### Local development (`.env`)
|
|
12
14
|
|
|
13
15
|
```
|
|
14
|
-
|
|
15
|
-
|
|
16
|
+
ADO_ORG=<org-slug>
|
|
17
|
+
ADO_PROJECT=<project-name>
|
|
16
18
|
ADO_PLAN_ID=<plan-id>
|
|
17
|
-
AZURE_TOKEN=<personal-access-token>
|
|
19
|
+
AZURE_TOKEN=<personal-access-token> # PAT with Test Management: Read & Write
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
> NEVER commit `.env` files to source control.
|
|
23
|
+
|
|
24
|
+
### CI/CD pipeline (`QA-Variables` Library Variable Group)
|
|
25
|
+
|
|
26
|
+
In the pipeline, `SYSTEM_ACCESSTOKEN` (OAuth) is used instead of a PAT -
|
|
27
|
+
no `AZURE_TOKEN` is needed. The reporter accepts both authentication methods.
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
ADO_ORG=<org-slug>
|
|
31
|
+
ADO_PROJECT=<project-name>
|
|
32
|
+
ADO_E2E_PLAN_ID=<plan-id>
|
|
33
|
+
ADO_API_PLAN_ID=<plan-id>
|
|
18
34
|
```
|
|
19
35
|
|
|
20
|
-
|
|
21
|
-
> NEVER store tokens in `.env` files committed to source control.
|
|
36
|
+
`SYSTEM_ACCESSTOKEN` is injected automatically by the pipeline via `$(System.AccessToken)`.
|
|
22
37
|
|
|
23
38
|
## playwright.config.ts snippet
|
|
24
39
|
|
|
25
40
|
```typescript
|
|
26
|
-
import
|
|
41
|
+
import { defineConfig } from '@playwright/test';
|
|
27
42
|
|
|
28
|
-
|
|
43
|
+
export default defineConfig({
|
|
29
44
|
reporter: [
|
|
30
45
|
['list'],
|
|
31
46
|
['html', { open: 'never' }],
|
|
32
47
|
['@alex_neo/playwright-azure-reporter', {
|
|
33
|
-
token:
|
|
34
|
-
planId:
|
|
35
|
-
projectName:
|
|
36
|
-
orgUrl:
|
|
37
|
-
testRunTitle:
|
|
48
|
+
token: process.env.SYSTEM_ACCESSTOKEN ?? process.env.AZURE_TOKEN!,
|
|
49
|
+
planId: Number(process.env.ADO_PLAN_ID),
|
|
50
|
+
projectName: process.env.ADO_PROJECT!,
|
|
51
|
+
orgUrl: `https://dev.azure.com/${process.env.ADO_ORG}`,
|
|
52
|
+
testRunTitle: `[Automated] ${new Date().toISOString().slice(0, 10)}`,
|
|
38
53
|
publishResultsOnFailure: true,
|
|
39
|
-
isDisabled: process.env.ADO_SYNC_DISABLED === 'true',
|
|
40
54
|
}],
|
|
41
55
|
],
|
|
42
|
-
};
|
|
56
|
+
});
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
> `token` falls back to `AZURE_TOKEN` when `SYSTEM_ACCESSTOKEN` is not present (local runs).
|
|
60
|
+
|
|
61
|
+
## Playwright Workspaces reporter (optional)
|
|
62
|
+
|
|
63
|
+
To also publish results to the Azure Playwright Workspaces dashboard, add
|
|
64
|
+
`@azure/playwright` alongside `@alex_neo/playwright-azure-reporter`.
|
|
43
65
|
|
|
44
|
-
|
|
66
|
+
```bash
|
|
67
|
+
npm install --save-dev @azure/playwright
|
|
45
68
|
```
|
|
46
69
|
|
|
70
|
+
Create a `playwright.service.config.ts` file next to `playwright.config.ts`:
|
|
71
|
+
|
|
72
|
+
```typescript
|
|
73
|
+
import { defineConfig } from '@playwright/test';
|
|
74
|
+
import { createAzurePlaywrightConfig } from '@azure/playwright';
|
|
75
|
+
import { DefaultAzureCredential } from '@azure/identity';
|
|
76
|
+
import config from './playwright.config';
|
|
77
|
+
|
|
78
|
+
export default defineConfig(
|
|
79
|
+
config,
|
|
80
|
+
createAzurePlaywrightConfig(config, {
|
|
81
|
+
credential: new DefaultAzureCredential(),
|
|
82
|
+
})
|
|
83
|
+
);
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Run using the service config in CI:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
npx playwright test -c playwright.service.config.ts
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Required variable (add to `QA-Variables` Library Variable Group):
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
PLAYWRIGHT_SERVICE_URL=<workspace-endpoint-url> # Azure Portal -> Playwright workspace -> Get Started
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Authentication uses Microsoft Entra ID via the pipeline's Azure service connection
|
|
99
|
+
(`AzureCLI@2` login task). No `PLAYWRIGHT_SERVICE_ACCESS_TOKEN` is needed when
|
|
100
|
+
Entra ID is configured. See `pipelines/azure-pipeline-qa.yml` for the commented
|
|
101
|
+
placeholder steps.
|
|
102
|
+
|
|
47
103
|
## Test case title format (required for ADO sync)
|
|
48
104
|
|
|
49
105
|
The reporter matches test titles to ADO Work Items by the numeric prefix.
|
|
@@ -73,16 +129,11 @@ file/describe block to its ADO Test Suite. Example:
|
|
|
73
129
|
}
|
|
74
130
|
```
|
|
75
131
|
|
|
76
|
-
## Disabling ADO sync locally
|
|
77
|
-
|
|
78
|
-
Set `ADO_SYNC_DISABLED=true` in your `.env` to skip reporting while running
|
|
79
|
-
tests locally.
|
|
80
|
-
|
|
81
132
|
## Troubleshooting
|
|
82
133
|
|
|
83
134
|
| Symptom | Cause | Fix |
|
|
84
135
|
|---------|-------|-----|
|
|
85
|
-
| `401 Unauthorized` | Expired PAT | Regenerate PAT in ADO
|
|
136
|
+
| `401 Unauthorized` (local) | Expired PAT | Regenerate PAT in ADO -> User Settings |
|
|
137
|
+
| `401 Unauthorized` (pipeline) | `SYSTEM_ACCESSTOKEN` not passed | Verify `env: SYSTEM_ACCESSTOKEN: $(System.AccessToken)` in the pipeline step |
|
|
86
138
|
| `Test case not found` | WI ID mismatch | Re-run `inject-ado-ids.ps1` |
|
|
87
|
-
| Results not published | `isDisabled: true` | Remove or set to `false` |
|
|
88
139
|
| Duplicate test runs | Reporter called twice | Check for duplicate reporter arrays |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@keber/qa-framework",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.11.3",
|
|
4
4
|
"description": "Reusable spec-driven QA framework for IDE-agent-assisted automated testing. Installable as an npm package. Provides structure, templates, agent instructions, and optional integrations for Playwright and Azure DevOps.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"qa",
|
|
@@ -38,10 +38,9 @@
|
|
|
38
38
|
"engines": {
|
|
39
39
|
"node": ">=18"
|
|
40
40
|
},
|
|
41
|
-
"devDependencies": {},
|
|
42
41
|
"peerDependencies": {
|
|
43
|
-
"@
|
|
44
|
-
"@
|
|
42
|
+
"@keber/ado-qa": ">=1.0.0",
|
|
43
|
+
"@playwright/test": ">=1.40.0"
|
|
45
44
|
},
|
|
46
45
|
"peerDependenciesMeta": {
|
|
47
46
|
"@playwright/test": {
|
package/qa-framework.config.json
CHANGED
package/scripts/init.js
CHANGED
|
@@ -84,6 +84,7 @@ const topLevelFolders = [
|
|
|
84
84
|
'04-test-data',
|
|
85
85
|
'05-test-execution',
|
|
86
86
|
'06-defects/open',
|
|
87
|
+
'06-defects/disputed',
|
|
87
88
|
'06-defects/resolved',
|
|
88
89
|
'07-automation/e2e/tests',
|
|
89
90
|
'07-automation/e2e/tests/helpers/debug',
|
|
@@ -98,6 +99,18 @@ for (const folder of topLevelFolders) {
|
|
|
98
99
|
console.log(` [created] ${path.relative(cwd, fullPath)}/`);
|
|
99
100
|
}
|
|
100
101
|
|
|
102
|
+
// --- 06-defects/disputed/README.md ---
|
|
103
|
+
writeIfMissing(path.join(qaRoot, '06-defects', 'disputed', 'README.md'),
|
|
104
|
+
`# 06-defects/disputed/
|
|
105
|
+
|
|
106
|
+
Use this folder for defects where QA has reproducible evidence but the business or
|
|
107
|
+
development team disputes whether the observed behavior is actually a defect.
|
|
108
|
+
|
|
109
|
+
A defect stays here while the functional decision is pending. Once the team reaches
|
|
110
|
+
a formal conclusion, move the file to open/ (confirmed defect) or resolved/ (accepted
|
|
111
|
+
behavior or won't fix).
|
|
112
|
+
`);
|
|
113
|
+
|
|
101
114
|
// --- qa/README.md ---
|
|
102
115
|
const qaReadmeTemplate = path.resolve(__dirname, '..', 'templates', 'qa-readme.md');
|
|
103
116
|
if (fs.existsSync(qaReadmeTemplate)) {
|
|
@@ -120,10 +133,17 @@ writeIfMissing(path.join(memoryDir, 'INDEX.md'),
|
|
|
120
133
|
|
|
121
134
|
| File | Topic | When to load |
|
|
122
135
|
|---|---|---|
|
|
123
|
-
|
|
124
|
-
|
|
136
|
+
| \`ci-pipeline-findings.md\` | CI pipeline issues, root causes, applied fixes, failure triage categories | Diagnosing CI failures; reviewing pipeline changes |
|
|
137
|
+
| \`e2e-stabilization-patterns.md\` | Flakiness patterns, crash-recovery techniques, strict-mode conflicts, reference timeouts | Investigating intermittent failures; refactoring fragile specs |
|
|
138
|
+
| \`data-volatility-strategies.md\` | Strategies for handling volatile test data (filter values, row counts, search inputs) | Fixing hardcoded-data failures; choosing an assertion resilience approach |
|
|
125
139
|
`);
|
|
126
140
|
|
|
141
|
+
// --- qa/memory template files ---
|
|
142
|
+
const memorySrc = path.resolve(__dirname, '..', 'templates', 'memory');
|
|
143
|
+
for (const file of ['ci-pipeline-findings.md', 'e2e-stabilization-patterns.md', 'data-volatility-strategies.md']) {
|
|
144
|
+
writeIfMissing(path.join(memoryDir, file), fs.readFileSync(path.join(memorySrc, file), 'utf8'));
|
|
145
|
+
}
|
|
146
|
+
|
|
127
147
|
// --- 03-test-cases README (optional directory marker) ---
|
|
128
148
|
writeIfMissing(path.join(qaRoot, '03-test-cases', 'README.md'),
|
|
129
149
|
`# 03-test-cases/ — Optional Standalone Test Cases
|
|
@@ -227,7 +247,7 @@ for (const file of ['playwright.config.ts', 'global-setup.ts', '.env.example', '
|
|
|
227
247
|
}
|
|
228
248
|
const fixturesDir = path.join(e2eScaffoldDir, 'fixtures');
|
|
229
249
|
fs.mkdirSync(fixturesDir, { recursive: true });
|
|
230
|
-
for (const file of ['auth.ts', 'test-helpers.ts']) {
|
|
250
|
+
for (const file of ['auth.ts', 'test-helpers.ts', 'base.ts']) {
|
|
231
251
|
const dest = path.join(fixturesDir, file);
|
|
232
252
|
if (!fs.existsSync(dest)) {
|
|
233
253
|
fs.copyFileSync(path.join(scaffoldSrc, 'fixtures', file), dest);
|
|
@@ -235,9 +255,26 @@ for (const file of ['auth.ts', 'test-helpers.ts']) {
|
|
|
235
255
|
}
|
|
236
256
|
}
|
|
237
257
|
|
|
238
|
-
// --- Integration
|
|
239
|
-
|
|
240
|
-
|
|
258
|
+
// --- Integration scaffold (lives in 07-automation/integration/) ---
|
|
259
|
+
const integrationDir = path.join(automationDir, 'integration');
|
|
260
|
+
const integrationScaffold = path.resolve(__dirname, '..', 'templates', 'integration-scaffold');
|
|
261
|
+
fs.mkdirSync(integrationDir, { recursive: true });
|
|
262
|
+
for (const file of ['package.json', 'playwright.config.ts', 'global-setup.ts', 'README.md', '.env.example']) {
|
|
263
|
+
const dest = path.join(integrationDir, file);
|
|
264
|
+
if (!fs.existsSync(dest)) {
|
|
265
|
+
fs.copyFileSync(path.join(integrationScaffold, file), dest);
|
|
266
|
+
console.log(` [created] ${path.relative(cwd, dest)}`);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
const integrationTestsDir = path.join(integrationDir, 'tests');
|
|
270
|
+
fs.mkdirSync(integrationTestsDir, { recursive: true });
|
|
271
|
+
const exampleSpec = path.join(integrationTestsDir, 'example.spec.ts');
|
|
272
|
+
if (!fs.existsSync(exampleSpec)) {
|
|
273
|
+
fs.copyFileSync(path.join(integrationScaffold, 'tests', 'example.spec.ts'), exampleSpec);
|
|
274
|
+
console.log(` [created] ${path.relative(cwd, exampleSpec)}`);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// --- Load test placeholder ---
|
|
241
278
|
writeIfMissing(path.join(automationDir, 'load', 'README.md'),
|
|
242
279
|
`# Load Tests\n\n> Placeholder for load and performance tests.\n> Each tool gets its own subdirectory (e.g. k6/, jmeter/).\n`);
|
|
243
280
|
|
|
@@ -272,37 +309,9 @@ if (fs.existsSync(structureGuideSrc)) {
|
|
|
272
309
|
const githubDir = path.join(cwd, '.github');
|
|
273
310
|
const copilotInstrPath = path.join(githubDir, 'instructions', 'qa-framework.instructions.md');
|
|
274
311
|
|
|
275
|
-
// Detect whether the ado-qa skill is available (installed or already copied)
|
|
276
|
-
const adoQaInstalled = fs.existsSync(path.join(cwd, 'node_modules', '@keber', 'ado-qa'))
|
|
277
|
-
|| fs.existsSync(path.join(cwd, 'node_modules', 'ado-qa'));
|
|
278
|
-
const adoSkillReady = fs.existsSync(path.join(cwd, '.github', 'skills', 'ado-qa'));
|
|
279
|
-
const adoAvailable = adoQaInstalled || adoSkillReady;
|
|
280
|
-
|
|
281
|
-
const adoSection = adoAvailable
|
|
282
|
-
? `## Azure DevOps integration (ado-qa)
|
|
283
|
-
|
|
284
|
-
The \`ado-qa\` skill is installed. Load it in PowerShell before any ADO operation:
|
|
285
|
-
\`\`\`powershell
|
|
286
|
-
. .github/skills/ado-qa/load.ps1
|
|
287
|
-
\`\`\`
|
|
288
|
-
Refer to \`qa/00-guides/AGENT-INSTRUCTIONS-ADO-INTEGRATION.md\` for the full ADO workflow.
|
|
289
|
-
|
|
290
|
-
`
|
|
291
|
-
: `## Azure DevOps integration (not installed)
|
|
292
|
-
|
|
293
|
-
> ⚠️ **Agent note:** \`ado-qa\` is not installed in this project.
|
|
294
|
-
> To enable Azure DevOps integration (query Work Items, Test Plans, create bugs, etc.) run:
|
|
295
|
-
> \`\`\`bash
|
|
296
|
-
> npm install github:keber/ado-qa
|
|
297
|
-
> \`\`\`
|
|
298
|
-
> This will automatically copy the ADO skills to \`.github/skills/\` via postinstall.
|
|
299
|
-
|
|
300
|
-
`;
|
|
301
|
-
|
|
302
312
|
const instrTemplatePath = path.resolve(__dirname, '..', 'templates', 'qa-framework.instructions.md');
|
|
303
313
|
const copilotInstrContent = fs.readFileSync(instrTemplatePath, 'utf8')
|
|
304
|
-
.replace('{{VERSION}}', config.frameworkVersion ?? '1.0.0')
|
|
305
|
-
.replace('{{ADO_SECTION}}', adoSection);
|
|
314
|
+
.replace('{{VERSION}}', config.frameworkVersion ?? '1.0.0');
|
|
306
315
|
writeIfMissing(copilotInstrPath, copilotInstrContent);
|
|
307
316
|
|
|
308
317
|
// --- AGENT-NEXT-STEPS.md — readable by the agent after install ---
|
|
@@ -328,15 +337,21 @@ Edit \`qa/qa-framework.config.json\` and add your project modules under the \`"m
|
|
|
328
337
|
npm install @keber/ado-qa
|
|
329
338
|
\`\`\`
|
|
330
339
|
|
|
331
|
-
**Playwright automation:**
|
|
340
|
+
**Playwright E2E automation:**
|
|
332
341
|
\`\`\`bash
|
|
333
342
|
npm install @playwright/test
|
|
334
343
|
cd qa/07-automation/e2e && npm install && npx playwright install chromium
|
|
335
344
|
\`\`\`
|
|
336
345
|
|
|
346
|
+
**Playwright API/integration automation:**
|
|
347
|
+
\`\`\`bash
|
|
348
|
+
cd qa/07-automation/integration && npm install
|
|
349
|
+
\`\`\`
|
|
350
|
+
|
|
337
351
|
### 3. Credentials
|
|
338
|
-
- Copy \`qa/07-automation/e2e/.env.example\`
|
|
339
|
-
-
|
|
352
|
+
- Copy \`qa/07-automation/e2e/.env.example\` -> \`qa/07-automation/e2e/.env\` and fill in credentials
|
|
353
|
+
- Copy \`qa/07-automation/integration/.env.example\` -> \`qa/07-automation/integration/.env\` and fill in credentials
|
|
354
|
+
- Add both \`.env\` files and both \`.auth/\` folders to \`.gitignore\`
|
|
340
355
|
`;
|
|
341
356
|
const nextStepsPath = path.join(qaRoot, 'AGENT-NEXT-STEPS.md');
|
|
342
357
|
writeIfMissing(nextStepsPath, nextStepsContent);
|
package/scripts/upgrade.js
CHANGED
|
@@ -93,8 +93,7 @@ if (!fs.existsSync(skillsSrc)) {
|
|
|
93
93
|
const copilotInstrPath = path.join(githubDir, 'instructions', 'qa-framework.instructions.md');
|
|
94
94
|
const instrTemplatePath = path.resolve(__dirname, '..', 'templates', 'qa-framework.instructions.md');
|
|
95
95
|
const copilotContent = fs.readFileSync(instrTemplatePath, 'utf8')
|
|
96
|
-
.replace('{{VERSION}}', config.frameworkVersion ?? '1.0.0')
|
|
97
|
-
.replace('{{ADO_SECTION}}', '');
|
|
96
|
+
.replace('{{VERSION}}', config.frameworkVersion ?? '1.0.0');
|
|
98
97
|
forceWrite(copilotInstrPath, copilotContent);
|
|
99
98
|
|
|
100
99
|
// ---------------------------------------------------------------------------
|
|
@@ -175,6 +174,22 @@ for (const file of scaffoldFiles) {
|
|
|
175
174
|
}
|
|
176
175
|
}
|
|
177
176
|
|
|
177
|
+
// 5a.1. Ensure the refreshed fixtures scaffold exists.
|
|
178
|
+
const scaffoldFixturesDir = path.join(e2eDir, 'fixtures');
|
|
179
|
+
const fixtureFiles = ['auth.ts', 'test-helpers.ts', 'base.ts'];
|
|
180
|
+
for (const file of fixtureFiles) {
|
|
181
|
+
const oldPath = path.join(automationDir, 'fixtures', file);
|
|
182
|
+
const newPath = path.join(scaffoldFixturesDir, file);
|
|
183
|
+
if (fs.existsSync(oldPath) && !fs.existsSync(newPath)) {
|
|
184
|
+
if (!dryRun) {
|
|
185
|
+
fs.mkdirSync(scaffoldFixturesDir, { recursive: true });
|
|
186
|
+
fs.renameSync(oldPath, newPath);
|
|
187
|
+
}
|
|
188
|
+
updated.push(newPath);
|
|
189
|
+
console.log(` [migrated] fixtures/${file} -> e2e/fixtures/${file}`);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
178
193
|
// 5b. fixtures/: 07-automation/fixtures/ -> 07-automation/e2e/fixtures/
|
|
179
194
|
const oldFixturesDir = path.join(automationDir, 'fixtures');
|
|
180
195
|
const newFixturesDir = path.join(e2eDir, 'fixtures');
|
|
@@ -211,17 +226,34 @@ if (fs.existsSync(e2eDir)) {
|
|
|
211
226
|
}
|
|
212
227
|
}
|
|
213
228
|
|
|
214
|
-
// 5d. Create integration
|
|
215
|
-
const
|
|
216
|
-
const
|
|
229
|
+
// 5d. Create integration scaffold if missing (uses template; existing files are never overwritten).
|
|
230
|
+
const integrationDir = path.join(automationDir, 'integration');
|
|
231
|
+
const integrationScaffold = path.resolve(__dirname, '..', 'templates', 'integration-scaffold');
|
|
232
|
+
const integrationReadme = path.join(integrationDir, 'README.md');
|
|
233
|
+
const loadReadme = path.join(automationDir, 'load', 'README.md');
|
|
217
234
|
if (!fs.existsSync(integrationReadme)) {
|
|
218
235
|
if (!dryRun) {
|
|
219
|
-
fs.mkdirSync(
|
|
220
|
-
|
|
221
|
-
|
|
236
|
+
fs.mkdirSync(integrationDir, { recursive: true });
|
|
237
|
+
for (const file of ['package.json', 'playwright.config.ts', 'global-setup.ts', 'README.md', '.env.example']) {
|
|
238
|
+
const dest = path.join(integrationDir, file);
|
|
239
|
+
if (!fs.existsSync(dest)) {
|
|
240
|
+
fs.copyFileSync(path.join(integrationScaffold, file), dest);
|
|
241
|
+
updated.push(dest);
|
|
242
|
+
console.log(` [created] ${path.relative(cwd, dest)}`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
const integrationTestsDir = path.join(integrationDir, 'tests');
|
|
246
|
+
fs.mkdirSync(integrationTestsDir, { recursive: true });
|
|
247
|
+
const exampleSpec = path.join(integrationTestsDir, 'example.spec.ts');
|
|
248
|
+
if (!fs.existsSync(exampleSpec)) {
|
|
249
|
+
fs.copyFileSync(path.join(integrationScaffold, 'tests', 'example.spec.ts'), exampleSpec);
|
|
250
|
+
updated.push(exampleSpec);
|
|
251
|
+
console.log(` [created] ${path.relative(cwd, exampleSpec)}`);
|
|
252
|
+
}
|
|
253
|
+
} else {
|
|
254
|
+
updated.push(integrationReadme);
|
|
255
|
+
console.log(` [created] ${path.relative(cwd, integrationReadme)}`);
|
|
222
256
|
}
|
|
223
|
-
updated.push(integrationReadme);
|
|
224
|
-
console.log(` [created] ${path.relative(cwd, integrationReadme)}`);
|
|
225
257
|
}
|
|
226
258
|
if (!fs.existsSync(loadReadme)) {
|
|
227
259
|
if (!dryRun) {
|
|
@@ -346,6 +378,69 @@ if (!fs.existsSync(testCasesReadme)) {
|
|
|
346
378
|
console.log(` [created] 03-test-cases/README.md`);
|
|
347
379
|
}
|
|
348
380
|
|
|
381
|
+
// ---------------------------------------------------------------------------
|
|
382
|
+
// 7. Ensure 06-defects/disputed/ exists (added in v1.11.0)
|
|
383
|
+
// Safe: only creates if missing. Never touches existing files.
|
|
384
|
+
// ---------------------------------------------------------------------------
|
|
385
|
+
const defectsDir = path.join(qaRoot, '06-defects');
|
|
386
|
+
const disputedDir = path.join(defectsDir, 'disputed');
|
|
387
|
+
const disputedReadme = path.join(disputedDir, 'README.md');
|
|
388
|
+
|
|
389
|
+
if (fs.existsSync(defectsDir) && !fs.existsSync(disputedDir)) {
|
|
390
|
+
if (!dryRun) {
|
|
391
|
+
fs.mkdirSync(disputedDir, { recursive: true });
|
|
392
|
+
fs.writeFileSync(disputedReadme,
|
|
393
|
+
`# 06-defects/disputed/
|
|
394
|
+
|
|
395
|
+
Use this folder for defects where QA has reproducible evidence but the business or
|
|
396
|
+
development team disputes whether the observed behavior is actually a defect.
|
|
397
|
+
|
|
398
|
+
A defect stays here while the functional decision is pending. Once the team reaches
|
|
399
|
+
a formal conclusion, move the file to open/ (confirmed defect) or resolved/ (accepted
|
|
400
|
+
behavior or won't fix).
|
|
401
|
+
`, 'utf8');
|
|
402
|
+
}
|
|
403
|
+
updated.push(disputedDir);
|
|
404
|
+
console.log(` [created] 06-defects/disputed/`);
|
|
405
|
+
console.log(` [created] 06-defects/disputed/README.md`);
|
|
406
|
+
} else if (!fs.existsSync(disputedReadme) && fs.existsSync(disputedDir)) {
|
|
407
|
+
if (!dryRun) {
|
|
408
|
+
fs.writeFileSync(disputedReadme,
|
|
409
|
+
`# 06-defects/disputed/
|
|
410
|
+
|
|
411
|
+
Use this folder for defects where QA has reproducible evidence but the business or
|
|
412
|
+
development team disputes whether the observed behavior is actually a defect.
|
|
413
|
+
|
|
414
|
+
A defect stays here while the functional decision is pending. Once the team reaches
|
|
415
|
+
a formal conclusion, move the file to open/ (confirmed defect) or resolved/ (accepted
|
|
416
|
+
behavior or won't fix).
|
|
417
|
+
`, 'utf8');
|
|
418
|
+
}
|
|
419
|
+
updated.push(disputedReadme);
|
|
420
|
+
console.log(` [created] 06-defects/disputed/README.md`);
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// ---------------------------------------------------------------------------
|
|
424
|
+
// 8. Seed qa/memory/ template files if missing (added in v1.11.0)
|
|
425
|
+
// Safe: only creates files that do not exist. Never overwrites user content.
|
|
426
|
+
// ---------------------------------------------------------------------------
|
|
427
|
+
const memoryDir = path.join(qaRoot, 'memory');
|
|
428
|
+
if (fs.existsSync(memoryDir)) {
|
|
429
|
+
const memorySrc = path.resolve(__dirname, '..', 'templates', 'memory');
|
|
430
|
+
for (const file of ['ci-pipeline-findings.md', 'e2e-stabilization-patterns.md', 'data-volatility-strategies.md']) {
|
|
431
|
+
const dest = path.join(memoryDir, file);
|
|
432
|
+
if (!fs.existsSync(dest)) {
|
|
433
|
+
if (!dryRun) {
|
|
434
|
+
fs.copyFileSync(path.join(memorySrc, file), dest);
|
|
435
|
+
}
|
|
436
|
+
updated.push(dest);
|
|
437
|
+
console.log(` [created] memory/${file}`);
|
|
438
|
+
} else {
|
|
439
|
+
skipped.push(dest);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
349
444
|
// ---------------------------------------------------------------------------
|
|
350
445
|
// Summary
|
|
351
446
|
// ---------------------------------------------------------------------------
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# -----------------------------------------------------------------------
|
|
1
|
+
# -----------------------------------------------------------------------
|
|
2
2
|
# keber/qa-framework — Environment Variables Reference
|
|
3
3
|
# -----------------------------------------------------------------------
|
|
4
4
|
# Copy this file to .env and fill in real values.
|
|
@@ -10,6 +10,20 @@
|
|
|
10
10
|
# -----------------------------------------------------------------------
|
|
11
11
|
QA_BASE_URL=https://your-app.qa.example.com
|
|
12
12
|
|
|
13
|
+
# -----------------------------------------------------------------------
|
|
14
|
+
# Session and login resilience
|
|
15
|
+
# -----------------------------------------------------------------------
|
|
16
|
+
QA_SESSION_TTL_MS=7200000
|
|
17
|
+
# Optional path to a server-side logout endpoint if the app exposes one.
|
|
18
|
+
# QA_LOGOUT_PATH=/logout
|
|
19
|
+
|
|
20
|
+
# When parallel user mode is enabled, limit auth to the matching project(s).
|
|
21
|
+
# QA_AUTH_USER=1
|
|
22
|
+
|
|
23
|
+
# Optional second test user for 2-project parallel execution.
|
|
24
|
+
# QA_USER2_EMAIL=qa-user2@example.com
|
|
25
|
+
# QA_USER2_PASSWORD=CHANGE_ME
|
|
26
|
+
|
|
13
27
|
# -----------------------------------------------------------------------
|
|
14
28
|
# Login Configuration
|
|
15
29
|
# These selectors allow the framework to work with any login form.
|
|
@@ -54,3 +68,8 @@ QA_USER_PASSWORD=CHANGE_ME
|
|
|
54
68
|
# -----------------------------------------------------------------------
|
|
55
69
|
# CI=true # Set automatically by most CI systems
|
|
56
70
|
# ADO_SYNC_DISABLED=false # Set to true to skip ADO reporter in local runs
|
|
71
|
+
|
|
72
|
+
# -----------------------------------------------------------------------
|
|
73
|
+
# Playwright Service (optional - only needed for cloud or self-hosted grid)
|
|
74
|
+
# -----------------------------------------------------------------------
|
|
75
|
+
# PLAYWRIGHT_SERVICE_URL=wss://eastus.api.playwright.microsoft.com/accounts/<your-id>/browsers
|