@keber/qa-framework 1.5.2 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -8
- package/README.md +150 -76
- package/docs/architecture.md +49 -44
- package/docs/folder-structure-guide.md +80 -23
- package/docs/installation.md +53 -43
- package/docs/usage-with-agent.md +52 -64
- package/integrations/ado-powershell/pipelines/azure-pipeline-qa.yml +1 -1
- package/integrations/playwright/README.md +1 -1
- package/package.json +1 -1
- package/scripts/init.js +22 -12
- package/scripts/upgrade.js +139 -11
- package/skills/qa-ado-integration/SKILL.md +2 -2
- package/skills/qa-automation/SKILL.md +6 -6
- package/skills/qa-maintenance/SKILL.md +2 -2
- package/templates/automation-scaffold/playwright.config.ts +3 -1
- package/agent-instructions/00-module-analysis.md +0 -266
- package/agent-instructions/01-spec-generation.md +0 -281
- package/agent-instructions/02-test-plan-generation.md +0 -205
- package/agent-instructions/03-test-case-generation.md +0 -150
- package/agent-instructions/04-automation-generation.md +0 -343
- package/agent-instructions/04b-test-stabilization.md +0 -309
- package/agent-instructions/05-ado-integration.md +0 -247
- package/agent-instructions/06-maintenance.md +0 -135
- package/docs/comparison-matrix.md +0 -131
- package/docs/generalization-decisions.md +0 -203
|
@@ -31,6 +31,59 @@ Agent skills live in `.github/skills/` (not inside `qa/`). See the `.github/skil
|
|
|
31
31
|
|
|
32
32
|
---
|
|
33
33
|
|
|
34
|
+
## Dual-Track Pipeline
|
|
35
|
+
|
|
36
|
+
All pipeline paths share the same input: `01-specifications/`. From there, work splits into
|
|
37
|
+
two tracks that can run in parallel or independently depending on team maturity.
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
01-specifications/ (source of truth — what the app does and what must be tested)
|
|
41
|
+
│
|
|
42
|
+
├─── Automation track ──────────────────────────────────────────────────
|
|
43
|
+
│ │
|
|
44
|
+
│ ▼
|
|
45
|
+
│ 02-test-plans/automated/ ← scope + priority decisions
|
|
46
|
+
│ │
|
|
47
|
+
│ ▼
|
|
48
|
+
│ 07-automation/e2e/tests/ ← Playwright .spec.ts files
|
|
49
|
+
│ │
|
|
50
|
+
│ ▼
|
|
51
|
+
│ 05-test-execution/automated/ ← execution reports
|
|
52
|
+
│ │
|
|
53
|
+
│ ▼
|
|
54
|
+
│ 08-azure-integration/ ← ADO sync via playwright-azure-reporter
|
|
55
|
+
│
|
|
56
|
+
└─── Manual track ──────────────────────────────────────────────────────
|
|
57
|
+
│
|
|
58
|
+
▼
|
|
59
|
+
02-test-plans/manual/ ← scope + manual test approach
|
|
60
|
+
│
|
|
61
|
+
▼
|
|
62
|
+
03-test-cases/manual/ ← TC-*.md with steps (for human testers)
|
|
63
|
+
│
|
|
64
|
+
▼
|
|
65
|
+
05-test-execution/manual/ ← manual run evidence
|
|
66
|
+
│
|
|
67
|
+
▼
|
|
68
|
+
08-azure-integration/ ← ADO sync via ado-integration skill
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
**Entry points:**
|
|
72
|
+
|
|
73
|
+
| Scenario | Entry point | Tracks active |
|
|
74
|
+
|---|---|---|
|
|
75
|
+
| New system, no prior QA | Module analysis → spec generation | Automation (primary), Manual (as needed) |
|
|
76
|
+
| Sprint with ADO stories | Stories → spec generation → test plan | Both, driven by sprint scope |
|
|
77
|
+
| Maintenance/continuous improvement | Changed module → maintenance skill | Automation (update existing tests) |
|
|
78
|
+
|
|
79
|
+
**Track selection guidance:**
|
|
80
|
+
- Use **automation track** when the goal is regression coverage and CI integration.
|
|
81
|
+
- Use **manual track** when: (a) team has manual testers who need step-by-step documents; (b) auditability or traceability to external standards is required; (c) features are not yet automatable.
|
|
82
|
+
- Both tracks use the same TC-IDs from `01-specifications/*/05-test-scenarios.md` — traceability is preserved regardless of which track executes a given TC.
|
|
83
|
+
- `04-test-data/` is optional in both tracks. Use it when test data factories or seeders are shared across multiple modules. Per-submodule data belongs in `01-specifications/*/04-test-data.md`.
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
34
87
|
## .github/skills/
|
|
35
88
|
|
|
36
89
|
**Purpose**: 3-layer QA pipeline skills installed by the framework. Loaded by the agent on demand — only the relevant skill for the current task is loaded, not all skills at once.
|
|
@@ -197,7 +250,7 @@ Most projects will have sparse content here — the 05-test-scenarios.md files i
|
|
|
197
250
|
```
|
|
198
251
|
06-defects/
|
|
199
252
|
├── README.md
|
|
200
|
-
├──
|
|
253
|
+
├── open/
|
|
201
254
|
│ └── DEF-{NNN}-{slug}.md
|
|
202
255
|
└── resolved/
|
|
203
256
|
└── DEF-{NNN}-{slug}.md
|
|
@@ -218,32 +271,36 @@ See `docs/generalization-decisions.md §7` for the full evaluation of this folde
|
|
|
218
271
|
|
|
219
272
|
```
|
|
220
273
|
07-automation/
|
|
221
|
-
├── README.md
|
|
222
|
-
├── e2e/
|
|
223
|
-
│ ├──
|
|
224
|
-
│ ├──
|
|
225
|
-
│ ├── global-setup.ts
|
|
226
|
-
│ ├── .env
|
|
227
|
-
│ ├── .env.example
|
|
228
|
-
│ ├── .auth/
|
|
229
|
-
│ ├── fixtures/
|
|
230
|
-
│ │ ├── auth.ts
|
|
231
|
-
│ │ └── test-helpers.ts
|
|
232
|
-
│ ├── page-objects/
|
|
233
|
-
│ ├── tests/
|
|
234
|
-
│ │ ├── {module}/
|
|
235
|
-
│ │ ├── helpers/debug/
|
|
236
|
-
│ │ └── seeds/
|
|
237
|
-
│
|
|
238
|
-
|
|
239
|
-
└──
|
|
274
|
+
├── README.md <- How to run, where reports go, worker config
|
|
275
|
+
├── e2e/ <- Playwright E2E project (self-contained)
|
|
276
|
+
│ ├── playwright.config.ts <- Test runner config (installed by init)
|
|
277
|
+
│ ├── package.json <- Playwright dependencies (installed by init)
|
|
278
|
+
│ ├── global-setup.ts <- Pre-suite login + storageState save (installed by init)
|
|
279
|
+
│ ├── .env <- Local credentials (NEVER commit)
|
|
280
|
+
│ ├── .env.example <- Template (always commit, installed by init)
|
|
281
|
+
│ ├── .auth/ <- storageState files (gitignored)
|
|
282
|
+
│ ├── fixtures/ <- Installed by init
|
|
283
|
+
│ │ ├── auth.ts <- loginAs(page, role) helper
|
|
284
|
+
│ │ └── test-helpers.ts <- EXEC_IDX, date helpers, etc.
|
|
285
|
+
│ ├── page-objects/ <- Page Object Model classes (project-maintained)
|
|
286
|
+
│ ├── tests/ <- Spec files, organized by module
|
|
287
|
+
│ │ ├── {module}/ <- Stub .spec.ts files generated by init per submodule
|
|
288
|
+
│ │ ├── helpers/debug/ <- Diagnostic scripts (not part of suite)
|
|
289
|
+
│ │ └── seeds/ <- Data seeding specs (excluded from runs)
|
|
290
|
+
│ └── diagnosis/ <- Screenshots and debug output (gitignored)
|
|
291
|
+
├── integration/ <- API-level tests (k6, JMeter, Azure Load Testing)
|
|
292
|
+
│ └── README.md <- Integration test setup and run instructions
|
|
293
|
+
└── load/ <- Load / performance tests (future)
|
|
294
|
+
└── README.md <- Load test setup and run instructions
|
|
240
295
|
```
|
|
241
296
|
|
|
242
297
|
**Important rules**:
|
|
243
|
-
-
|
|
298
|
+
- Run Playwright from `qa/07-automation/e2e/` (where `playwright.config.ts` lives): `cd qa/07-automation/e2e && npx playwright test`
|
|
244
299
|
- Seeds are excluded from normal test runs via `testIgnore` in `playwright.config.ts`
|
|
245
|
-
- Debug artifacts go into `diagnosis/`
|
|
246
|
-
- `.auth/`
|
|
300
|
+
- Debug artifacts go into `e2e/diagnosis/` -- never into the repo root
|
|
301
|
+
- `.auth/` and `.env` are always gitignored
|
|
302
|
+
- Files installed by `init` must not be deleted -- they are the project's automation bootstrap
|
|
303
|
+
- `integration/` and `load/` are independent projects with their own toolchains (k6, JMeter, etc.) -- they do not depend on `e2e/`
|
|
247
304
|
|
|
248
305
|
---
|
|
249
306
|
|
package/docs/installation.md
CHANGED
|
@@ -16,56 +16,48 @@
|
|
|
16
16
|
## Option A — npm install (recommended)
|
|
17
17
|
|
|
18
18
|
```bash
|
|
19
|
-
|
|
20
|
-
npm install --save-dev keber/qa-framework
|
|
21
|
-
|
|
22
|
-
# Run the init command
|
|
23
|
-
npx qa-framework init
|
|
19
|
+
npm install --save-dev @keber/qa-framework
|
|
24
20
|
```
|
|
25
21
|
|
|
26
|
-
The `init`
|
|
22
|
+
The `postinstall` script runs `init --skip-if-exists` automatically. On first install it
|
|
23
|
+
creates the full `qa/` structure without prompts, reading `qa-framework.config.json` if
|
|
24
|
+
present or bootstrapping defaults.
|
|
27
25
|
|
|
28
|
-
|
|
29
|
-
2. **QA base URL** (e.g., `https://myproject-qa.example.com`)
|
|
30
|
-
3. **Login path** (e.g., `/login` or `/Seguridad/Login`)
|
|
31
|
-
4. **Number of test user roles** (1 = single role, 2+ = multi-role)
|
|
32
|
-
5. **Enable Playwright integration?** (y/n)
|
|
33
|
-
6. **Enable Azure DevOps integration?** (y/n)
|
|
26
|
+
After install, read `qa/AGENT-NEXT-STEPS.md` for the exact follow-up steps.
|
|
34
27
|
|
|
35
|
-
|
|
28
|
+
`init` creates:
|
|
36
29
|
|
|
37
30
|
```
|
|
38
31
|
qa/
|
|
39
|
-
├── README.md
|
|
40
|
-
├──
|
|
41
|
-
├──
|
|
42
|
-
├──
|
|
43
|
-
├── 00-standards/
|
|
44
|
-
├── 01-specifications/
|
|
45
|
-
├── 02-test-plans/
|
|
46
|
-
├── 03-test-cases/
|
|
47
|
-
├── 04-test-data/
|
|
48
|
-
├── 05-test-execution/
|
|
49
|
-
├── 06-defects/
|
|
50
|
-
├── 07-automation/
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
└── auth.ts
|
|
32
|
+
├── README.md <- Prefilled with your project name
|
|
33
|
+
├── AGENT-NEXT-STEPS.md <- Follow-up checklist for the agent
|
|
34
|
+
├── QA-STRUCTURE-GUIDE.md <- Full structure guide (framework-owned)
|
|
35
|
+
├── qa-framework.config.json <- Your project config
|
|
36
|
+
├── 00-standards/ <- Templates and naming conventions
|
|
37
|
+
├── 01-specifications/
|
|
38
|
+
├── 02-test-plans/
|
|
39
|
+
├── 03-test-cases/
|
|
40
|
+
├── 04-test-data/
|
|
41
|
+
├── 05-test-execution/
|
|
42
|
+
├── 06-defects/open|resolved/
|
|
43
|
+
├── 07-automation/
|
|
44
|
+
│ ├── e2e/ <- Playwright project (self-contained)
|
|
45
|
+
│ │ ├── playwright.config.ts
|
|
46
|
+
│ │ ├── global-setup.ts
|
|
47
|
+
│ │ ├── .env.example
|
|
48
|
+
│ │ └── fixtures/auth.ts + test-helpers.ts
|
|
49
|
+
│ ├── integration/README.md <- Placeholder for k6, JMeter, etc.
|
|
50
|
+
│ └── load/README.md <- Placeholder for load tests
|
|
51
|
+
└── 08-azure-integration/README.md
|
|
52
|
+
└── memory/INDEX.md
|
|
53
|
+
|
|
54
|
+
.github/copilot-instructions.md <- Generated agent rules
|
|
55
|
+
.github/skills/ <- 8 agent skill sets (SKILL.md + references/)
|
|
64
56
|
```
|
|
65
57
|
|
|
66
58
|
---
|
|
67
59
|
|
|
68
|
-
## Option B — Clone or copy (
|
|
60
|
+
## Option B — Clone or copy (no npm registry)
|
|
69
61
|
|
|
70
62
|
```bash
|
|
71
63
|
# From your project root
|
|
@@ -82,15 +74,33 @@ node tools/qa-framework/scripts/cli.js init
|
|
|
82
74
|
If you prefer to control exactly what gets created:
|
|
83
75
|
|
|
84
76
|
```bash
|
|
85
|
-
# Clone the package
|
|
86
77
|
git clone https://github.com/your-org/qa-framework.git tools/qa-framework
|
|
87
78
|
|
|
88
|
-
# Copy
|
|
79
|
+
# Copy templates
|
|
89
80
|
cp -r tools/qa-framework/templates/specification qa/01-specifications/shared/
|
|
90
|
-
cp -r tools/qa-framework/agent-instructions qa/00-guides/
|
|
91
81
|
cp -r tools/qa-framework/templates/automation-scaffold qa/07-automation/e2e/
|
|
92
82
|
|
|
93
|
-
# Then
|
|
83
|
+
# Then fill in qa-framework.config.json manually
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## Upgrading
|
|
89
|
+
|
|
90
|
+
After `npm update @keber/qa-framework`:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
npx qa-framework upgrade
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
This overwrites only framework-owned files (`.github/skills/`, `copilot-instructions.md`,
|
|
97
|
+
`QA-STRUCTURE-GUIDE.md`) and migrates the `07-automation/` structure if upgrading from
|
|
98
|
+
v1.5.x. Your specs, tests, and memory are never touched.
|
|
99
|
+
|
|
100
|
+
Use `--dry-run` to preview without writing:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
npx qa-framework upgrade --dry-run
|
|
94
104
|
```
|
|
95
105
|
|
|
96
106
|
---
|
package/docs/usage-with-agent.md
CHANGED
|
@@ -20,16 +20,13 @@ This framework is designed so that an IDE agent (such as GitHub Copilot in VS Co
|
|
|
20
20
|
|
|
21
21
|
```
|
|
22
22
|
# Install the framework
|
|
23
|
-
"Run
|
|
23
|
+
"Run npm install @keber/qa-framework for this project"
|
|
24
24
|
|
|
25
|
-
#
|
|
26
|
-
"Read
|
|
27
|
-
|
|
28
|
-
# Generate the QA structure
|
|
29
|
-
"Generate the full qa/ directory structure using the framework config"
|
|
25
|
+
# Let the agent self-configure
|
|
26
|
+
"Read qa/AGENT-NEXT-STEPS.md and complete the setup steps"
|
|
30
27
|
|
|
31
28
|
# Analyze a module
|
|
32
|
-
"Analyze the module at [URL/route]
|
|
29
|
+
"Analyze the module at [URL/route] using the qa-module-analysis skill"
|
|
33
30
|
|
|
34
31
|
# Generate spec, plan, cases, and automation
|
|
35
32
|
"For submodule {name}, generate all 6 spec files following the templates in qa/00-standards/"
|
|
@@ -37,53 +34,44 @@ This framework is designed so that an IDE agent (such as GitHub Copilot in VS Co
|
|
|
37
34
|
"Write Playwright E2E tests for TC-{ID} through TC-{ID}"
|
|
38
35
|
|
|
39
36
|
# Integrate results with Azure DevOps
|
|
40
|
-
"
|
|
37
|
+
"Load the qa-ado-integration skill and sync results with ADO"
|
|
41
38
|
|
|
42
39
|
# Maintenance
|
|
43
|
-
"Module {name} was updated.
|
|
40
|
+
"Module {name} was updated. Load the qa-maintenance skill and update the affected artifacts"
|
|
44
41
|
```
|
|
45
42
|
|
|
46
43
|
---
|
|
47
44
|
|
|
48
45
|
## Setting Up Agent Instructions in VS Code
|
|
49
46
|
|
|
50
|
-
### Option A —
|
|
51
|
-
|
|
52
|
-
Simply paste the relevant instruction file path into the Copilot chat:
|
|
53
|
-
|
|
54
|
-
```
|
|
55
|
-
Read qa/00-guides/AGENT-INSTRUCTIONS-MODULE-ANALYSIS.md and then analyze the module at {URL}
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
### Option B — Workspace instructions file (recommended)
|
|
47
|
+
### Option A — Workspace instructions file (recommended)
|
|
59
48
|
|
|
60
|
-
|
|
49
|
+
`init` generates `.github/copilot-instructions.md` automatically. It contains the full
|
|
50
|
+
pipeline sequencer with 11 agent behavior rules, the QA pipeline table, and ADO detection.
|
|
51
|
+
No manual setup needed.
|
|
61
52
|
|
|
62
|
-
|
|
63
|
-
# QA Framework Instructions
|
|
53
|
+
To regenerate after an upgrade:
|
|
64
54
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
55
|
+
```bash
|
|
56
|
+
npx qa-framework upgrade
|
|
57
|
+
```
|
|
68
58
|
|
|
69
|
-
|
|
70
|
-
2. Always save artifacts in the correct `qa/` subfolder per `qa/QA-STRUCTURE-GUIDE.md`
|
|
71
|
-
3. Never hardcode credentials — always use env vars and `<PLACEHOLDER>` in documentation
|
|
72
|
-
4. follow the naming conventions in `qa/00-standards/naming-conventions.md`
|
|
59
|
+
### Option B — Reference skills directly in the agent conversation
|
|
73
60
|
|
|
74
|
-
|
|
61
|
+
Paste a skill path into the Copilot chat:
|
|
75
62
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
- Test cases: `qa/00-guides/AGENT-INSTRUCTIONS-TEST-CASES.md`
|
|
79
|
-
- Automation: `qa/00-guides/AGENT-INSTRUCTIONS-AUTOMATION.md`
|
|
80
|
-
- ADO integration: `qa/00-guides/AGENT-INSTRUCTIONS-ADO-INTEGRATION.md`
|
|
63
|
+
```
|
|
64
|
+
Read .github/skills/qa-module-analysis/SKILL.md and analyze the module at {URL}
|
|
81
65
|
```
|
|
82
66
|
|
|
83
67
|
### Option C — Individual task prompts
|
|
84
68
|
|
|
85
|
-
For each major QA task,
|
|
86
|
-
|
|
69
|
+
For each major QA task, ask the agent to load the relevant skill:
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
Load the qa-automation skill (.github/skills/qa-automation/SKILL.md) and implement
|
|
73
|
+
tests for the submodule {name}
|
|
74
|
+
```
|
|
87
75
|
|
|
88
76
|
---
|
|
89
77
|
|
|
@@ -92,11 +80,11 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
92
80
|
### Session 1: Initial Module Analysis
|
|
93
81
|
|
|
94
82
|
```
|
|
95
|
-
1. Agent reads qa/
|
|
96
|
-
2. Agent
|
|
97
|
-
3. Agent
|
|
98
|
-
4. Agent
|
|
99
|
-
5. Agent
|
|
83
|
+
1. Agent reads qa/AGENT-NEXT-STEPS.md (auto, on conversation start)
|
|
84
|
+
2. Agent loads .github/skills/qa-module-analysis/SKILL.md
|
|
85
|
+
3. Agent navigates to QA_BASE_URL using browser automation
|
|
86
|
+
4. Agent explores the module: screenshots, routes, form fields, APIs
|
|
87
|
+
5. Agent produces qa/01-specifications/module-{name}/submodule-{x}/ (6 files each)
|
|
100
88
|
6. Human reviews specs and approves before proceeding
|
|
101
89
|
```
|
|
102
90
|
|
|
@@ -104,7 +92,7 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
104
92
|
|
|
105
93
|
```
|
|
106
94
|
1. Agent reads approved specs from 01-specifications/
|
|
107
|
-
2. Agent
|
|
95
|
+
2. Agent loads .github/skills/qa-test-plan/SKILL.md
|
|
108
96
|
3. Agent selects TCs for automation (P0/P1) and manual (P2/P3)
|
|
109
97
|
4. Agent writes qa/02-test-plans/{module}-automated-test-plan.md
|
|
110
98
|
5. Human reviews priorities
|
|
@@ -114,9 +102,9 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
114
102
|
|
|
115
103
|
```
|
|
116
104
|
1. Agent reads approved test plan
|
|
117
|
-
2. Agent
|
|
105
|
+
2. Agent loads .github/skills/qa-automation/SKILL.md
|
|
118
106
|
3. Agent writes Playwright spec files in qa/07-automation/e2e/tests/{module}/
|
|
119
|
-
4. Agent runs tests
|
|
107
|
+
4. Agent runs tests from qa/07-automation/e2e/ (cd e2e && npx playwright test)
|
|
120
108
|
5. Agent writes qa/05-test-execution/automated/{date}.md
|
|
121
109
|
6. Human reviews automation coverage
|
|
122
110
|
```
|
|
@@ -124,7 +112,7 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
124
112
|
### Session 4: ADO Sync (if enabled)
|
|
125
113
|
|
|
126
114
|
```
|
|
127
|
-
1. Agent
|
|
115
|
+
1. Agent loads .github/skills/qa-ado-integration/SKILL.md
|
|
128
116
|
2. Agent runs inject-ado-ids.ps1 to prefix test titles
|
|
129
117
|
3. Agent verifies runner output appears in ADO Test Plan
|
|
130
118
|
4. Agent updates qa/08-azure-integration/ado-ids-mapping-{project}.json
|
|
@@ -136,23 +124,23 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
136
124
|
|
|
137
125
|
### DO
|
|
138
126
|
|
|
139
|
-
-
|
|
140
|
-
-
|
|
141
|
-
-
|
|
142
|
-
-
|
|
143
|
-
-
|
|
144
|
-
-
|
|
145
|
-
-
|
|
127
|
+
- Read `qa/AGENT-NEXT-STEPS.md` at the start of each conversation (auto-enforced by copilot-instructions.md)
|
|
128
|
+
- Load the relevant skill from `.github/skills/` before starting any QA task
|
|
129
|
+
- Save all artifacts in the exact path specified by `qa/QA-STRUCTURE-GUIDE.md`
|
|
130
|
+
- Use TC-ID, RN-ID, FL-ID naming consistently
|
|
131
|
+
- Replace credentials with `<PLACEHOLDER>` in any markdown output
|
|
132
|
+
- Reference the DEF-ID or ADO WI ID in every `test.skip()` call
|
|
133
|
+
- Validate test stability with >= 2 passes using different `EXEC_IDX` values
|
|
134
|
+
- Run Playwright from `qa/07-automation/e2e/` (`cd qa/07-automation/e2e && npx playwright test`)
|
|
146
135
|
|
|
147
136
|
### DON'T
|
|
148
137
|
|
|
149
|
-
-
|
|
150
|
-
-
|
|
151
|
-
-
|
|
152
|
-
-
|
|
153
|
-
-
|
|
154
|
-
-
|
|
155
|
-
- ❌ Remove a `test.skip()` without verifying the underlying defect is resolved
|
|
138
|
+
- Invent test cases without first observing the QA environment
|
|
139
|
+
- Hardcode credentials, DNIs, or personal data in any file
|
|
140
|
+
- Save screenshots or debug output to the project root
|
|
141
|
+
- Modify files in `00-standards/` (those are project-customized after install)
|
|
142
|
+
- Create test cases for features marked `BLOCKED-PERMISSIONS` without noting the blocker
|
|
143
|
+
- Remove a `test.skip()` without verifying the underlying defect is resolved
|
|
156
144
|
|
|
157
145
|
---
|
|
158
146
|
|
|
@@ -182,21 +170,21 @@ npx qa-framework validate
|
|
|
182
170
|
```
|
|
183
171
|
|
|
184
172
|
This performs:
|
|
185
|
-
- Structure check (all
|
|
173
|
+
- Structure check (all required folders exist)
|
|
186
174
|
- Config check (required fields populated)
|
|
187
175
|
- Credential scan (no hardcoded passwords in markdown)
|
|
188
|
-
- Naming check (
|
|
176
|
+
- Naming check (TC IDs follow `TC-{MODULE}-{SUBMODULE}-{NNN}` pattern)
|
|
189
177
|
- Coverage check (all spec TCs have matching test file references)
|
|
190
178
|
|
|
191
179
|
---
|
|
192
180
|
|
|
193
181
|
## Continuing from a Previous Session
|
|
194
182
|
|
|
195
|
-
|
|
183
|
+
The agent reads `qa/AGENT-NEXT-STEPS.md` automatically at the start of every conversation
|
|
184
|
+
(enforced by `.github/copilot-instructions.md` rule 0). For additional context:
|
|
196
185
|
|
|
197
186
|
```
|
|
198
|
-
"Read qa/
|
|
199
|
-
and what the next steps are."
|
|
187
|
+
"Read qa/README.md and qa/memory/INDEX.md, then tell me where we left off."
|
|
200
188
|
```
|
|
201
189
|
|
|
202
190
|
The session summary + README combination gives the agent sufficient context to resume
|
|
@@ -17,7 +17,7 @@ Peer dependency: `@playwright/test >= 1.40.0`
|
|
|
17
17
|
## Configuration
|
|
18
18
|
|
|
19
19
|
Copy `templates/automation-scaffold/playwright.config.ts` into your project's
|
|
20
|
-
`qa/07-automation/` directory and replace all `{{PLACEHOLDER}}` values.
|
|
20
|
+
`qa/07-automation/e2e/` directory and replace all `{{PLACEHOLDER}}` values.
|
|
21
21
|
|
|
22
22
|
## Key conventions
|
|
23
23
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@keber/qa-framework",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.2",
|
|
4
4
|
"description": "Reusable spec-driven QA framework for IDE-agent-assisted automated testing. Installable as an npm package. Provides structure, templates, agent instructions, and optional integrations for Playwright and Azure DevOps.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"qa",
|
package/scripts/init.js
CHANGED
|
@@ -85,7 +85,9 @@ const topLevelFolders = [
|
|
|
85
85
|
'05-test-execution',
|
|
86
86
|
'06-defects/open',
|
|
87
87
|
'06-defects/resolved',
|
|
88
|
-
'07-automation/e2e',
|
|
88
|
+
'07-automation/e2e/tests',
|
|
89
|
+
'07-automation/e2e/tests/helpers/debug',
|
|
90
|
+
'07-automation/e2e/tests/seeds',
|
|
89
91
|
'08-azure-integration',
|
|
90
92
|
'memory',
|
|
91
93
|
];
|
|
@@ -185,8 +187,8 @@ for (const mod of modules) {
|
|
|
185
187
|
}
|
|
186
188
|
}
|
|
187
189
|
|
|
188
|
-
// Create automation
|
|
189
|
-
const e2eDir = path.join(qaRoot, '07-automation', 'e2e', moduleKey);
|
|
190
|
+
// Create automation spec stub in e2e/tests/{module}/
|
|
191
|
+
const e2eDir = path.join(qaRoot, '07-automation', 'e2e', 'tests', moduleKey);
|
|
190
192
|
fs.mkdirSync(e2eDir, { recursive: true });
|
|
191
193
|
const specTs = path.join(e2eDir, `${subKey}.spec.ts`);
|
|
192
194
|
if (!fs.existsSync(specTs)) {
|
|
@@ -196,26 +198,34 @@ for (const mod of modules) {
|
|
|
196
198
|
}
|
|
197
199
|
}
|
|
198
200
|
|
|
199
|
-
// --- Automation scaffold ---
|
|
200
|
-
const automationDir
|
|
201
|
-
const
|
|
201
|
+
// --- Automation scaffold (lives in 07-automation/e2e/) ---
|
|
202
|
+
const automationDir = path.join(qaRoot, '07-automation');
|
|
203
|
+
const e2eScaffoldDir = path.join(automationDir, 'e2e');
|
|
204
|
+
const scaffoldSrc = path.resolve(__dirname, '..', 'templates', 'automation-scaffold');
|
|
205
|
+
fs.mkdirSync(e2eScaffoldDir, { recursive: true });
|
|
202
206
|
for (const file of ['playwright.config.ts', 'global-setup.ts', '.env.example', 'package.json']) {
|
|
203
|
-
const dest = path.join(
|
|
207
|
+
const dest = path.join(e2eScaffoldDir, file);
|
|
204
208
|
if (!fs.existsSync(dest)) {
|
|
205
209
|
fs.copyFileSync(path.join(scaffoldSrc, file), dest);
|
|
206
210
|
console.log(` [created] ${path.relative(cwd, dest)}`);
|
|
207
211
|
}
|
|
208
212
|
}
|
|
209
|
-
const fixturesDir = path.join(
|
|
213
|
+
const fixturesDir = path.join(e2eScaffoldDir, 'fixtures');
|
|
210
214
|
fs.mkdirSync(fixturesDir, { recursive: true });
|
|
211
215
|
for (const file of ['auth.ts', 'test-helpers.ts']) {
|
|
212
216
|
const dest = path.join(fixturesDir, file);
|
|
213
217
|
if (!fs.existsSync(dest)) {
|
|
214
218
|
fs.copyFileSync(path.join(scaffoldSrc, 'fixtures', file), dest);
|
|
215
|
-
console.log(` [created] ${path.relative(cwd
|
|
219
|
+
console.log(` [created] ${path.relative(cwd, dest)}`);
|
|
216
220
|
}
|
|
217
221
|
}
|
|
218
222
|
|
|
223
|
+
// --- Integration + Load test placeholders ---
|
|
224
|
+
writeIfMissing(path.join(automationDir, 'integration', 'README.md'),
|
|
225
|
+
`# Integration Tests\n\n> Placeholder for API/integration tests (k6, JMeter, Azure Load Testing, etc.).\n> Each tool gets its own subdirectory.\n`);
|
|
226
|
+
writeIfMissing(path.join(automationDir, 'load', 'README.md'),
|
|
227
|
+
`# Load Tests\n\n> Placeholder for load and performance tests.\n> Each tool gets its own subdirectory (e.g. k6/, jmeter/).\n`);
|
|
228
|
+
|
|
219
229
|
// --- ADO integration placeholder ---
|
|
220
230
|
const adoDir = path.join(qaRoot, '08-azure-integration');
|
|
221
231
|
writeIfMissing(path.join(adoDir, 'README.md'), `# ADO Integration\n\nSee keber/qa-framework integrations/ado-powershell/ for setup instructions.\n`);
|
|
@@ -344,12 +354,12 @@ npm install @keber/ado-qa
|
|
|
344
354
|
**Playwright automation:**
|
|
345
355
|
\`\`\`bash
|
|
346
356
|
npm install @playwright/test
|
|
347
|
-
cd qa/07-automation && npm install && npx playwright install chromium
|
|
357
|
+
cd qa/07-automation/e2e && npm install && npx playwright install chromium
|
|
348
358
|
\`\`\`
|
|
349
359
|
|
|
350
360
|
### 3. Credentials
|
|
351
|
-
- Copy \`qa/07-automation/.env.example\` → \`qa/07-automation/.env\` and fill in credentials
|
|
352
|
-
- Add \`qa/07-automation/.env\` and \`qa/07-automation/.auth/\` to \`.gitignore\`
|
|
361
|
+
- Copy \`qa/07-automation/e2e/.env.example\` → \`qa/07-automation/e2e/.env\` and fill in credentials
|
|
362
|
+
- Add \`qa/07-automation/e2e/.env\` and \`qa/07-automation/e2e/.auth/\` to \`.gitignore\`
|
|
353
363
|
`;
|
|
354
364
|
const nextStepsPath = path.join(qaRoot, 'AGENT-NEXT-STEPS.md');
|
|
355
365
|
fs.writeFileSync(nextStepsPath, nextStepsContent, 'utf8');
|