@keber/qa-framework 1.5.1 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -6
- package/README.md +150 -76
- package/docs/architecture.md +49 -44
- package/docs/folder-structure-guide.md +27 -23
- package/docs/installation.md +53 -43
- package/docs/usage-with-agent.md +52 -64
- package/integrations/ado-powershell/pipelines/azure-pipeline-qa.yml +1 -1
- package/integrations/playwright/README.md +1 -1
- package/package.json +1 -1
- package/scripts/init.js +20 -12
- package/scripts/upgrade.js +100 -11
- package/skills/qa-ado-integration/SKILL.md +2 -2
- package/skills/qa-automation/SKILL.md +6 -6
- package/skills/qa-maintenance/SKILL.md +2 -2
- package/templates/automation-scaffold/playwright.config.ts +22 -11
- package/agent-instructions/00-module-analysis.md +0 -266
- package/agent-instructions/01-spec-generation.md +0 -281
- package/agent-instructions/02-test-plan-generation.md +0 -205
- package/agent-instructions/03-test-case-generation.md +0 -150
- package/agent-instructions/04-automation-generation.md +0 -343
- package/agent-instructions/04b-test-stabilization.md +0 -309
- package/agent-instructions/05-ado-integration.md +0 -247
- package/agent-instructions/06-maintenance.md +0 -135
- package/docs/comparison-matrix.md +0 -131
- package/docs/generalization-decisions.md +0 -203
|
@@ -197,7 +197,7 @@ Most projects will have sparse content here — the 05-test-scenarios.md files i
|
|
|
197
197
|
```
|
|
198
198
|
06-defects/
|
|
199
199
|
├── README.md
|
|
200
|
-
├──
|
|
200
|
+
├── open/
|
|
201
201
|
│ └── DEF-{NNN}-{slug}.md
|
|
202
202
|
└── resolved/
|
|
203
203
|
└── DEF-{NNN}-{slug}.md
|
|
@@ -218,32 +218,36 @@ See `docs/generalization-decisions.md §7` for the full evaluation of this folde
|
|
|
218
218
|
|
|
219
219
|
```
|
|
220
220
|
07-automation/
|
|
221
|
-
├── README.md
|
|
222
|
-
├── e2e/
|
|
223
|
-
│ ├──
|
|
224
|
-
│ ├──
|
|
225
|
-
│ ├── global-setup.ts
|
|
226
|
-
│ ├── .env
|
|
227
|
-
│ ├── .env.example
|
|
228
|
-
│ ├── .auth/
|
|
229
|
-
│ ├── fixtures/
|
|
230
|
-
│ │ ├── auth.ts
|
|
231
|
-
│ │ └── test-helpers.ts
|
|
232
|
-
│ ├── page-objects/
|
|
233
|
-
│ ├── tests/
|
|
234
|
-
│ │ ├── {module}/
|
|
235
|
-
│ │ ├── helpers/debug/
|
|
236
|
-
│ │ └── seeds/
|
|
237
|
-
│
|
|
238
|
-
|
|
239
|
-
└──
|
|
221
|
+
├── README.md <- How to run, where reports go, worker config
|
|
222
|
+
├── e2e/ <- Playwright E2E project (self-contained)
|
|
223
|
+
│ ├── playwright.config.ts <- Test runner config (installed by init)
|
|
224
|
+
│ ├── package.json <- Playwright dependencies (installed by init)
|
|
225
|
+
│ ├── global-setup.ts <- Pre-suite login + storageState save (installed by init)
|
|
226
|
+
│ ├── .env <- Local credentials (NEVER commit)
|
|
227
|
+
│ ├── .env.example <- Template (always commit, installed by init)
|
|
228
|
+
│ ├── .auth/ <- storageState files (gitignored)
|
|
229
|
+
│ ├── fixtures/ <- Installed by init
|
|
230
|
+
│ │ ├── auth.ts <- loginAs(page, role) helper
|
|
231
|
+
│ │ └── test-helpers.ts <- EXEC_IDX, date helpers, etc.
|
|
232
|
+
│ ├── page-objects/ <- Page Object Model classes (project-maintained)
|
|
233
|
+
│ ├── tests/ <- Spec files, organized by module
|
|
234
|
+
│ │ ├── {module}/ <- Stub .spec.ts files generated by init per submodule
|
|
235
|
+
│ │ ├── helpers/debug/ <- Diagnostic scripts (not part of suite)
|
|
236
|
+
│ │ └── seeds/ <- Data seeding specs (excluded from runs)
|
|
237
|
+
│ └── diagnosis/ <- Screenshots and debug output (gitignored)
|
|
238
|
+
├── integration/ <- API-level tests (k6, JMeter, Azure Load Testing)
|
|
239
|
+
│ └── README.md <- Integration test setup and run instructions
|
|
240
|
+
└── load/ <- Load / performance tests (future)
|
|
241
|
+
└── README.md <- Load test setup and run instructions
|
|
240
242
|
```
|
|
241
243
|
|
|
242
244
|
**Important rules**:
|
|
243
|
-
-
|
|
245
|
+
- Run Playwright from `qa/07-automation/e2e/` (where `playwright.config.ts` lives): `cd qa/07-automation/e2e && npx playwright test`
|
|
244
246
|
- Seeds are excluded from normal test runs via `testIgnore` in `playwright.config.ts`
|
|
245
|
-
- Debug artifacts go into `diagnosis/`
|
|
246
|
-
- `.auth/`
|
|
247
|
+
- Debug artifacts go into `e2e/diagnosis/` -- never into the repo root
|
|
248
|
+
- `.auth/` and `.env` are always gitignored
|
|
249
|
+
- Files installed by `init` must not be deleted -- they are the project's automation bootstrap
|
|
250
|
+
- `integration/` and `load/` are independent projects with their own toolchains (k6, JMeter, etc.) -- they do not depend on `e2e/`
|
|
247
251
|
|
|
248
252
|
---
|
|
249
253
|
|
package/docs/installation.md
CHANGED
|
@@ -16,56 +16,48 @@
|
|
|
16
16
|
## Option A — npm install (recommended)
|
|
17
17
|
|
|
18
18
|
```bash
|
|
19
|
-
|
|
20
|
-
npm install --save-dev keber/qa-framework
|
|
21
|
-
|
|
22
|
-
# Run the init command
|
|
23
|
-
npx qa-framework init
|
|
19
|
+
npm install --save-dev @keber/qa-framework
|
|
24
20
|
```
|
|
25
21
|
|
|
26
|
-
The `init`
|
|
22
|
+
The `postinstall` script runs `init --skip-if-exists` automatically. On first install it
|
|
23
|
+
creates the full `qa/` structure without prompts, reading `qa-framework.config.json` if
|
|
24
|
+
present or bootstrapping defaults.
|
|
27
25
|
|
|
28
|
-
|
|
29
|
-
2. **QA base URL** (e.g., `https://myproject-qa.example.com`)
|
|
30
|
-
3. **Login path** (e.g., `/login` or `/Seguridad/Login`)
|
|
31
|
-
4. **Number of test user roles** (1 = single role, 2+ = multi-role)
|
|
32
|
-
5. **Enable Playwright integration?** (y/n)
|
|
33
|
-
6. **Enable Azure DevOps integration?** (y/n)
|
|
26
|
+
After install, read `qa/AGENT-NEXT-STEPS.md` for the exact follow-up steps.
|
|
34
27
|
|
|
35
|
-
|
|
28
|
+
`init` creates:
|
|
36
29
|
|
|
37
30
|
```
|
|
38
31
|
qa/
|
|
39
|
-
├── README.md
|
|
40
|
-
├──
|
|
41
|
-
├──
|
|
42
|
-
├──
|
|
43
|
-
├── 00-standards/
|
|
44
|
-
├── 01-specifications/
|
|
45
|
-
├── 02-test-plans/
|
|
46
|
-
├── 03-test-cases/
|
|
47
|
-
├── 04-test-data/
|
|
48
|
-
├── 05-test-execution/
|
|
49
|
-
├── 06-defects/
|
|
50
|
-
├── 07-automation/
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
└── auth.ts
|
|
32
|
+
├── README.md <- Prefilled with your project name
|
|
33
|
+
├── AGENT-NEXT-STEPS.md <- Follow-up checklist for the agent
|
|
34
|
+
├── QA-STRUCTURE-GUIDE.md <- Full structure guide (framework-owned)
|
|
35
|
+
├── qa-framework.config.json <- Your project config
|
|
36
|
+
├── 00-standards/ <- Templates and naming conventions
|
|
37
|
+
├── 01-specifications/
|
|
38
|
+
├── 02-test-plans/
|
|
39
|
+
├── 03-test-cases/
|
|
40
|
+
├── 04-test-data/
|
|
41
|
+
├── 05-test-execution/
|
|
42
|
+
├── 06-defects/open|resolved/
|
|
43
|
+
├── 07-automation/
|
|
44
|
+
│ ├── e2e/ <- Playwright project (self-contained)
|
|
45
|
+
│ │ ├── playwright.config.ts
|
|
46
|
+
│ │ ├── global-setup.ts
|
|
47
|
+
│ │ ├── .env.example
|
|
48
|
+
│ │ └── fixtures/auth.ts + test-helpers.ts
|
|
49
|
+
│ ├── integration/README.md <- Placeholder for k6, JMeter, etc.
|
|
50
|
+
│ └── load/README.md <- Placeholder for load tests
|
|
51
|
+
└── 08-azure-integration/README.md
|
|
52
|
+
└── memory/INDEX.md
|
|
53
|
+
|
|
54
|
+
.github/copilot-instructions.md <- Generated agent rules
|
|
55
|
+
.github/skills/ <- 8 agent skill sets (SKILL.md + references/)
|
|
64
56
|
```
|
|
65
57
|
|
|
66
58
|
---
|
|
67
59
|
|
|
68
|
-
## Option B — Clone or copy (
|
|
60
|
+
## Option B — Clone or copy (no npm registry)
|
|
69
61
|
|
|
70
62
|
```bash
|
|
71
63
|
# From your project root
|
|
@@ -82,15 +74,33 @@ node tools/qa-framework/scripts/cli.js init
|
|
|
82
74
|
If you prefer to control exactly what gets created:
|
|
83
75
|
|
|
84
76
|
```bash
|
|
85
|
-
# Clone the package
|
|
86
77
|
git clone https://github.com/your-org/qa-framework.git tools/qa-framework
|
|
87
78
|
|
|
88
|
-
# Copy
|
|
79
|
+
# Copy templates
|
|
89
80
|
cp -r tools/qa-framework/templates/specification qa/01-specifications/shared/
|
|
90
|
-
cp -r tools/qa-framework/agent-instructions qa/00-guides/
|
|
91
81
|
cp -r tools/qa-framework/templates/automation-scaffold qa/07-automation/e2e/
|
|
92
82
|
|
|
93
|
-
# Then
|
|
83
|
+
# Then fill in qa-framework.config.json manually
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## Upgrading
|
|
89
|
+
|
|
90
|
+
After `npm update @keber/qa-framework`:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
npx qa-framework upgrade
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
This overwrites only framework-owned files (`.github/skills/`, `copilot-instructions.md`,
|
|
97
|
+
`QA-STRUCTURE-GUIDE.md`) and migrates the `07-automation/` structure if upgrading from
|
|
98
|
+
v1.5.x. Your specs, tests, and memory are never touched.
|
|
99
|
+
|
|
100
|
+
Use `--dry-run` to preview without writing:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
npx qa-framework upgrade --dry-run
|
|
94
104
|
```
|
|
95
105
|
|
|
96
106
|
---
|
package/docs/usage-with-agent.md
CHANGED
|
@@ -20,16 +20,13 @@ This framework is designed so that an IDE agent (such as GitHub Copilot in VS Co
|
|
|
20
20
|
|
|
21
21
|
```
|
|
22
22
|
# Install the framework
|
|
23
|
-
"Run
|
|
23
|
+
"Run npm install @keber/qa-framework for this project"
|
|
24
24
|
|
|
25
|
-
#
|
|
26
|
-
"Read
|
|
27
|
-
|
|
28
|
-
# Generate the QA structure
|
|
29
|
-
"Generate the full qa/ directory structure using the framework config"
|
|
25
|
+
# Let the agent self-configure
|
|
26
|
+
"Read qa/AGENT-NEXT-STEPS.md and complete the setup steps"
|
|
30
27
|
|
|
31
28
|
# Analyze a module
|
|
32
|
-
"Analyze the module at [URL/route]
|
|
29
|
+
"Analyze the module at [URL/route] using the qa-module-analysis skill"
|
|
33
30
|
|
|
34
31
|
# Generate spec, plan, cases, and automation
|
|
35
32
|
"For submodule {name}, generate all 6 spec files following the templates in qa/00-standards/"
|
|
@@ -37,53 +34,44 @@ This framework is designed so that an IDE agent (such as GitHub Copilot in VS Co
|
|
|
37
34
|
"Write Playwright E2E tests for TC-{ID} through TC-{ID}"
|
|
38
35
|
|
|
39
36
|
# Integrate results with Azure DevOps
|
|
40
|
-
"
|
|
37
|
+
"Load the qa-ado-integration skill and sync results with ADO"
|
|
41
38
|
|
|
42
39
|
# Maintenance
|
|
43
|
-
"Module {name} was updated.
|
|
40
|
+
"Module {name} was updated. Load the qa-maintenance skill and update the affected artifacts"
|
|
44
41
|
```
|
|
45
42
|
|
|
46
43
|
---
|
|
47
44
|
|
|
48
45
|
## Setting Up Agent Instructions in VS Code
|
|
49
46
|
|
|
50
|
-
### Option A —
|
|
51
|
-
|
|
52
|
-
Simply paste the relevant instruction file path into the Copilot chat:
|
|
53
|
-
|
|
54
|
-
```
|
|
55
|
-
Read qa/00-guides/AGENT-INSTRUCTIONS-MODULE-ANALYSIS.md and then analyze the module at {URL}
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
### Option B — Workspace instructions file (recommended)
|
|
47
|
+
### Option A — Workspace instructions file (recommended)
|
|
59
48
|
|
|
60
|
-
|
|
49
|
+
`init` generates `.github/copilot-instructions.md` automatically. It contains the full
|
|
50
|
+
pipeline sequencer with 11 agent behavior rules, the QA pipeline table, and ADO detection.
|
|
51
|
+
No manual setup needed.
|
|
61
52
|
|
|
62
|
-
|
|
63
|
-
# QA Framework Instructions
|
|
53
|
+
To regenerate after an upgrade:
|
|
64
54
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
55
|
+
```bash
|
|
56
|
+
npx qa-framework upgrade
|
|
57
|
+
```
|
|
68
58
|
|
|
69
|
-
|
|
70
|
-
2. Always save artifacts in the correct `qa/` subfolder per `qa/QA-STRUCTURE-GUIDE.md`
|
|
71
|
-
3. Never hardcode credentials — always use env vars and `<PLACEHOLDER>` in documentation
|
|
72
|
-
4. follow the naming conventions in `qa/00-standards/naming-conventions.md`
|
|
59
|
+
### Option B — Reference skills directly in the agent conversation
|
|
73
60
|
|
|
74
|
-
|
|
61
|
+
Paste a skill path into the Copilot chat:
|
|
75
62
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
- Test cases: `qa/00-guides/AGENT-INSTRUCTIONS-TEST-CASES.md`
|
|
79
|
-
- Automation: `qa/00-guides/AGENT-INSTRUCTIONS-AUTOMATION.md`
|
|
80
|
-
- ADO integration: `qa/00-guides/AGENT-INSTRUCTIONS-ADO-INTEGRATION.md`
|
|
63
|
+
```
|
|
64
|
+
Read .github/skills/qa-module-analysis/SKILL.md and analyze the module at {URL}
|
|
81
65
|
```
|
|
82
66
|
|
|
83
67
|
### Option C — Individual task prompts
|
|
84
68
|
|
|
85
|
-
For each major QA task,
|
|
86
|
-
|
|
69
|
+
For each major QA task, ask the agent to load the relevant skill:
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
Load the qa-automation skill (.github/skills/qa-automation/SKILL.md) and implement
|
|
73
|
+
tests for the submodule {name}
|
|
74
|
+
```
|
|
87
75
|
|
|
88
76
|
---
|
|
89
77
|
|
|
@@ -92,11 +80,11 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
92
80
|
### Session 1: Initial Module Analysis
|
|
93
81
|
|
|
94
82
|
```
|
|
95
|
-
1. Agent reads qa/
|
|
96
|
-
2. Agent
|
|
97
|
-
3. Agent
|
|
98
|
-
4. Agent
|
|
99
|
-
5. Agent
|
|
83
|
+
1. Agent reads qa/AGENT-NEXT-STEPS.md (auto, on conversation start)
|
|
84
|
+
2. Agent loads .github/skills/qa-module-analysis/SKILL.md
|
|
85
|
+
3. Agent navigates to QA_BASE_URL using browser automation
|
|
86
|
+
4. Agent explores the module: screenshots, routes, form fields, APIs
|
|
87
|
+
5. Agent produces qa/01-specifications/module-{name}/submodule-{x}/ (6 files each)
|
|
100
88
|
6. Human reviews specs and approves before proceeding
|
|
101
89
|
```
|
|
102
90
|
|
|
@@ -104,7 +92,7 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
104
92
|
|
|
105
93
|
```
|
|
106
94
|
1. Agent reads approved specs from 01-specifications/
|
|
107
|
-
2. Agent
|
|
95
|
+
2. Agent loads .github/skills/qa-test-plan/SKILL.md
|
|
108
96
|
3. Agent selects TCs for automation (P0/P1) and manual (P2/P3)
|
|
109
97
|
4. Agent writes qa/02-test-plans/{module}-automated-test-plan.md
|
|
110
98
|
5. Human reviews priorities
|
|
@@ -114,9 +102,9 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
114
102
|
|
|
115
103
|
```
|
|
116
104
|
1. Agent reads approved test plan
|
|
117
|
-
2. Agent
|
|
105
|
+
2. Agent loads .github/skills/qa-automation/SKILL.md
|
|
118
106
|
3. Agent writes Playwright spec files in qa/07-automation/e2e/tests/{module}/
|
|
119
|
-
4. Agent runs tests
|
|
107
|
+
4. Agent runs tests from qa/07-automation/e2e/ (cd e2e && npx playwright test)
|
|
120
108
|
5. Agent writes qa/05-test-execution/automated/{date}.md
|
|
121
109
|
6. Human reviews automation coverage
|
|
122
110
|
```
|
|
@@ -124,7 +112,7 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
124
112
|
### Session 4: ADO Sync (if enabled)
|
|
125
113
|
|
|
126
114
|
```
|
|
127
|
-
1. Agent
|
|
115
|
+
1. Agent loads .github/skills/qa-ado-integration/SKILL.md
|
|
128
116
|
2. Agent runs inject-ado-ids.ps1 to prefix test titles
|
|
129
117
|
3. Agent verifies runner output appears in ADO Test Plan
|
|
130
118
|
4. Agent updates qa/08-azure-integration/ado-ids-mapping-{project}.json
|
|
@@ -136,23 +124,23 @@ as the starting prompt. These are designed to be copied and pasted directly.
|
|
|
136
124
|
|
|
137
125
|
### DO
|
|
138
126
|
|
|
139
|
-
-
|
|
140
|
-
-
|
|
141
|
-
-
|
|
142
|
-
-
|
|
143
|
-
-
|
|
144
|
-
-
|
|
145
|
-
-
|
|
127
|
+
- Read `qa/AGENT-NEXT-STEPS.md` at the start of each conversation (auto-enforced by copilot-instructions.md)
|
|
128
|
+
- Load the relevant skill from `.github/skills/` before starting any QA task
|
|
129
|
+
- Save all artifacts in the exact path specified by `qa/QA-STRUCTURE-GUIDE.md`
|
|
130
|
+
- Use TC-ID, RN-ID, FL-ID naming consistently
|
|
131
|
+
- Replace credentials with `<PLACEHOLDER>` in any markdown output
|
|
132
|
+
- Reference the DEF-ID or ADO WI ID in every `test.skip()` call
|
|
133
|
+
- Validate test stability with >= 2 passes using different `EXEC_IDX` values
|
|
134
|
+
- Run Playwright from `qa/07-automation/e2e/` (`cd qa/07-automation/e2e && npx playwright test`)
|
|
146
135
|
|
|
147
136
|
### DON'T
|
|
148
137
|
|
|
149
|
-
-
|
|
150
|
-
-
|
|
151
|
-
-
|
|
152
|
-
-
|
|
153
|
-
-
|
|
154
|
-
-
|
|
155
|
-
- ❌ Remove a `test.skip()` without verifying the underlying defect is resolved
|
|
138
|
+
- Invent test cases without first observing the QA environment
|
|
139
|
+
- Hardcode credentials, DNIs, or personal data in any file
|
|
140
|
+
- Save screenshots or debug output to the project root
|
|
141
|
+
- Modify files in `00-standards/` (those are project-customized after install)
|
|
142
|
+
- Create test cases for features marked `BLOCKED-PERMISSIONS` without noting the blocker
|
|
143
|
+
- Remove a `test.skip()` without verifying the underlying defect is resolved
|
|
156
144
|
|
|
157
145
|
---
|
|
158
146
|
|
|
@@ -182,21 +170,21 @@ npx qa-framework validate
|
|
|
182
170
|
```
|
|
183
171
|
|
|
184
172
|
This performs:
|
|
185
|
-
- Structure check (all
|
|
173
|
+
- Structure check (all required folders exist)
|
|
186
174
|
- Config check (required fields populated)
|
|
187
175
|
- Credential scan (no hardcoded passwords in markdown)
|
|
188
|
-
- Naming check (
|
|
176
|
+
- Naming check (TC IDs follow `TC-{MODULE}-{SUBMODULE}-{NNN}` pattern)
|
|
189
177
|
- Coverage check (all spec TCs have matching test file references)
|
|
190
178
|
|
|
191
179
|
---
|
|
192
180
|
|
|
193
181
|
## Continuing from a Previous Session
|
|
194
182
|
|
|
195
|
-
|
|
183
|
+
The agent reads `qa/AGENT-NEXT-STEPS.md` automatically at the start of every conversation
|
|
184
|
+
(enforced by `.github/copilot-instructions.md` rule 0). For additional context:
|
|
196
185
|
|
|
197
186
|
```
|
|
198
|
-
"Read qa/
|
|
199
|
-
and what the next steps are."
|
|
187
|
+
"Read qa/README.md and qa/memory/INDEX.md, then tell me where we left off."
|
|
200
188
|
```
|
|
201
189
|
|
|
202
190
|
The session summary + README combination gives the agent sufficient context to resume
|
|
@@ -17,7 +17,7 @@ Peer dependency: `@playwright/test >= 1.40.0`
|
|
|
17
17
|
## Configuration
|
|
18
18
|
|
|
19
19
|
Copy `templates/automation-scaffold/playwright.config.ts` into your project's
|
|
20
|
-
`qa/07-automation/` directory and replace all `{{PLACEHOLDER}}` values.
|
|
20
|
+
`qa/07-automation/e2e/` directory and replace all `{{PLACEHOLDER}}` values.
|
|
21
21
|
|
|
22
22
|
## Key conventions
|
|
23
23
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@keber/qa-framework",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.1",
|
|
4
4
|
"description": "Reusable spec-driven QA framework for IDE-agent-assisted automated testing. Installable as an npm package. Provides structure, templates, agent instructions, and optional integrations for Playwright and Azure DevOps.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"qa",
|
package/scripts/init.js
CHANGED
|
@@ -85,7 +85,7 @@ const topLevelFolders = [
|
|
|
85
85
|
'05-test-execution',
|
|
86
86
|
'06-defects/open',
|
|
87
87
|
'06-defects/resolved',
|
|
88
|
-
'07-automation/e2e',
|
|
88
|
+
'07-automation/e2e/tests',
|
|
89
89
|
'08-azure-integration',
|
|
90
90
|
'memory',
|
|
91
91
|
];
|
|
@@ -185,8 +185,8 @@ for (const mod of modules) {
|
|
|
185
185
|
}
|
|
186
186
|
}
|
|
187
187
|
|
|
188
|
-
// Create automation
|
|
189
|
-
const e2eDir = path.join(qaRoot, '07-automation', 'e2e', moduleKey);
|
|
188
|
+
// Create automation spec stub in e2e/tests/{module}/
|
|
189
|
+
const e2eDir = path.join(qaRoot, '07-automation', 'e2e', 'tests', moduleKey);
|
|
190
190
|
fs.mkdirSync(e2eDir, { recursive: true });
|
|
191
191
|
const specTs = path.join(e2eDir, `${subKey}.spec.ts`);
|
|
192
192
|
if (!fs.existsSync(specTs)) {
|
|
@@ -196,26 +196,34 @@ for (const mod of modules) {
|
|
|
196
196
|
}
|
|
197
197
|
}
|
|
198
198
|
|
|
199
|
-
// --- Automation scaffold ---
|
|
200
|
-
const automationDir
|
|
201
|
-
const
|
|
199
|
+
// --- Automation scaffold (lives in 07-automation/e2e/) ---
|
|
200
|
+
const automationDir = path.join(qaRoot, '07-automation');
|
|
201
|
+
const e2eScaffoldDir = path.join(automationDir, 'e2e');
|
|
202
|
+
const scaffoldSrc = path.resolve(__dirname, '..', 'templates', 'automation-scaffold');
|
|
203
|
+
fs.mkdirSync(e2eScaffoldDir, { recursive: true });
|
|
202
204
|
for (const file of ['playwright.config.ts', 'global-setup.ts', '.env.example', 'package.json']) {
|
|
203
|
-
const dest = path.join(
|
|
205
|
+
const dest = path.join(e2eScaffoldDir, file);
|
|
204
206
|
if (!fs.existsSync(dest)) {
|
|
205
207
|
fs.copyFileSync(path.join(scaffoldSrc, file), dest);
|
|
206
208
|
console.log(` [created] ${path.relative(cwd, dest)}`);
|
|
207
209
|
}
|
|
208
210
|
}
|
|
209
|
-
const fixturesDir = path.join(
|
|
211
|
+
const fixturesDir = path.join(e2eScaffoldDir, 'fixtures');
|
|
210
212
|
fs.mkdirSync(fixturesDir, { recursive: true });
|
|
211
213
|
for (const file of ['auth.ts', 'test-helpers.ts']) {
|
|
212
214
|
const dest = path.join(fixturesDir, file);
|
|
213
215
|
if (!fs.existsSync(dest)) {
|
|
214
216
|
fs.copyFileSync(path.join(scaffoldSrc, 'fixtures', file), dest);
|
|
215
|
-
console.log(` [created] ${path.relative(cwd
|
|
217
|
+
console.log(` [created] ${path.relative(cwd, dest)}`);
|
|
216
218
|
}
|
|
217
219
|
}
|
|
218
220
|
|
|
221
|
+
// --- Integration + Load test placeholders ---
|
|
222
|
+
writeIfMissing(path.join(automationDir, 'integration', 'README.md'),
|
|
223
|
+
`# Integration Tests\n\n> Placeholder for API/integration tests (k6, JMeter, Azure Load Testing, etc.).\n> Each tool gets its own subdirectory.\n`);
|
|
224
|
+
writeIfMissing(path.join(automationDir, 'load', 'README.md'),
|
|
225
|
+
`# Load Tests\n\n> Placeholder for load and performance tests.\n> Each tool gets its own subdirectory (e.g. k6/, jmeter/).\n`);
|
|
226
|
+
|
|
219
227
|
// --- ADO integration placeholder ---
|
|
220
228
|
const adoDir = path.join(qaRoot, '08-azure-integration');
|
|
221
229
|
writeIfMissing(path.join(adoDir, 'README.md'), `# ADO Integration\n\nSee keber/qa-framework integrations/ado-powershell/ for setup instructions.\n`);
|
|
@@ -344,12 +352,12 @@ npm install @keber/ado-qa
|
|
|
344
352
|
**Playwright automation:**
|
|
345
353
|
\`\`\`bash
|
|
346
354
|
npm install @playwright/test
|
|
347
|
-
cd qa/07-automation && npm install && npx playwright install chromium
|
|
355
|
+
cd qa/07-automation/e2e && npm install && npx playwright install chromium
|
|
348
356
|
\`\`\`
|
|
349
357
|
|
|
350
358
|
### 3. Credentials
|
|
351
|
-
- Copy \`qa/07-automation/.env.example\` → \`qa/07-automation/.env\` and fill in credentials
|
|
352
|
-
- Add \`qa/07-automation/.env\` and \`qa/07-automation/.auth/\` to \`.gitignore\`
|
|
359
|
+
- Copy \`qa/07-automation/e2e/.env.example\` → \`qa/07-automation/e2e/.env\` and fill in credentials
|
|
360
|
+
- Add \`qa/07-automation/e2e/.env\` and \`qa/07-automation/e2e/.auth/\` to \`.gitignore\`
|
|
353
361
|
`;
|
|
354
362
|
const nextStepsPath = path.join(qaRoot, 'AGENT-NEXT-STEPS.md');
|
|
355
363
|
fs.writeFileSync(nextStepsPath, nextStepsContent, 'utf8');
|
package/scripts/upgrade.js
CHANGED
|
@@ -8,21 +8,30 @@
|
|
|
8
8
|
*
|
|
9
9
|
* What this command does (safe by design):
|
|
10
10
|
* ALWAYS updates (framework-owned):
|
|
11
|
-
* - .github/skills/
|
|
12
|
-
* - .github/copilot-instructions.md
|
|
13
|
-
* - qa/QA-STRUCTURE-GUIDE.md
|
|
11
|
+
* - .github/skills/ <- New skills architecture; always current
|
|
12
|
+
* - .github/copilot-instructions.md <- Pipeline sequencer; framework-owned
|
|
13
|
+
* - qa/QA-STRUCTURE-GUIDE.md <- Folder reference; framework-owned
|
|
14
|
+
*
|
|
15
|
+
* MIGRATES (v1.5.x -> v1.6.0, non-destructive):
|
|
16
|
+
* - qa/07-automation/playwright.config.ts -> qa/07-automation/e2e/
|
|
17
|
+
* - qa/07-automation/global-setup.ts -> qa/07-automation/e2e/
|
|
18
|
+
* - qa/07-automation/package.json -> qa/07-automation/e2e/
|
|
19
|
+
* - qa/07-automation/.env.example -> qa/07-automation/e2e/
|
|
20
|
+
* - qa/07-automation/fixtures/ -> qa/07-automation/e2e/fixtures/
|
|
21
|
+
* - qa/07-automation/e2e/{module}/ -> qa/07-automation/e2e/tests/{module}/
|
|
22
|
+
* - Creates integration/ and load/ placeholders
|
|
14
23
|
*
|
|
15
24
|
* NEVER touches (project-owned):
|
|
16
|
-
* - qa/01-specifications/
|
|
17
|
-
* - qa/02-test-plans/
|
|
18
|
-
* - qa/03-test-cases/
|
|
19
|
-
* - qa/07-automation/
|
|
20
|
-
* - qa/memory/
|
|
21
|
-
* - qa/README.md
|
|
22
|
-
* - qa/AGENT-NEXT-STEPS.md
|
|
25
|
+
* - qa/01-specifications/ <- Your specs
|
|
26
|
+
* - qa/02-test-plans/ <- Your test plans
|
|
27
|
+
* - qa/03-test-cases/ <- Your TCs
|
|
28
|
+
* - qa/07-automation/e2e/tests/ <- Your Playwright tests (only moves, never overwrites)
|
|
29
|
+
* - qa/memory/ <- Your learnings
|
|
30
|
+
* - qa/README.md <- Your living index
|
|
31
|
+
* - qa/AGENT-NEXT-STEPS.md <- Your sprint queue
|
|
23
32
|
*
|
|
24
33
|
* REPORTS but does not delete:
|
|
25
|
-
* - qa/00-guides/
|
|
34
|
+
* - qa/00-guides/ <- Old location (can be deleted manually)
|
|
26
35
|
*/
|
|
27
36
|
|
|
28
37
|
'use strict';
|
|
@@ -98,6 +107,86 @@ if (fs.existsSync(oldGuidesDir)) {
|
|
|
98
107
|
);
|
|
99
108
|
}
|
|
100
109
|
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
// 5. Migration v1.5.x -> v1.6.0: restructure qa/07-automation/
|
|
112
|
+
// Safe: only moves files when source exists and destination does NOT.
|
|
113
|
+
// ---------------------------------------------------------------------------
|
|
114
|
+
const automationDir = path.join(qaRoot, '07-automation');
|
|
115
|
+
const e2eDir = path.join(automationDir, 'e2e');
|
|
116
|
+
|
|
117
|
+
// 5a. Scaffold files: 07-automation/*.* -> 07-automation/e2e/*.*
|
|
118
|
+
const scaffoldFiles = ['playwright.config.ts', 'global-setup.ts', 'package.json', '.env.example', 'package-lock.json'];
|
|
119
|
+
for (const file of scaffoldFiles) {
|
|
120
|
+
const oldPath = path.join(automationDir, file);
|
|
121
|
+
const newPath = path.join(e2eDir, file);
|
|
122
|
+
if (fs.existsSync(oldPath) && !fs.existsSync(newPath)) {
|
|
123
|
+
if (!dryRun) {
|
|
124
|
+
fs.mkdirSync(e2eDir, { recursive: true });
|
|
125
|
+
fs.renameSync(oldPath, newPath);
|
|
126
|
+
}
|
|
127
|
+
updated.push(newPath);
|
|
128
|
+
console.log(` [migrated] ${path.relative(cwd, oldPath)} -> e2e/${file}`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// 5b. fixtures/: 07-automation/fixtures/ -> 07-automation/e2e/fixtures/
|
|
133
|
+
const oldFixturesDir = path.join(automationDir, 'fixtures');
|
|
134
|
+
const newFixturesDir = path.join(e2eDir, 'fixtures');
|
|
135
|
+
if (fs.existsSync(oldFixturesDir) && !fs.existsSync(newFixturesDir)) {
|
|
136
|
+
if (!dryRun) {
|
|
137
|
+
fs.mkdirSync(e2eDir, { recursive: true });
|
|
138
|
+
fs.renameSync(oldFixturesDir, newFixturesDir);
|
|
139
|
+
}
|
|
140
|
+
updated.push(newFixturesDir);
|
|
141
|
+
console.log(` [migrated] ${path.relative(cwd, oldFixturesDir)}/ -> e2e/fixtures/`);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// 5c. Spec stubs: e2e/{module}/*.spec.ts -> e2e/tests/{module}/
|
|
145
|
+
// Moves any directory directly under e2e/ that is not a known non-module dir.
|
|
146
|
+
const testsDir = path.join(e2eDir, 'tests');
|
|
147
|
+
const knownE2eDirs = new Set(['tests', 'page-objects', 'fixtures', 'diagnosis', 'scripts', 'seeds', 'helpers']);
|
|
148
|
+
if (fs.existsSync(e2eDir)) {
|
|
149
|
+
for (const entry of fs.readdirSync(e2eDir, { withFileTypes: true })) {
|
|
150
|
+
if (!entry.isDirectory() || knownE2eDirs.has(entry.name)) continue;
|
|
151
|
+
const oldModDir = path.join(e2eDir, entry.name);
|
|
152
|
+
const newModDir = path.join(testsDir, entry.name);
|
|
153
|
+
if (!fs.existsSync(newModDir)) {
|
|
154
|
+
if (!dryRun) {
|
|
155
|
+
fs.mkdirSync(testsDir, { recursive: true });
|
|
156
|
+
fs.renameSync(oldModDir, newModDir);
|
|
157
|
+
}
|
|
158
|
+
updated.push(newModDir);
|
|
159
|
+
console.log(` [migrated] e2e/${entry.name}/ -> e2e/tests/${entry.name}/`);
|
|
160
|
+
} else {
|
|
161
|
+
warnings.push(
|
|
162
|
+
`Cannot migrate e2e/${entry.name}/ — target e2e/tests/${entry.name}/ already exists. Merge manually.`
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// 5d. Create integration/ and load/ placeholders if missing.
|
|
169
|
+
const integrationReadme = path.join(automationDir, 'integration', 'README.md');
|
|
170
|
+
const loadReadme = path.join(automationDir, 'load', 'README.md');
|
|
171
|
+
if (!fs.existsSync(integrationReadme)) {
|
|
172
|
+
if (!dryRun) {
|
|
173
|
+
fs.mkdirSync(path.dirname(integrationReadme), { recursive: true });
|
|
174
|
+
fs.writeFileSync(integrationReadme,
|
|
175
|
+
'# Integration Tests\n\n> Placeholder for API/integration tests (k6, JMeter, Azure Load Testing, etc.).\n> Each tool gets its own subdirectory.\n', 'utf8');
|
|
176
|
+
}
|
|
177
|
+
updated.push(integrationReadme);
|
|
178
|
+
console.log(` [created] ${path.relative(cwd, integrationReadme)}`);
|
|
179
|
+
}
|
|
180
|
+
if (!fs.existsSync(loadReadme)) {
|
|
181
|
+
if (!dryRun) {
|
|
182
|
+
fs.mkdirSync(path.dirname(loadReadme), { recursive: true });
|
|
183
|
+
fs.writeFileSync(loadReadme,
|
|
184
|
+
'# Load Tests\n\n> Placeholder for load and performance tests.\n> Each tool gets its own subdirectory (e.g. k6/, jmeter/).\n', 'utf8');
|
|
185
|
+
}
|
|
186
|
+
updated.push(loadReadme);
|
|
187
|
+
console.log(` [created] ${path.relative(cwd, loadReadme)}`);
|
|
188
|
+
}
|
|
189
|
+
|
|
101
190
|
// ---------------------------------------------------------------------------
|
|
102
191
|
// Summary
|
|
103
192
|
// ---------------------------------------------------------------------------
|