@keber/qa-framework 1.11.3 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +340 -0
- package/README.md +42 -2
- package/docs/architecture.md +30 -26
- package/docs/decisions/lane-heartbeat-caller.md +86 -0
- package/docs/final-report.md +23 -23
- package/docs/folder-structure-guide.md +100 -80
- package/docs/installation.md +42 -7
- package/docs/skills-architecture.md +23 -23
- package/docs/spec-driven-philosophy.md +9 -9
- package/docs/usage-with-agent.md +88 -9
- package/docs/visuals/diag-01-ejecutivo-1.png +0 -0
- package/docs/visuals/diag-02-tactico-1.png +0 -0
- package/docs/visuals/diag-03-tecnico-1.png +0 -0
- package/docs/visuals/info-01-ejecutivo-1.png +0 -0
- package/docs/visuals/info-02-tactico-1.png +0 -0
- package/docs/visuals/info-03-tecnico-1.png +0 -0
- package/docs/visuals/pres-01-ejecutivo-1.png +0 -0
- package/docs/visuals/pres-02-tactico-1.png +0 -0
- package/docs/visuals/pres-03-tecnico-1.png +0 -0
- package/docs/visuals/qa-01-problema-1.png +0 -0
- package/docs/visuals/qa-02-que-es-1.png +0 -0
- package/docs/visuals/qa-03-pipeline-1.png +0 -0
- package/docs/visuals/qa-04-6-archivos-1.png +0 -0
- package/docs/visuals/qa-05-humano-1.png +0 -0
- package/docs/visuals/qa-06-agente-1.png +0 -0
- package/docs/visuals/qa-07-skills-arch-1.png +0 -0
- package/docs/visuals/qa-08-traceability-1.png +0 -0
- package/docs/visuals/qa-en-01-problem-1.png +0 -0
- package/docs/visuals/qa-en-02-what-is-1.png +0 -0
- package/docs/visuals/qa-en-03-pipeline-1.png +0 -0
- package/docs/visuals/qa-en-04-6files-1.png +0 -0
- package/docs/visuals/qa-en-05-human-1.png +0 -0
- package/docs/visuals/qa-en-06-agent-1.png +0 -0
- package/docs/visuals/qa-en-07-skills-arch-1.png +0 -0
- package/docs/visuals/qa-en-08-traceability-1.png +0 -0
- package/examples/module-example/suppliers/00-inventory.md +1 -1
- package/examples/module-example/suppliers/suppliers-create.spec.ts +8 -8
- package/integrations/ado-powershell/README.md +1 -1
- package/integrations/ado-powershell/scripts/sync-ado-titles.ps1 +2 -2
- package/integrations/playwright/README.md +4 -4
- package/package.json +7 -3
- package/qa-framework.config.json +25 -3
- package/scripts/check-forbidden-chars.js +187 -0
- package/scripts/check-spanish-accents.js +159 -0
- package/scripts/cli.js +1 -1
- package/scripts/generate.js +2 -2
- package/scripts/init.js +101 -12
- package/scripts/lib/claude-agents.js +347 -0
- package/scripts/lib/claude-commands.js +56 -0
- package/scripts/lib/module-keys.js +54 -0
- package/scripts/sync-version.js +63 -0
- package/scripts/upgrade.js +369 -20
- package/scripts/validate.js +46 -21
- package/skills/qa-ado-integration/SKILL.md +19 -19
- package/skills/qa-ado-integration/references/scripts-and-config.md +6 -6
- package/skills/qa-automation/SKILL.md +49 -27
- package/skills/qa-automation/references/config-checklist.md +4 -4
- package/skills/qa-automation/references/dom-inspection-template.js +7 -7
- package/skills/qa-automation/references/patterns.md +20 -20
- package/skills/qa-automation/references/pom-template.md +161 -0
- package/skills/qa-maintenance/SKILL.md +11 -11
- package/skills/qa-maintenance/references/update-rules.md +8 -8
- package/skills/qa-module-analysis/SKILL.md +20 -19
- package/skills/qa-module-analysis/references/exploration-checklist.md +4 -4
- package/skills/qa-module-analysis/references/output-verification.md +69 -0
- package/skills/qa-module-analysis/references/spec-file-formats.md +14 -14
- package/skills/qa-spec-generation/SKILL.md +13 -7
- package/skills/qa-test-cases/SKILL.md +11 -81
- package/skills/qa-test-cases/references/test-case-template.md +5 -5
- package/skills/qa-test-plan/SKILL.md +22 -86
- package/skills/qa-test-plan/references/plan-de-pruebas-template.md +24 -24
- package/skills/qa-test-stabilization/SKILL.md +59 -30
- package/skills/qa-test-stabilization/references/classification-protocol.md +35 -35
- package/templates/agent-next-steps.md +10 -10
- package/templates/agents/qa-analisis.md +142 -0
- package/templates/agents/qa-asesoria.md +48 -0
- package/templates/agents/qa-informe-resultados.md +236 -0
- package/templates/agents/qa-investigador.md +143 -0
- package/templates/agents/qa-plan.md +131 -0
- package/templates/agents/qa-reporte-ejecucion.md +138 -0
- package/templates/automation-scaffold/.env.example +36 -6
- package/templates/automation-scaffold/fixtures/auth.ts +27 -5
- package/templates/automation-scaffold/fixtures/base.ts +23 -1
- package/templates/automation-scaffold/fixtures/test-helpers.ts +1 -1
- package/templates/automation-scaffold/global-setup.ts +117 -48
- package/templates/automation-scaffold/playwright.config.ts +72 -32
- package/templates/automation-scaffold/scripts/global-setup-guards.js +184 -0
- package/templates/automation-scaffold/scripts/lane-config.js +269 -0
- package/templates/automation-scaffold/scripts/lane-lock.js +589 -0
- package/templates/defect-report.md +17 -3
- package/templates/execution-report.md +9 -9
- package/templates/integration-scaffold/.env.example +1 -1
- package/templates/integration-scaffold/global-setup.ts +21 -1
- package/templates/integration-scaffold/playwright.config.ts +15 -1
- package/templates/qa-framework.instructions.md +9 -8
- package/templates/qa-framework.rules.md +53 -0
- package/templates/qa-readme.md +8 -8
- package/templates/session-summary.md +2 -2
- package/templates/specification/00-inventory.md +3 -3
- package/templates/specification/01-business-rules.md +6 -6
- package/templates/specification/02-workflows.md +1 -1
- package/templates/specification/03-roles-permissions.md +1 -1
- package/templates/specification/04-test-data.md +8 -8
- package/templates/specification/05-test-scenarios.md +5 -5
- package/templates/test-case.md +3 -3
- package/templates/test-data-guidelines.md +7 -7
- package/templates/test-plan-sprint.md +13 -13
- package/templates/test-plan.md +3 -3
- package/skills/qa-test-plan/references/priority-and-feasibility.md +0 -159
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: qa-reporte-ejecucion
|
|
3
|
+
description: Generates or updates a sprint's Reporte de Ejecución de Pruebas (raw ADO results table PlanId/SuiteId/TestCaseId/.../Outcome/.../AttachmentUrls exported from Azure DevOps, plus a summary and a promoted-findings table). Use it when the user asks for "reporte de ejecución de pruebas", "reporte de ejecución del sprint N", "reporte de pruebas del sprint" or "exportar los resultados del test plan". It does NOT generate the Informe de Resultados de Pruebas -- that is the `qa-informe-resultados` agent, which CONSUMES this Reporte as its input.
|
|
4
|
+
tools: Read, Write, Grep, Glob, Bash
|
|
5
|
+
model: sonnet
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# Role
|
|
9
|
+
|
|
10
|
+
You are a QA assistant focused on **promoting the raw Azure DevOps export into a stable sprint artifact**. You do not interpret business results and you do not write a closeout narrative -- that is the `qa-informe-resultados` agent's job. Your job is to export (when needed), version the raw output, and produce the Reporte de Ejecución de Pruebas.
|
|
11
|
+
|
|
12
|
+
# Audience and style
|
|
13
|
+
|
|
14
|
+
- Audience: **internal QA team and ADO**. This document is ADO-centric and serves as technical input; it is not what business/PM reads (that is the Informe de Resultados).
|
|
15
|
+
- Tone: technical, no business interpretation.
|
|
16
|
+
- **The generated document is written in Spanish**, and so are its headings. These instructions are in English; the artifact is not.
|
|
17
|
+
- Dates in the body of the document use the **{{DATE_FORMAT}}** format in the **{{TIMEZONE}}** timezone. The only exception is the `ado-export-<YYYY-MM-DD>/` folder name, which stays ISO `YYYY-MM-DD` so the folders sort chronologically.
|
|
18
|
+
- **Accents and n-tilde are mandatory and always permitted** in the generated document (`á é í ó ú ñ ü`, and any Latin Extended character U+00C0-U+024F). NEVER strip accents to make text "ASCII-safe": stripped accents are a known, recurring defect in agent-generated artifacts, not a convention. Write correct Spanish, including on uppercase letters.
|
|
19
|
+
- The encoding prohibition is narrow and covers exactly 4 symbol classes, which you MUST replace: em-dash and en-dash -> ` - ` (hyphen with surrounding spaces); unicode ellipsis -> `...`; curly quotes -> `"` and `'`; unicode arrows -> `->` and `<-`. Emojis are allowed.
|
|
20
|
+
- `.md` files: UTF-8 **without** BOM.
|
|
21
|
+
|
|
22
|
+
# The four sprint closeout artifacts (do not conflate them)
|
|
23
|
+
|
|
24
|
+
This agent exists because these documents are easy to confuse: a request for "el reporte de ejecución del sprint" must never be answered with the Informe de Resultados. These are the distinct documents a user may ask for:
|
|
25
|
+
|
|
26
|
+
1. **Raw ADO export** (not your final output - it is your input): what `getADO.ps1` from the `ado-qa` skill produces -- `plan_<id>_suite_multi_results.{md,csv}` plus one `TC_*.txt` per test case. It is kept **versioned by date** under `qa/02-test-plans/historical/sprint-{N}/ado-export-<YYYY-MM-DD>/` and is **NEVER overwritten**: every new export goes to its own dated folder. This is what enables progress history and intra-sprint interim reports.
|
|
27
|
+
2. **Reporte de Ejecución de Pruebas** (THIS IS YOUR OUTPUT): the promoted copy of the most recent raw export, with 3 added layers (provenance header, summary, findings). See the template below.
|
|
28
|
+
3. **Informe de Resultados de Pruebas**: the narrative closeout document business/PM reads to decide whether to release the sprint. Produced by the `qa-informe-resultados` agent, which **consumes** the Reporte you generate. You do NOT produce this document.
|
|
29
|
+
4. **Execution report for a single Playwright run** (`qa/00-standards/execution-report-template.md`): a fourth, distinct document covering pass/fail/skip of one specific run with screenshots. It lives under `qa/05-test-execution/`. This is not your document either -- do not generate it and do not confuse it with the sprint Reporte.
|
|
30
|
+
|
|
31
|
+
# Process
|
|
32
|
+
|
|
33
|
+
1. **Decide whether an export is needed.** If the user provides a `-PlanId` and no fresh export exists for this sprint (or the user explicitly asks to refresh), run the raw export through the `ado-qa` skill. Keep it all in ONE command/script so the PowerShell session persists:
|
|
34
|
+
|
|
35
|
+
```powershell
|
|
36
|
+
. .\config.ps1
|
|
37
|
+
. .\.github\skills\ado-qa\load.ps1
|
|
38
|
+
& '.github/skills/ado-qa/scripts/getADO/getADO.ps1' -PlanId <id> -OutDir './qa/02-test-plans/historical/sprint-<N>/ado-export-<YYYY-MM-DD>'
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Credentials (`ADO_ORG`, `ADO_PROJECT`, `ADO_PAT`) come from `config.ps1` (gitignored) -- **never hardcode them** and never request them in plain text.
|
|
42
|
+
|
|
43
|
+
2. **Promote the raw export into the Reporte.** Copy the table from `plan_<id>_suite_multi_results.md` (or `plan_<id>_results.md`, depending on the script version) verbatim into the body of the Reporte, adding:
|
|
44
|
+
- a provenance header (command used, export date, pointer to the `ado-export-<fecha>/` folder),
|
|
45
|
+
- a `## Resumen` section with the totals,
|
|
46
|
+
- a `## Hallazgos` section with the promoted-candidates table (see the rule below).
|
|
47
|
+
|
|
48
|
+
3. **Build the findings table.**
|
|
49
|
+
|
|
50
|
+
## Promotion rule (BLOCKING)
|
|
51
|
+
|
|
52
|
+
Source: `qa/06-defects/` (subfolders `open/`, `resolved/`, `disputed/`, `dismissed/`), filtered by the sprint in question.
|
|
53
|
+
|
|
54
|
+
**A `DEF-{{DEFECT_ID_PREFIX}}-NNN` is not a defect: it is a defect candidate.** It is promoted to an official defect only once it has an ADO Work Item, and that happens only after ruling out a test implementation defect, ruling out environmental conditions, and reproducing it 3 consecutive times.
|
|
55
|
+
|
|
56
|
+
Therefore:
|
|
57
|
+
- **Only promoted candidates enter the findings table**, meaning those that have an ADO Id. A `DEF-{{DEFECT_ID_PREFIX}}-NNN` with no ADO Id is not a sprint finding and does NOT belong in the table -- it is ongoing investigation. Including it is a classification error.
|
|
58
|
+
- **Never emit local identifiers** (`DEF-{{DEFECT_ID_PREFIX}}-NNN`, `{{TC_ID_PATTERN}}`) in the body of the Reporte. They are used internally to DERIVE the table, but only numeric `TestCaseId` and `ADO Id` values go out. This document's audience is ADO-centric.
|
|
59
|
+
|
|
60
|
+
## Metric rules (BLOCKING)
|
|
61
|
+
|
|
62
|
+
The `## Resumen` table uses the same metric definitions as `qa-informe-resultados`, so both documents always agree:
|
|
63
|
+
|
|
64
|
+
- `ejecutados` = `passed + failed`. Son los únicos casos que realmente corrieron y produjeron un veredicto de ejecución.
|
|
65
|
+
- `na` = `NotApplicable + Blocked + sin_outcome` (`sin_outcome` = filas sin `Outcome` registrado). Ninguno de estos se ejecutó.
|
|
66
|
+
- `exito_ejecutados` = `passed / ejecutados` si `ejecutados > 0`; si no, `N/A`.
|
|
67
|
+
- `cobertura` = `ejecutados / total_planificados` (`total_planificados` = total de filas del reporte).
|
|
68
|
+
- Percentages use `{{DECIMAL_SEPARATOR}}` as the decimal separator.
|
|
69
|
+
|
|
70
|
+
**A `NotApplicable` does NOT count as executed**: it is a `test.skip()`, exactly like a row without `Outcome`. In the `## Resumen` table, `Total TCs` is `total_planificados`, `Ejecutados` is `ejecutados`, `% éxito sobre ejecutados` is `exito_ejecutados` and `Cobertura` is `cobertura`; `Blocked`, `NotApplicable` and `Sin Outcome` are reported separately and add up to `na`.
|
|
71
|
+
|
|
72
|
+
Reference: approval threshold {{APPROVAL_THRESHOLD_PERCENT}}% of executed cases (`exito_ejecutados` estrictamente mayor que {{APPROVAL_THRESHOLD_PERCENT}}%, i.e. `> {{APPROVAL_THRESHOLD_RATIO}}`); the verdict is applied by qa-informe-resultados, not here.
|
|
73
|
+
|
|
74
|
+
# Output -- Reporte template
|
|
75
|
+
|
|
76
|
+
The generated document is in Spanish. Reproduce these headings verbatim, accents included:
|
|
77
|
+
|
|
78
|
+
```md
|
|
79
|
+
# Reporte de Ejecución de Pruebas - Sprint <N>
|
|
80
|
+
|
|
81
|
+
**Fuente**: export directo de Azure DevOps, Test Plan <id> (`getADO -PlanId <id>`), <fecha en {{DATE_FORMAT}}>.
|
|
82
|
+
Export crudo (TXT/CSV/MD por Test Case) en `<ruta de la carpeta ado-export-fecha>`.
|
|
83
|
+
|
|
84
|
+
## Resumen
|
|
85
|
+
|
|
86
|
+
| Total TCs | Passed | Failed | Blocked | NotApplicable | Sin Outcome | Ejecutados | % éxito sobre ejecutados | Cobertura |
|
|
87
|
+
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
88
|
+
| <n> | <n> | <n> | <n> | <n> | <n> | <n> | <%> | <%> |
|
|
89
|
+
|
|
90
|
+
## Test Plan: <nombre del plan>
|
|
91
|
+
|
|
92
|
+
| PlanId | SuiteId | TestCaseId | Title | TestPointIds | Outcome | CompletedDate | RunId | Observations | AttachmentUrls |
|
|
93
|
+
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
94
|
+
| <fila cruda tal cual sale del export> |
|
|
95
|
+
|
|
96
|
+
## Hallazgos (Bugs/Issues generados)
|
|
97
|
+
|
|
98
|
+
| Tipo | TestCaseId | Outcome | RunId | ADO Id | ADO Link |
|
|
99
|
+
| --- | --- | --- | --- | --- | --- |
|
|
100
|
+
| Bug/Issue | <id> | Failed/Blocked | <runId> | <ado_id> | <link completo> |
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The pointer to the raw export folder in the header is **mandatory**: it is what keeps the raw export from being "hidden" and preserves traceability back to the version that produced the Reporte.
|
|
104
|
+
|
|
105
|
+
# Delivery rule
|
|
106
|
+
|
|
107
|
+
- Path and filename: `qa/02-test-plans/historical/sprint-{N}/Reporte-de-Ejecucion-de-Pruebas-{{PROJECT_NAME}}-Sprint-{N}.md` if the sprint is already closed, or `qa/02-test-plans/sprints/Sprint-{N}/Reporte-de-Ejecucion-de-Pruebas-{{PROJECT_NAME}}-Sprint-{N}.md` if it is still active. This is the canonical name that `qa-informe-resultados` looks for, so do not vary it; `{N}` is the sprint number **without zero-padding**. Older reports named `Reporte-ejecución-de-pruebas-...` may exist in the repository: leave them as they are and do not rename them. Ask the user if you are unsure whether the sprint is active or historical.
|
|
108
|
+
- If the directory does not exist, create it before writing (use `Write`, which creates intermediate paths when the harness allows it; if it does not, say so in your response).
|
|
109
|
+
- **If a Reporte already exists for that sprint, do not blindly overwrite it.** The new raw export always goes to its own dated folder (`ado-export-<today>/`, never on top of an earlier dated folder), and the Reporte is regenerated from the latest available raw export, recording in the header which export date was used.
|
|
110
|
+
- When you finish, your response must state the **exact relative path** of the file written.
|
|
111
|
+
- Never report the task as complete if the file was not written with the `Write` tool.
|
|
112
|
+
|
|
113
|
+
## Mandatory closing message (always)
|
|
114
|
+
|
|
115
|
+
Besides the file path, close your response with this block (kept in Spanish, since it is addressed to the user):
|
|
116
|
+
|
|
117
|
+
```
|
|
118
|
+
Reporte generado/actualizado: <ruta>
|
|
119
|
+
Métricas: Total=<n>, Passed=<n>, Failed=<n>, Blocked=<n>, NotApplicable=<n>, Sin Outcome=<n>, Ejecutados=<n>, Éxito=<%>, Cobertura=<%>
|
|
120
|
+
Hallazgos promovidos (con ADO Id): <n>
|
|
121
|
+
Candidatos excluidos (DEF-{{DEFECT_ID_PREFIX}}-NNN sin ADO Id, no entran a la tabla): <n>
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
# Quality checklist (tick before delivering)
|
|
125
|
+
|
|
126
|
+
- [ ] If a new export was run, it used `config.ps1` + `load.ps1` + `getADO.ps1` in a single command/script, with no hardcoded credentials.
|
|
127
|
+
- [ ] The new raw export landed in its own `ado-export-<fecha>/` folder without overwriting earlier exports.
|
|
128
|
+
- [ ] The Reporte header explicitly points to the raw export folder used.
|
|
129
|
+
- [ ] The `## Test Plan: ...` table reproduces the raw export rows verbatim, without reinterpreting any `Outcome`.
|
|
130
|
+
- [ ] The `## Resumen` metrics follow the metric rules: `NotApplicable`, `Blocked` and rows without `Outcome` are NOT counted as executed.
|
|
131
|
+
- [ ] The findings table contains ONLY candidates with an ADO Id (promoted defects), never bare `DEF-{{DEFECT_ID_PREFIX}}-NNN`.
|
|
132
|
+
- [ ] No local identifier (`DEF-{{DEFECT_ID_PREFIX}}-NNN`, `{{TC_ID_PATTERN}}`) appears in the body of the Reporte.
|
|
133
|
+
- [ ] If a Reporte already existed for this sprint, it was not blindly overwritten: it was regenerated from the most recent raw export and the export date is recorded in the header.
|
|
134
|
+
- [ ] The file `Reporte-de-Ejecucion-de-Pruebas-{{PROJECT_NAME}}-Sprint-{N}.md` physically exists at the stated path.
|
|
135
|
+
- [ ] The mandatory closing message (path + metrics + promoted findings + excluded candidates) is included in the response.
|
|
136
|
+
- [ ] The generated document is written in correct Spanish, WITH accents and n-tilde. No accent was stripped to make it "ASCII-safe".
|
|
137
|
+
|
|
138
|
+
If any condition is unmet, the response is considered incomplete.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
# keber/qa-framework
|
|
1
|
+
# -----------------------------------------------------------------------
|
|
2
|
+
# keber/qa-framework - Environment Variables Reference
|
|
3
3
|
# -----------------------------------------------------------------------
|
|
4
4
|
# Copy this file to .env and fill in real values.
|
|
5
5
|
# NEVER commit .env to source control.
|
|
@@ -17,12 +17,42 @@ QA_SESSION_TTL_MS=7200000
|
|
|
17
17
|
# Optional path to a server-side logout endpoint if the app exposes one.
|
|
18
18
|
# QA_LOGOUT_PATH=/logout
|
|
19
19
|
|
|
20
|
-
#
|
|
20
|
+
# -----------------------------------------------------------------------
|
|
21
|
+
# Parallel QA lanes
|
|
22
|
+
# -----------------------------------------------------------------------
|
|
23
|
+
# A lane is a distinct QA account. Lanes exist because the app under test
|
|
24
|
+
# invalidates a session when the same account logs in again, so two workers
|
|
25
|
+
# sharing one account silently destroy each other's session.
|
|
26
|
+
#
|
|
27
|
+
# The lane table itself lives in qa-framework.config.json (parallelLanes.lanes).
|
|
28
|
+
# Each lane needs its own credentials here, named after that lane's "account"
|
|
29
|
+
# prefix. With no parallelLanes config, only the QA_USER pair below is used.
|
|
30
|
+
|
|
31
|
+
# Pin a run to a single lane. Required by global-setup unless QA_COLD_SETUP=1.
|
|
32
|
+
# QA_LANE_ONLY=1
|
|
33
|
+
|
|
34
|
+
# Escape hatch for an intentional relogin of EVERY lane. A cold setup can
|
|
35
|
+
# invalidate another agent's or CI job's running session, so it is never implicit.
|
|
36
|
+
# QA_COLD_SETUP=1
|
|
37
|
+
|
|
38
|
+
# Log in a specific set of lanes (comma-separated lane ids).
|
|
21
39
|
# QA_AUTH_USER=1
|
|
22
40
|
|
|
23
|
-
#
|
|
41
|
+
# Shared lane-lock state directory. Set this to an absolute shared path when
|
|
42
|
+
# running from multiple git worktrees or checkouts: with the per-checkout default
|
|
43
|
+
# each one gets its OWN lock state, so they never contend, every command reports
|
|
44
|
+
# success, and two sessions can land on the same account. The failure is silent.
|
|
45
|
+
# QA_LANE_LOCKS_DIR=/absolute/path/to/shared/.locks
|
|
46
|
+
|
|
47
|
+
# Minutes before a held lane is swept as stale. A lane with a recent heartbeat is
|
|
48
|
+
# never swept regardless of age.
|
|
49
|
+
# QA_LANE_STALE_MINUTES=20
|
|
50
|
+
|
|
51
|
+
# Credentials for additional lanes, one pair per lane account prefix.
|
|
24
52
|
# QA_USER2_EMAIL=qa-user2@example.com
|
|
25
53
|
# QA_USER2_PASSWORD=CHANGE_ME
|
|
54
|
+
# QA_USER3_EMAIL=qa-user3@example.com
|
|
55
|
+
# QA_USER3_PASSWORD=CHANGE_ME
|
|
26
56
|
|
|
27
57
|
# -----------------------------------------------------------------------
|
|
28
58
|
# Login Configuration
|
|
@@ -56,12 +86,12 @@ QA_USER_PASSWORD=CHANGE_ME
|
|
|
56
86
|
# QA_USER_RUT=12345678-9
|
|
57
87
|
|
|
58
88
|
# -----------------------------------------------------------------------
|
|
59
|
-
# Azure DevOps Integration (optional
|
|
89
|
+
# Azure DevOps Integration (optional - leave blank to disable)
|
|
60
90
|
# -----------------------------------------------------------------------
|
|
61
91
|
# ADO_ORG_URL=https://dev.azure.com/your-org
|
|
62
92
|
# ADO_PROJECT_NAME=YourProject
|
|
63
93
|
# ADO_PLAN_ID=00000
|
|
64
|
-
# AZURE_TOKEN=your-pat-token-here # Personal Access Token
|
|
94
|
+
# AZURE_TOKEN=your-pat-token-here # Personal Access Token - NEVER commit
|
|
65
95
|
|
|
66
96
|
# -----------------------------------------------------------------------
|
|
67
97
|
# CI / Pipeline overrides
|
|
@@ -10,11 +10,11 @@
|
|
|
10
10
|
* await loginAs(page, 'admin');
|
|
11
11
|
*
|
|
12
12
|
* Alternatively, use storageState via global-setup.ts (recommended for
|
|
13
|
-
* performance
|
|
13
|
+
* performance - avoids repeating the login flow on every test).
|
|
14
14
|
* Use loginAs() only in tests that need to validate the login flow itself
|
|
15
15
|
* or switch users mid-test.
|
|
16
16
|
*
|
|
17
|
-
* Supported roles are driven by env vars
|
|
17
|
+
* Supported roles are driven by env vars - see .env.example.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
import { Page } from '@playwright/test';
|
|
@@ -26,7 +26,7 @@ interface LoginConfig {
|
|
|
26
26
|
password: string;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
/** Map role name
|
|
29
|
+
/** Map role name -> credentials from environment variables */
|
|
30
30
|
function getCredentials(role: QARole): LoginConfig {
|
|
31
31
|
switch (role) {
|
|
32
32
|
case 'admin':
|
|
@@ -58,7 +58,7 @@ export async function loginAs(page: Page, role: QARole = 'default'): Promise<voi
|
|
|
58
58
|
if (!email || !password) {
|
|
59
59
|
throw new Error(
|
|
60
60
|
`[qa-framework/auth] Missing credentials for role "${role}". ` +
|
|
61
|
-
`Check your .env file
|
|
61
|
+
`Check your .env file - expected ${role.toUpperCase()}_EMAIL and ${role.toUpperCase()}_PASSWORD.`
|
|
62
62
|
);
|
|
63
63
|
}
|
|
64
64
|
|
|
@@ -71,7 +71,29 @@ export async function loginAs(page: Page, role: QARole = 'default'): Promise<voi
|
|
|
71
71
|
|
|
72
72
|
await page.goto(`${baseURL}${loginPath}`, { waitUntil: 'domcontentloaded' });
|
|
73
73
|
await page.locator(emailSelector).fill(email);
|
|
74
|
-
|
|
74
|
+
// Trace safety: Playwright records fill() argument values in traces, and this
|
|
75
|
+
// project runs with trace/video 'retain-on-failure'. This helper runs for every
|
|
76
|
+
// test that logs in as a role, so a fill() here would leak the plaintext password
|
|
77
|
+
// into far more failure artifacts than the one-time global setup does. Setting the
|
|
78
|
+
// value through evaluate() keeps it out of the trace.
|
|
79
|
+
// Do not "simplify" this back to fill().
|
|
80
|
+
// The input/change events are required because assigning .value directly does
|
|
81
|
+
// not notify SPA frameworks (Blazor/Radzen bind on those events).
|
|
82
|
+
// evaluate() has no auto-wait, so the explicit waitFor replaces the one that
|
|
83
|
+
// locator().fill() performed implicitly.
|
|
84
|
+
await page.locator(passwordSelector).waitFor({ state: 'visible', timeout: 30_000 });
|
|
85
|
+
await page.evaluate(
|
|
86
|
+
([selector, pwd]) => {
|
|
87
|
+
const input = document.querySelector(selector) as HTMLInputElement | null;
|
|
88
|
+
if (!input) {
|
|
89
|
+
throw new Error(`[qa-framework] Password input not found for selector: ${selector}`);
|
|
90
|
+
}
|
|
91
|
+
input.value = pwd;
|
|
92
|
+
input.dispatchEvent(new Event('input', { bubbles: true }));
|
|
93
|
+
input.dispatchEvent(new Event('change', { bubbles: true }));
|
|
94
|
+
},
|
|
95
|
+
[passwordSelector, password] as const
|
|
96
|
+
);
|
|
75
97
|
await page.locator(submitSelector).click();
|
|
76
98
|
await page.waitForSelector(successSelector, { timeout: 15_000 });
|
|
77
99
|
}
|
|
@@ -63,7 +63,29 @@ async function reauth(browser: import('@playwright/test').Browser, stateFile: st
|
|
|
63
63
|
try {
|
|
64
64
|
await page.goto(loginPath, { waitUntil: 'load', timeout: 60_000 });
|
|
65
65
|
await page.locator(emailSelector).fill(email);
|
|
66
|
-
|
|
66
|
+
// Trace safety: Playwright records fill() argument values in traces, and this
|
|
67
|
+
// project runs with trace/video 'retain-on-failure'. This reauth path runs for
|
|
68
|
+
// every test whose session exceeded the TTL, so a fill() here would leak the
|
|
69
|
+
// plaintext password into far more failure artifacts than the one-time global
|
|
70
|
+
// setup does. Setting the value through evaluate() keeps it out of the trace.
|
|
71
|
+
// Do not "simplify" this back to fill().
|
|
72
|
+
// The input/change events are required because assigning .value directly does
|
|
73
|
+
// not notify SPA frameworks (Blazor/Radzen bind on those events).
|
|
74
|
+
// evaluate() has no auto-wait, so the explicit waitFor replaces the one that
|
|
75
|
+
// locator().fill() performed implicitly.
|
|
76
|
+
await page.locator(passwordSelector).waitFor({ state: 'visible', timeout: 30_000 });
|
|
77
|
+
await page.evaluate(
|
|
78
|
+
([selector, pwd]) => {
|
|
79
|
+
const input = document.querySelector(selector) as HTMLInputElement | null;
|
|
80
|
+
if (!input) {
|
|
81
|
+
throw new Error(`[qa-framework] Password input not found for selector: ${selector}`);
|
|
82
|
+
}
|
|
83
|
+
input.value = pwd;
|
|
84
|
+
input.dispatchEvent(new Event('input', { bubbles: true }));
|
|
85
|
+
input.dispatchEvent(new Event('change', { bubbles: true }));
|
|
86
|
+
},
|
|
87
|
+
[passwordSelector, password] as const
|
|
88
|
+
);
|
|
67
89
|
await page.locator(submitSelector).click();
|
|
68
90
|
await page.waitForSelector(successSelector, { timeout: 30_000 });
|
|
69
91
|
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
// -----------------------------------------------------------------------
|
|
9
|
-
// EXEC_IDX
|
|
9
|
+
// EXEC_IDX - unique numeric seed per minute-long execution window
|
|
10
10
|
// -----------------------------------------------------------------------
|
|
11
11
|
// Use this wherever test data needs to be unique (e.g., names, emails,
|
|
12
12
|
// codes) to avoid collisions between parallel or consecutive test runs
|
|
@@ -11,14 +11,14 @@
|
|
|
11
11
|
* - Reference the matching storageState in playwright.config.ts projects[].
|
|
12
12
|
*
|
|
13
13
|
* Environment variables required:
|
|
14
|
-
* QA_BASE_URL
|
|
15
|
-
* QA_USER_EMAIL
|
|
16
|
-
* QA_USER_PASSWORD
|
|
17
|
-
* QA_LOGIN_PATH
|
|
18
|
-
* QA_LOGIN_EMAIL_SELECTOR
|
|
19
|
-
* QA_LOGIN_PASSWORD_SELECTOR
|
|
20
|
-
* QA_LOGIN_SUBMIT_SELECTOR
|
|
21
|
-
* QA_LOGIN_SUCCESS_SELECTOR
|
|
14
|
+
* QA_BASE_URL - Base URL of the application under test
|
|
15
|
+
* QA_USER_EMAIL - Default QA user email (or username/RUT)
|
|
16
|
+
* QA_USER_PASSWORD - Default QA user password
|
|
17
|
+
* QA_LOGIN_PATH - Relative path to the login page (default: /login)
|
|
18
|
+
* QA_LOGIN_EMAIL_SELECTOR - CSS selector for the username/email input
|
|
19
|
+
* QA_LOGIN_PASSWORD_SELECTOR - CSS selector for the password input
|
|
20
|
+
* QA_LOGIN_SUBMIT_SELECTOR - CSS selector for the submit button
|
|
21
|
+
* QA_LOGIN_SUCCESS_SELECTOR - CSS selector that confirms successful login
|
|
22
22
|
*
|
|
23
23
|
* See .env.example for all supported variables.
|
|
24
24
|
*/
|
|
@@ -30,6 +30,20 @@ import * as fs from 'fs';
|
|
|
30
30
|
|
|
31
31
|
dotenv.config();
|
|
32
32
|
|
|
33
|
+
// Lane table and guards. Both are CommonJS so the guard logic stays testable with
|
|
34
|
+
// `node --test` without adding ts-node or tsx as a dependency.
|
|
35
|
+
const laneLock = require('./scripts/lane-lock.js');
|
|
36
|
+
const guards = require('./scripts/global-setup-guards.js');
|
|
37
|
+
|
|
38
|
+
type LaneInfo = {
|
|
39
|
+
account: string;
|
|
40
|
+
storageState: string;
|
|
41
|
+
mcpNamespace: string | null;
|
|
42
|
+
runnerProject: string | null;
|
|
43
|
+
capabilities: string[];
|
|
44
|
+
};
|
|
45
|
+
const LANE_INFO: Record<string, LaneInfo> = laneLock.LANE_INFO;
|
|
46
|
+
|
|
33
47
|
function resolveVar(param: string | undefined, envKey: string, defaultValue: string): [string, string] {
|
|
34
48
|
if (param !== undefined) {
|
|
35
49
|
return [param, 'param'];
|
|
@@ -40,37 +54,51 @@ function resolveVar(param: string | undefined, envKey: string, defaultValue: str
|
|
|
40
54
|
return [defaultValue, 'default'];
|
|
41
55
|
}
|
|
42
56
|
|
|
43
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Decides which lanes to log in.
|
|
59
|
+
*
|
|
60
|
+
* Resolution order:
|
|
61
|
+
* 1. QA_LANE_ONLY - pins the run to exactly one lane. This is the normal path and
|
|
62
|
+
* the only one the guards permit without an explicit cold-setup opt-in.
|
|
63
|
+
* 2. QA_AUTH_USER - comma-separated lane ids, for an intentional multi-lane setup.
|
|
64
|
+
* 3. the projects' storageState, matched against the lane table.
|
|
65
|
+
* 4. every configured lane (the cold-setup path, gated by QA_COLD_SETUP=1).
|
|
66
|
+
*
|
|
67
|
+
* Step 3 matches storageState by EXACT equality against the lane table rather than by
|
|
68
|
+
* substring. The previous scaffold used storageState.includes('user-2'), which is what
|
|
69
|
+
* breaks first beyond two lanes: 'user-2' is a substring of 'user-20', and there was no
|
|
70
|
+
* branch at all for a third lane, so lanes past the second were silently never logged in.
|
|
71
|
+
*/
|
|
72
|
+
function resolveAuthLanes(config: FullConfig): string[] {
|
|
73
|
+
const laneIds = Object.keys(LANE_INFO);
|
|
74
|
+
|
|
75
|
+
const laneOnly = process.env.QA_LANE_ONLY;
|
|
76
|
+
if (laneOnly) {
|
|
77
|
+
// Throws on an unknown lane rather than guessing. Guessing is how a run logs into
|
|
78
|
+
// an account it does not hold.
|
|
79
|
+
return [laneLock.laneIdForQaLaneOnly(laneOnly)];
|
|
80
|
+
}
|
|
81
|
+
|
|
44
82
|
const explicit = process.env.QA_AUTH_USER;
|
|
45
83
|
if (explicit) {
|
|
46
|
-
const resolved =
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
}
|
|
52
|
-
if (resolved.size) {
|
|
53
|
-
return resolved;
|
|
54
|
-
}
|
|
84
|
+
const resolved = explicit
|
|
85
|
+
.split(',')
|
|
86
|
+
.map((value) => value.trim())
|
|
87
|
+
.filter((value) => laneIds.includes(value));
|
|
88
|
+
if (resolved.length) return resolved;
|
|
55
89
|
}
|
|
56
90
|
|
|
57
|
-
const fromProjects = new Set<
|
|
91
|
+
const fromProjects = new Set<string>();
|
|
58
92
|
for (const project of config.projects) {
|
|
59
93
|
const storageState = (project.use as { storageState?: unknown } | undefined)?.storageState;
|
|
60
|
-
if (typeof storageState
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
} else if (storageState.includes('user-1')) {
|
|
64
|
-
fromProjects.add('1');
|
|
65
|
-
}
|
|
94
|
+
if (typeof storageState !== 'string') continue;
|
|
95
|
+
for (const laneId of laneIds) {
|
|
96
|
+
if (LANE_INFO[laneId].storageState === storageState) fromProjects.add(laneId);
|
|
66
97
|
}
|
|
67
98
|
}
|
|
99
|
+
if (fromProjects.size) return [...fromProjects];
|
|
68
100
|
|
|
69
|
-
|
|
70
|
-
return fromProjects;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
return new Set(['1']);
|
|
101
|
+
return laneIds;
|
|
74
102
|
}
|
|
75
103
|
|
|
76
104
|
async function dismissOnboardingFlow(page: import('@playwright/test').Page): Promise<void> {
|
|
@@ -133,7 +161,27 @@ async function loginAs(params: {
|
|
|
133
161
|
for (let attempt = 1; attempt <= 3 && !signedIn; attempt++) {
|
|
134
162
|
await page.goto(loginUrl, { waitUntil: 'load', timeout: 60_000 });
|
|
135
163
|
await page.locator(emailSelector).fill(email);
|
|
136
|
-
|
|
164
|
+
// Trace safety: Playwright records fill() argument values in traces, and this
|
|
165
|
+
// project runs with trace/video 'retain-on-failure' - a failed CI run would
|
|
166
|
+
// persist the plaintext password in the artifact. Setting the value through
|
|
167
|
+
// evaluate() keeps it out of the trace. Do not "simplify" this back to fill().
|
|
168
|
+
// The input/change events are required because assigning .value directly does
|
|
169
|
+
// not notify SPA frameworks (Blazor/Radzen bind on those events).
|
|
170
|
+
// evaluate() has no auto-wait, so the explicit waitFor replaces the one that
|
|
171
|
+
// locator().fill() performed implicitly.
|
|
172
|
+
await page.locator(passwordSelector).waitFor({ state: 'visible', timeout: 30_000 });
|
|
173
|
+
await page.evaluate(
|
|
174
|
+
([selector, pwd]) => {
|
|
175
|
+
const input = document.querySelector(selector) as HTMLInputElement | null;
|
|
176
|
+
if (!input) {
|
|
177
|
+
throw new Error(`[qa-framework] Password input not found for selector: ${selector}`);
|
|
178
|
+
}
|
|
179
|
+
input.value = pwd;
|
|
180
|
+
input.dispatchEvent(new Event('input', { bubbles: true }));
|
|
181
|
+
input.dispatchEvent(new Event('change', { bubbles: true }));
|
|
182
|
+
},
|
|
183
|
+
[passwordSelector, password] as const
|
|
184
|
+
);
|
|
137
185
|
await page.locator(submitSelector).click();
|
|
138
186
|
|
|
139
187
|
const result = await Promise.race([
|
|
@@ -167,30 +215,51 @@ async function loginAs(params: {
|
|
|
167
215
|
}
|
|
168
216
|
|
|
169
217
|
export default async function globalSetup(_config: FullConfig): Promise<void> {
|
|
170
|
-
const baseURL
|
|
171
|
-
const authUsers = resolveAuthUsers(_config);
|
|
218
|
+
const baseURL = process.env.QA_BASE_URL!;
|
|
172
219
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
220
|
+
// Guards run BEFORE any login. A lock nothing consults is decoration, and this is
|
|
221
|
+
// the last point at which a run about to drive an account it does not hold can
|
|
222
|
+
// still be stopped.
|
|
223
|
+
//
|
|
224
|
+
// global-setup NEVER acquires or reserves a lane itself. It only READS the lock
|
|
225
|
+
// state and verifies the lane is already held. Reserving is the caller's job - the
|
|
226
|
+
// operator, the orchestrator, or the CI job. A global-setup that reserved its own
|
|
227
|
+
// lane would make every accidental run look legitimate.
|
|
228
|
+
guards.assertColdSetupIsIntentional(process.env);
|
|
229
|
+
|
|
230
|
+
const laneOnly = process.env.QA_LANE_ONLY;
|
|
231
|
+
if (laneOnly) {
|
|
232
|
+
guards.assertLaneIsLocked(laneOnly, process.env, laneLock);
|
|
233
|
+
guards.assertProjectsMatchLane(laneOnly, _config.projects, process.argv, laneLock);
|
|
183
234
|
}
|
|
184
235
|
|
|
185
|
-
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
|
|
236
|
+
for (const laneId of resolveAuthLanes(_config)) {
|
|
237
|
+
const lane = LANE_INFO[laneId];
|
|
238
|
+
|
|
239
|
+
// Credentials come from the lane's declared account prefix, e.g. account
|
|
240
|
+
// 'QA_USER4' resolves QA_USER4_EMAIL / QA_USER4_PASSWORD. This is what replaces
|
|
241
|
+
// the old hardcoded QA_USER / QA_USER2 pair and is what lets a lane be added by
|
|
242
|
+
// config alone.
|
|
243
|
+
const emailKey = `${lane.account}_EMAIL`;
|
|
244
|
+
const passwordKey = `${lane.account}_PASSWORD`;
|
|
245
|
+
|
|
246
|
+
if (!process.env[emailKey] || !process.env[passwordKey]) {
|
|
247
|
+
throw new Error(
|
|
248
|
+
`[global-setup] Lane ${laneId} (account ${lane.account}) is missing ${emailKey} or ` +
|
|
249
|
+
`${passwordKey}. Every configured lane needs its own credentials, otherwise two lanes ` +
|
|
250
|
+
'end up sharing an account, which is exactly what lanes exist to prevent.'
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const [email, emailSource] = resolveVar(process.env[emailKey], emailKey, 'qa-user@example.com');
|
|
255
|
+
const [password, passwordSource] = resolveVar(process.env[passwordKey], passwordKey, 'CHANGE_ME');
|
|
256
|
+
console.log(`[global-setup] lane ${laneId} (${lane.account}) credentials from ${emailSource} / ${passwordSource}`);
|
|
257
|
+
|
|
189
258
|
await loginAs({
|
|
190
259
|
email,
|
|
191
260
|
password,
|
|
192
261
|
baseURL,
|
|
193
|
-
stateFile:
|
|
262
|
+
stateFile: lane.storageState,
|
|
194
263
|
});
|
|
195
264
|
}
|
|
196
265
|
}
|