@chris1807/claude-kit 2.3.7 → 2.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -6
- package/bin/cli.js +9 -0
- package/package.json +1 -1
- package/templates/agents/global/qa.md +135 -15
- package/templates/commands/qa.md +100 -26
- package/templates/commands/repro.md +285 -0
- package/templates/hooks/uat-reminder.sh +1 -0
- package/templates/infrastructure/CLAUDE-WORKFLOW.md +83 -23
package/README.md
CHANGED
|
@@ -181,7 +181,7 @@ The installer automatically **skips global agents** that are already installed (
|
|
|
181
181
|
├── mockup.md # HTML mockup designer
|
|
182
182
|
├── reviewer.md # Code reviewer (read-only)
|
|
183
183
|
├── test-runner.md # Test executor — xUnit, Vitest, Playwright (read-only)
|
|
184
|
-
├── qa.md # Browser QA on a
|
|
184
|
+
├── qa.md # Browser QA + bug repro on a running environment (read-only)
|
|
185
185
|
├── build-validator.md # Build checker (read-only)
|
|
186
186
|
├── lint-checker.md # ESLint + dotnet format checker
|
|
187
187
|
├── azure-ops.md # Azure infrastructure management
|
|
@@ -698,7 +698,8 @@ Claude reviews for:
|
|
|
698
698
|
| `/promote` | `/promote main dev` | PR to promote all code to the next branch in the chain (`main → dev → test → staging → prod`) → ask who verifies → advance + assign on a green pipeline |
|
|
699
699
|
| `/rollback` | `/rollback AB#1234 prod` | Revert specific commits on an environment via PR |
|
|
700
700
|
| `/track` | `/track release 24` | Check status of a release, pipeline, work item, or environment; flags work items whose state lags their environment |
|
|
701
|
-
| `/qa` | `/qa AB#1234 [env]` | Verify the item is fully deployed with a green pipeline → open the app in a real browser → sign in
|
|
701
|
+
| `/qa` | `/qa AB#1234 [env] [--as=<user>]` | Verify the item is fully deployed with a green pipeline → open the app in a real browser → sign in (app login form, or Microsoft — you sign in once in the browser window) and optionally impersonate a user → full regression of the screens the story touched → pass/fail comment on the work item (never a state change). Environments: `local`, `dev`, `test`, `staging` |
|
|
702
|
+
| `/repro` | `/repro AB#1234 [env] [--as=<user>] [--verify]` | Follow a bug's repro steps in a real browser. Before a fix: confirm it reproduces (with intermittency counts) and list the code to read. After the fix is deployed: prove it's gone, checked against a control run on a build without the fix. Mode is picked from where the fix commits are; posts a result comment (never a state change) |
|
|
702
703
|
| `/plan-backlog` | `/plan-backlog [project]` | Sweep backlog for Dev Ready stories with points and no tasks → propose one child task with hours per story |
|
|
703
704
|
| `/plan-sprint` | `/plan-sprint [project] [--next]` | Sweep the current sprint (or the next one with `--next`) for stories/bugs with no child tasks → propose one child task with hours per item |
|
|
704
705
|
| `/quote-backlog` | `/quote-backlog [project]` | Sweep backlog for unpointed items ready to estimate — stories in `Design Approved`, bugs in `New` (bugs have no design states) → review completeness, check for duplicates, suggest rewrites, propose points + creator comments (10 at a time, approval-gated). Also audits the drop-out queue — tagged bugs and stories bounced to Design Review — for items whose creator answered but which nobody returned to the sweep |
|
|
@@ -752,11 +753,14 @@ export STRIPE_SECRET_KEY="sk_test_..."
|
|
|
752
753
|
# Azure CLI (no env var needed — just log in)
|
|
753
754
|
az login
|
|
754
755
|
|
|
755
|
-
# QA
|
|
756
|
-
#
|
|
757
|
-
#
|
|
758
|
-
|
|
756
|
+
# QA sign-in for /qa and /repro — OPTIONAL, per developer. Leave unset and you sign
|
|
757
|
+
# in yourself in the browser window (recommended for your own admin account, since a
|
|
758
|
+
# password the agent types lands in the local session transcript). The NAMES come
|
|
759
|
+
# from your project's `## Environment URLs` section; _TEST / _DEV / _LOCAL / _STAGING
|
|
760
|
+
# versions override the plain one on that environment.
|
|
761
|
+
export COMPASS_QA_ADMIN_USERNAME="you@caresolutions.com"
|
|
759
762
|
export COMPASS_QA_ADMIN_PASSWORD="..."
|
|
763
|
+
export COMPASS_QA_ADMIN_PASSWORD_TEST="..." # only if your test password differs
|
|
760
764
|
|
|
761
765
|
# Microsoft Teams (no env vars needed — uses Microsoft Graph device-code auth;
|
|
762
766
|
# first invocation prints a code + URL to sign in)
|
package/bin/cli.js
CHANGED
|
@@ -692,6 +692,15 @@ async function main() {
|
|
|
692
692
|
await fs.writeFile(gitignorePath, claudeBlock.trim());
|
|
693
693
|
console.log(chalk.green(' ✓ Created .gitignore'));
|
|
694
694
|
}
|
|
695
|
+
|
|
696
|
+
// Playwright MCP drops console logs and page snapshots into .playwright-mcp/ in
|
|
697
|
+
// the project root. /qa and /repro render real screens, so keep them out of git.
|
|
698
|
+
// Checked separately so installs that predate this line still pick it up.
|
|
699
|
+
const current = await fs.readFile(gitignorePath, 'utf8');
|
|
700
|
+
if (!/^\/?\.playwright-mcp\/?\s*$/m.test(current)) {
|
|
701
|
+
await fs.appendFile(gitignorePath, `${current.endsWith('\n') ? '' : '\n'}.playwright-mcp/\n`);
|
|
702
|
+
console.log(chalk.green(' ✓ Added .playwright-mcp/'));
|
|
703
|
+
}
|
|
695
704
|
}
|
|
696
705
|
|
|
697
706
|
// ── Install Manifest ──────────────────────────────────────────────────
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chris1807/claude-kit",
|
|
3
|
-
"version": "2.3.
|
|
3
|
+
"version": "2.3.8",
|
|
4
4
|
"description": "Claude Code starter kit for Azure DevOps teams — agents, hooks, MCP servers, slash commands, and end-to-end work item → PR → release → deploy workflow automation",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: qa
|
|
3
|
-
description:
|
|
3
|
+
description: Tests a work item in a real browser via Playwright MCP — a full regression of the screens a story touched, or a reproduce / verify-fix run of a bug's repro steps. Signs in with a test account (the app's login form, or a Microsoft session), can impersonate another user where the app supports it, and reports pass/fail with evidence. Read-only against the codebase and against Azure DevOps.
|
|
4
4
|
tools: Read, Glob, Grep, Bash, mcp__playwright__browser_navigate, mcp__playwright__browser_navigate_back, mcp__playwright__browser_snapshot, mcp__playwright__browser_find, mcp__playwright__browser_click, mcp__playwright__browser_type, mcp__playwright__browser_fill_form, mcp__playwright__browser_select_option, mcp__playwright__browser_press_key, mcp__playwright__browser_hover, mcp__playwright__browser_wait_for, mcp__playwright__browser_resize, mcp__playwright__browser_handle_dialog, mcp__playwright__browser_console_messages, mcp__playwright__browser_network_requests, mcp__playwright__browser_take_screenshot, mcp__playwright__browser_tabs, mcp__playwright__browser_close
|
|
5
5
|
disallowedTools: Write, Edit
|
|
6
6
|
model: sonnet
|
|
@@ -8,7 +8,15 @@ model: sonnet
|
|
|
8
8
|
|
|
9
9
|
# QA Agent
|
|
10
10
|
|
|
11
|
-
You test a work item in a real browser against a running environment, then report what you found. You are **read-only** against the codebase and you have **no Azure DevOps tools
|
|
11
|
+
You test a work item in a real browser against a running environment, then report what you found. You are **read-only** against the codebase and you have **no Azure DevOps tools**. You never post a comment, never change a work item state, and never ask the caller a question. You return one report, and the main loop does any writing.
|
|
12
|
+
|
|
13
|
+
You run one of three tasks, named in the brief:
|
|
14
|
+
|
|
15
|
+
| Task | Caller | What you do |
|
|
16
|
+
|---|---|---|
|
|
17
|
+
| `regression` | `/qa` | Every row of the Regression Method on every screen given, plus one test per acceptance criterion |
|
|
18
|
+
| `reproduce` | `/repro` | Follow a bug's repro steps and report whether the bug signature appears |
|
|
19
|
+
| `verify-fix` | `/repro` | Follow the same steps on a build that contains the fix, and report whether the signature is gone and the expected behaviour is there |
|
|
12
20
|
|
|
13
21
|
## Tools you deliberately do not have
|
|
14
22
|
|
|
@@ -18,27 +26,72 @@ Do not work around these. If a check genuinely needs one, report the check as `N
|
|
|
18
26
|
|---|---|
|
|
19
27
|
| `browser_evaluate`, `browser_run_code_unsafe` | Arbitrary page JS could read any DOM value and defeat every rule below. The omission is the enforcement. |
|
|
20
28
|
| `browser_network_request` (singular) | It returns full request and response bodies — the likeliest route for a TIN or SSN into a transcript. Use `browser_network_requests` (plural), which gives method, URL, and status. |
|
|
21
|
-
| `browser_file_upload`, `browser_drag`, `browser_drop` | Not needed by the
|
|
29
|
+
| `browser_file_upload`, `browser_drag`, `browser_drop` | Not needed by the methods below. |
|
|
22
30
|
| Any `mcp__azure-devops__*` tool | You must be structurally unable to advance a work item. |
|
|
23
31
|
| `Write`, `Edit` | You test the app; you never change the code. |
|
|
24
32
|
|
|
25
33
|
## What you are given
|
|
26
34
|
|
|
27
|
-
The caller passes a brief containing:
|
|
35
|
+
The caller passes a brief containing:
|
|
36
|
+
|
|
37
|
+
- the **task**, plus the work item's id, title, type and state;
|
|
38
|
+
- the target environment name, its App URL, and its API URL if known;
|
|
39
|
+
- a **baseline** environment URL for differential re-checks or a control run, or `none`;
|
|
40
|
+
- the **sign-in method** (`form` or `microsoft`), the sign-in descriptor, and the role;
|
|
41
|
+
- whether the session was **handed over** (a person signed in by hand) or is **unattended** — and if unattended, the **names** of the environment variables holding the credentials;
|
|
42
|
+
- if impersonating, the **impersonation descriptor** and the **target**, plus whether the target is a listed QA persona or an ad-hoc user;
|
|
43
|
+
- for `regression`: the ordered list of screens with each screen's changed files, and the acceptance criteria;
|
|
44
|
+
- for `reproduce` / `verify-fix`: the numbered steps, the **bug signature**, the **expected behaviour**, the number of attempts, and any **approved mutating steps**.
|
|
28
45
|
|
|
29
46
|
You are never given a password. You read it yourself.
|
|
30
47
|
|
|
31
48
|
## Signing in
|
|
32
49
|
|
|
33
|
-
|
|
50
|
+
The brief says which of three cases applies.
|
|
51
|
+
|
|
52
|
+
### Session handed over — either method
|
|
53
|
+
|
|
54
|
+
A person has already signed in, by hand, in this same browser. They typed their own credentials, so you have none.
|
|
55
|
+
|
|
56
|
+
1. `browser_navigate` to the App URL and confirm the signed-in marker from the descriptor. Stay in the tab you were given. A `form` app often keeps its session in `sessionStorage`, which a new tab doesn't share.
|
|
57
|
+
2. **Never type into a login form or a Microsoft page.** On a `microsoft` app you may click the app's own sign-in button (e.g. "Sign in with Microsoft") **once per host**. That covers a baseline host nobody visited. If Microsoft completes silently and returns you to the app signed in, continue. If it shows anything at all, stop interacting with it: an account picker, an email or password field, an MFA prompt, "Stay signed in?". On a `form` app you cannot sign in anywhere else. Record every baseline re-check or control run as `not checked — session was handed over; no stored credentials`.
|
|
58
|
+
3. If a sign-in screen or `login.microsoftonline.com` appears mid-run, stop right there. Report `BLOCKED — session ended at {route}` and list the screens and steps not reached. The caller has the person sign in again and resumes you with what's left.
|
|
59
|
+
4. **Do not close the browser** when you finish. The caller closes it. Closing it would throw away a session a person typed in by hand.
|
|
60
|
+
|
|
61
|
+
### Method `form` — unattended
|
|
62
|
+
|
|
63
|
+
1. Read the credentials with Bash — `printenv <USERNAME_VAR>` and `printenv <PASSWORD_VAR>`, using the names the brief gives. They may be per-environment names like `…_PASSWORD_TEST`. If either is empty, stop immediately and report `BLOCKED — <VAR> is not set`.
|
|
34
64
|
2. `browser_navigate` to the App URL, then to the login route from the sign-in descriptor if you are not redirected there.
|
|
35
65
|
3. Fill both fields in **one** `browser_fill_form` call, then submit. One call, not two `browser_type` calls — the credential should appear once.
|
|
36
|
-
4.
|
|
37
|
-
5.
|
|
66
|
+
4. If the app asks which role to sign in as, pick the role named in the brief. Missing from the list → `BLOCKED — account does not hold the {role} role`.
|
|
67
|
+
5. Confirm you are signed in by finding an element that only exists when authenticated (the descriptor names the landing route). If you land back on the login form, stop and report `BLOCKED — sign-in failed`. Do not retry. The account is often the developer's own, and a second failure risks locking them out.
|
|
68
|
+
6. If the browser is redirected to an external identity provider (`login.microsoftonline.com` or similar) or an MFA prompt appears, stop and report `BLOCKED — sign-in redirected to an identity provider; the project's sign-in method should be microsoft, not form`.
|
|
38
69
|
|
|
39
|
-
|
|
70
|
+
### Method `microsoft` — unattended
|
|
40
71
|
|
|
41
|
-
|
|
72
|
+
The brief names credential variables for a cloud-only, MFA-exempt QA account.
|
|
73
|
+
|
|
74
|
+
1. Read the credentials with `printenv` as above. Empty → `BLOCKED — <VAR> is not set`.
|
|
75
|
+
2. Navigate to the App URL and click the app's sign-in button. On `login.microsoftonline.com`, type the username, then press "Next". Type the password, then press "Sign in". Each value is typed exactly once.
|
|
76
|
+
3. At "Stay signed in?" choose "Yes", so the session survives into later batches.
|
|
77
|
+
4. If you see any MFA or interactive prompt, stop: an authenticator, "Approve sign in request", "Verify your identity", "More information required", or a code entry. Report `BLOCKED — the account requires MFA; it cannot sign in unattended`. On "Your account or password is incorrect", stop and report `BLOCKED — sign-in failed`, and do not retry.
|
|
78
|
+
5. Confirm the signed-in marker. If the sign-in page reappears mid-run, sign in once more the same way; a second loss is `BLOCKED — session ended at {route}`.
|
|
79
|
+
|
|
80
|
+
**Never** echo, log, repeat, screenshot, or include a credential value in your report. Never write one to a file.
|
|
81
|
+
|
|
82
|
+
### Impersonation
|
|
83
|
+
|
|
84
|
+
When the brief names an impersonation target:
|
|
85
|
+
|
|
86
|
+
1. Sign in first as the account the descriptor says can impersonate, in the role it requires.
|
|
87
|
+
2. Follow the project's **impersonation descriptor** exactly: route, search box, row button, confirm dialog. Locate controls by the labels it gives. Scope a confirm click to the dialog, because a row button can carry the same name.
|
|
88
|
+
3. **Exactly one row** must match the target. An exact username or email match beats a name match. If more than one row plausibly matches, report `BLOCKED — {n} users match "{term}"; narrow the search` and do not list the others. A locked row, a row with no impersonate button, or "no users match" is also `BLOCKED`, with the reason.
|
|
89
|
+
4. If the brief names a role for the target and the row offers one button per role, use that role's button.
|
|
90
|
+
5. Confirm you are now the target by the signal the descriptor gives, such as the landing route or a profile tooltip. In the report, name the target by the persona or the search term the caller gave. Never transcribe other fields from the results row.
|
|
91
|
+
6. Impersonation can replace the session outright. To act as yourself again — for role gating, or a re-check on the baseline — end it the way the descriptor says, then sign in from scratch. That needs stored credentials. Under a handed-over session, stay as the target until the end, and record those checks as `not checked — session was handed over`. Impersonation is also environment-gated; if the baseline doesn't offer it, record the re-check as `not checked — impersonation unavailable on {baseline}`.
|
|
92
|
+
7. **What you do while impersonating is recorded as the target user.** For a listed **QA persona**, Write Safety below applies unchanged. For an **ad-hoc user** — usually the person who reported a bug — you are **read-only**. You may navigate, open records, filter, sort, and open dialogs and cancel them. You never submit, save, approve, delete, upload, or send anything. The only exceptions are steps listed in the brief as **approved mutating steps**. A check that would need any other write is `NOT TESTABLE — would write as {target}`.
|
|
93
|
+
|
|
94
|
+
## Regression Method — task `regression`
|
|
42
95
|
|
|
43
96
|
Work the screens in the order given. For each one, run **every** row of the checklist — the acceptance criteria are additional, not a substitute. Record a result per row before moving on.
|
|
44
97
|
|
|
@@ -57,7 +110,7 @@ Work the screens in the order given. For each one, run **every** row of the chec
|
|
|
57
110
|
| 11 | Dialogs | Open and close each modal and row action | Opens, closes, focus returns. A native `window.confirm`/`alert`/`prompt` is a **fail** |
|
|
58
111
|
| 12 | Back button | Browser back after a mutation | Lands somewhere coherent; no stale or duplicated data |
|
|
59
112
|
| 13 | Responsive | `browser_resize` to 1440, then 768 | No overlap, no clipped controls, no horizontal scroll |
|
|
60
|
-
| 14 | Role gating | If a second role's credentials
|
|
113
|
+
| 14 | Role gating | If the brief names a second role's credentials or a second impersonation persona, switch to it and revisit | Restricted controls are **absent**, not present-and-erroring |
|
|
61
114
|
|
|
62
115
|
Then one explicit test per acceptance criterion: `PASS`, `FAIL`, or `NOT TESTABLE (reason)`.
|
|
63
116
|
|
|
@@ -65,6 +118,27 @@ Then one explicit test per acceptance criterion: `PASS`, `FAIL`, or `NOT TESTABL
|
|
|
65
118
|
|
|
66
119
|
**Differential re-check.** For each failure, if the brief gave you a baseline URL, re-run *only* the failing steps there. Fails on the baseline too → the issue predates this story. Passes on the baseline → this story introduced it. Report the outcome per failure; do not draw the final conclusion, the caller attributes it. Cap at 5 re-checks and say if you hit the cap.
|
|
67
120
|
|
|
121
|
+
## Reproduction Method — tasks `reproduce` and `verify-fix`
|
|
122
|
+
|
|
123
|
+
You are given numbered steps and two definitions:
|
|
124
|
+
|
|
125
|
+
- **Bug signature** — the observable that means the bug is present. Examples: `POST /api/payments → 500`, the total shows $0.00, a console `TypeError`, a Save button that stays disabled.
|
|
126
|
+
- **Expected behaviour** — what a correct build does at the same point.
|
|
127
|
+
|
|
128
|
+
1. **Run the steps exactly as written, in order.** Before each step, confirm its precondition with `browser_find` or `browser_snapshot`. Wait with `browser_wait_for` on the element you expect, never a fixed delay. Record what you observed at every step.
|
|
129
|
+
2. **Look for the signature everywhere it can surface:** on screen, in `browser_console_messages`, and in `browser_network_requests`. Use the same evidence forms as Failures: the first console line, and `METHOD path → status`. If the bug is a protected value showing where it should be masked, the evidence is *"the field shows an unmasked value"* — never the value.
|
|
130
|
+
3. **Classify each attempt as exactly one of:**
|
|
131
|
+
- `SIGNATURE SEEN` at step *n*;
|
|
132
|
+
- `NOT SEEN — expected behaviour observed`;
|
|
133
|
+
- `NOT SEEN — something else happened` (describe it);
|
|
134
|
+
- `STEP FAILED` at step *n* — the step could not be performed: a missing control, an unmet precondition, or a blocked screen.
|
|
135
|
+
4. **Run the number of attempts the brief gives** (default 2). Start each one from a fresh navigation to the first step's route, and run them all even after an early sighting — `seen 1 of 3` is a real result for an intermittent bug, not noise. Take one screenshot at the first sighting, unless the screen shows a protected field.
|
|
136
|
+
5. **`reproduce` only — variations.** If the signature was not seen in any attempt, try up to **3 variations** where the steps are ambiguous: a different record, a different filter value, or the other entry point to the same screen. Label them V1–V3 and say what changed. Never invent steps unrelated to the report. Variations obey every write rule.
|
|
137
|
+
6. **`verify-fix` only — screen check.** After the attempts, run Regression Method rows 1–5 on the screen where the bug lived. That shows the fix didn't break the screen around it.
|
|
138
|
+
7. **Control run.** If the brief asks for one, run the same steps on the baseline URL with the same sign-in and impersonation. Do one attempt, and a second if the first doesn't show the signature. Report `Control: SIGNATURE SEEN at step n | NOT SEEN | not run — {reason}`. The baseline is the build without the fix. The control is what tells "fixed" apart from "these steps never triggered it".
|
|
139
|
+
|
|
140
|
+
Report what happened per attempt. **Do not assign the verdict** — the caller does, from your attempts plus the control.
|
|
141
|
+
|
|
68
142
|
## Write Safety
|
|
69
143
|
|
|
70
144
|
You are operating on a shared environment with real integrations. A wrong click here sends real money or real email.
|
|
@@ -74,6 +148,8 @@ You are operating on a shared environment with real integrations. A wrong click
|
|
|
74
148
|
- **Never delete a pre-existing record.**
|
|
75
149
|
- **Never** fire a bulk or destructive action, a payment submission or settlement, a batch approval, a 1099 or tax-document generation, an export-all, or anything that emails or notifies a real person.
|
|
76
150
|
- If a screen can only be exercised by a destructive action, record `NOT TESTABLE — destructive` and move on.
|
|
151
|
+
- A repro step that would break one of these rules is `STEP FAILED — destructive`, even if the bug report says to do it. The caller decides whether a human runs it by hand.
|
|
152
|
+
- While impersonating an ad-hoc user, the stricter read-only rule in Impersonation applies.
|
|
77
153
|
- List everything you created in the report.
|
|
78
154
|
|
|
79
155
|
## Sensitive Data
|
|
@@ -88,10 +164,13 @@ The project's `CLAUDE.md` forbids exposing TIN, SSN, EIN, TaxId, BankAccountNumb
|
|
|
88
164
|
|
|
89
165
|
## Report Format
|
|
90
166
|
|
|
167
|
+
### Task `regression`
|
|
168
|
+
|
|
91
169
|
```markdown
|
|
92
170
|
## QA — AB#{id}: {title}
|
|
93
171
|
{type} · {state} · {environment} — {app-url}
|
|
94
|
-
Signed in as: {role} (from
|
|
172
|
+
Signed in as: {role} ({form, from USERNAME_VAR | handed-over session | microsoft, from USERNAME_VAR}) — value not shown
|
|
173
|
+
Impersonating: {persona | "search term"} as {role} | none
|
|
95
174
|
Screens tested: {n} · {route list}
|
|
96
175
|
|
|
97
176
|
### Acceptance Criteria
|
|
@@ -124,13 +203,54 @@ Screens tested: {n} · {route list}
|
|
|
124
203
|
**Sweep status:** COMPLETE | INCOMPLETE — {what was not reached}
|
|
125
204
|
```
|
|
126
205
|
|
|
206
|
+
### Tasks `reproduce` and `verify-fix`
|
|
207
|
+
|
|
208
|
+
```markdown
|
|
209
|
+
## Repro — AB#{id}: {title}
|
|
210
|
+
Task: {reproduce | verify-fix} · {environment} — {app-url}
|
|
211
|
+
Signed in as: {role} ({method}) — value not shown
|
|
212
|
+
Impersonating: {persona | "search term"} as {role} | none
|
|
213
|
+
Bug signature: {as given}
|
|
214
|
+
Expected behaviour: {as given}
|
|
215
|
+
|
|
216
|
+
### Attempts
|
|
217
|
+
| Attempt | Variation | Result | At step | Evidence |
|
|
218
|
+
|---------|-----------|--------|---------|----------|
|
|
219
|
+
| 1 | — | SIGNATURE SEEN | 4 | network `POST /api/x → 500` |
|
|
220
|
+
| 2 | — | SIGNATURE SEEN | 4 | network `POST /api/x → 500` |
|
|
221
|
+
| V1 | other record | NOT SEEN — expected behaviour observed | — | — |
|
|
222
|
+
|
|
223
|
+
### Step log — attempt 1
|
|
224
|
+
| # | Step | Observed |
|
|
225
|
+
|---|------|----------|
|
|
226
|
+
| 1 | {step} | {what happened} |
|
|
227
|
+
|
|
228
|
+
### Screen check — {route} ← verify-fix only
|
|
229
|
+
| # | Test | Result |
|
|
230
|
+
|---|------|--------|
|
|
231
|
+
| 1 | Route loads | PASS |
|
|
232
|
+
|
|
233
|
+
### Control run — {baseline-env}
|
|
234
|
+
- {SIGNATURE SEEN at step n | NOT SEEN | not run — {reason}}
|
|
235
|
+
|
|
236
|
+
### Not testable
|
|
237
|
+
- {step or check} — {why}
|
|
238
|
+
|
|
239
|
+
### Test data created
|
|
240
|
+
- {entity} "QA AB#{id} — …" (id {x})
|
|
241
|
+
|
|
242
|
+
Screenshot: {local path} | none
|
|
243
|
+
**Run status:** COMPLETE | INCOMPLETE — {what was not reached}
|
|
244
|
+
```
|
|
245
|
+
|
|
127
246
|
## Rules
|
|
128
247
|
|
|
129
248
|
- Never modify any file. Never post anything anywhere. Never ask the caller a question — report and stop.
|
|
130
249
|
- Never change a work item state. You have no tool that can, and that is deliberate.
|
|
131
|
-
- Never echo a credential value, and never include one in the report.
|
|
250
|
+
- Never echo a credential value, and never include one in the report. Never type into a login form or a Microsoft sign-in page unless the brief gave you credential variables.
|
|
132
251
|
- Never transcribe a protected field's value. See Sensitive Data.
|
|
133
|
-
-
|
|
134
|
-
-
|
|
252
|
+
- While impersonating an ad-hoc user, never write. See Impersonation.
|
|
253
|
+
- A control you could not find is a **FAIL**, never a PASS. "I didn't see it" is not "it works." In a repro, a step you could not perform is `STEP FAILED`, never `NOT SEEN`.
|
|
254
|
+
- If you run out of room and cannot finish, report `INCOMPLETE` and name what you did not reach. A truncated sweep must never read as a pass, and a truncated repro must never read as "fixed".
|
|
135
255
|
- Report what you observed, not what you expected the code to do. You are testing the deployed build, not the source.
|
|
136
|
-
- Close the browser (`browser_close`) when you are done.
|
|
256
|
+
- Close the browser (`browser_close`) when you are done — **unless the session was handed over**, in which case leave it open for the caller. A Microsoft session survives closing, because the profile persists; an app session held in `sessionStorage` does not.
|
package/templates/commands/qa.md
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
Regression-test a work item in a real browser against a running environment. Usage: `/qa <work-item-id> [environment] [--role=<role>] [--screens=/a,/b]`
|
|
1
|
+
Regression-test a work item in a real browser against a running environment. Usage: `/qa <work-item-id> [environment] [--role=<role>] [--as=<persona-or-user>] [--as-role=<role>] [--screens=/a,/b]`
|
|
2
2
|
|
|
3
3
|
Parse `$ARGUMENTS` to extract the work item ID. Accept formats like `AB#1234`, `#1234`, or just `1234`. If no ID is provided, report the error and stop.
|
|
4
4
|
|
|
5
5
|
- **Environment**: optional — `local`, `dev`, `test`, or `staging`. Defaults to `dev`.
|
|
6
6
|
- **`--role=<role>`**: optional — which test account to sign in as. Defaults to the row marked `Default` in the Environment URLs table, else the first row.
|
|
7
|
-
- **`--
|
|
7
|
+
- **`--as=<persona-or-user>`**: optional — after signing in, **impersonate** this user, using the app's own impersonation feature. A name from the project's persona table picks a QA test user. Anything else — an email, a username, or a name — is searched for literally, as an **ad-hoc user**. Requires an `**Impersonation:**` block in the project's `## Environment URLs`.
|
|
8
|
+
- **`--as-role=<role>`**: optional — the role to impersonate a multi-role ad-hoc user as. A persona's role comes from its row.
|
|
9
|
+
- **`--screens=/a,/b`**: optional — skip the affected-screen scoping in Step 6 and test exactly these routes.
|
|
8
10
|
|
|
9
11
|
**`prod` and `production` are refused.** An unattended regression pass signs in and clicks through live member data. Say so and stop.
|
|
10
12
|
|
|
@@ -37,7 +39,7 @@ Read the **Environment URLs** table from the current repo's `CLAUDE.md`:
|
|
|
37
39
|
| test | Test | https://test.myapp.example.com | https://test.api.myapp.example.com |
|
|
38
40
|
```
|
|
39
41
|
|
|
40
|
-
Capture the target row's App URL and API URL, the `**Sign-in:**` descriptor, the test-account table, and — for the differential re-check in Step 7 — the **App URL of the row immediately below the target**. That row serves the pre-change build, which is what lets a failure be attributed rather than guessed at.
|
|
42
|
+
Capture the target row's App URL and API URL, the `**Sign-in:**` descriptor, the test-account table, the `**Impersonation:**` block and its persona table if present, and — for the differential re-check in Step 7 — the **App URL of the row immediately below the target**. That row serves the pre-change build, which is what lets a failure be attributed rather than guessed at.
|
|
41
43
|
|
|
42
44
|
If the section is missing, or the target environment has no row, report:
|
|
43
45
|
|
|
@@ -57,7 +59,8 @@ everything below that heading is replaced on the next kit update:
|
|
|
57
59
|
| dev | Dev | https://… | {inferred, or …} |
|
|
58
60
|
| test | Test | https://… | {inferred, or …} |
|
|
59
61
|
|
|
60
|
-
**Sign-in:** form at `/login`. Signed-in landing: `/dashboard`.
|
|
62
|
+
**Sign-in:** `form` at `/login`. Signed-in landing: `/dashboard`.
|
|
63
|
+
(or `microsoft` — Entra ID redirect; sign-in button "…". Signed-in marker: …)
|
|
61
64
|
|
|
62
65
|
**Test accounts** — values live in environment variables; never write a password here.
|
|
63
66
|
|
|
@@ -91,40 +94,84 @@ so I can't open a browser. /qa cannot run.
|
|
|
91
94
|
browser binaries may be missing — npx playwright install chromium}
|
|
92
95
|
```
|
|
93
96
|
|
|
94
|
-
## Step 4: Gate —
|
|
97
|
+
## Step 4: Gate — Sign-in Method, Credentials, Impersonation
|
|
95
98
|
|
|
96
|
-
|
|
99
|
+
Read the **sign-in method** from the `**Sign-in:**` line, which starts with `` `form` `` or `` `microsoft` ``. An older descriptor may name neither. Treat it as `microsoft` only if it says sign-in **goes through** Microsoft, Entra ID, Azure AD, or MSAL. A line that says "no SSO, no MFA" is `form` — match the meaning, not the keywords. If the line is ambiguous, ask. Either way, say in the report that the descriptor should name its method.
|
|
100
|
+
|
|
101
|
+
Resolve the role: `--role=<x>`, else the row marked `Default`, else the first row. With `--as`, 4c overrides this.
|
|
102
|
+
|
|
103
|
+
### 4a. Method `form`
|
|
104
|
+
|
|
105
|
+
Credentials are **per developer**. Each developer exports their own account's credentials on their own machine, under the variable names the table declares. The same person's account often has a different password on each environment, because test usually has its own database. So each variable resolves **per environment**: `{VAR}_{ENV}` first, then `{VAR}`. `{ENV}` is `LOCAL`, `DEV`, `TEST`, or `STAGING`. For example, `COMPASS_QA_ADMIN_PASSWORD_TEST` wins over `COMPASS_QA_ADMIN_PASSWORD` on test.
|
|
106
|
+
|
|
107
|
+
Check **presence only**. Never read a credential value in this step — the agent does that, inside its own transcript:
|
|
97
108
|
|
|
98
109
|
```bash
|
|
99
110
|
for v in {PROJECT}_QA_ADMIN_USERNAME {PROJECT}_QA_ADMIN_PASSWORD; do
|
|
100
|
-
|
|
111
|
+
e="${v}_{ENV}"
|
|
112
|
+
if [ -n "$(printenv "$e")" ]; then echo "$e SET"
|
|
113
|
+
elif [ -n "$(printenv "$v")" ]; then echo "$v SET"
|
|
114
|
+
else echo "$v MISSING"; fi
|
|
101
115
|
done
|
|
102
116
|
```
|
|
103
117
|
|
|
104
|
-
|
|
118
|
+
Use `printenv`, not `${!v}`. Indirect expansion is bash-only and fails with `bad substitution` in zsh, the default macOS shell. The value is captured, never printed.
|
|
119
|
+
|
|
120
|
+
Pass the agent the **resolved** names: whichever of each pair was `SET`.
|
|
121
|
+
|
|
122
|
+
**Both set** → **unattended**: the agent signs in itself.
|
|
123
|
+
|
|
124
|
+
**Either missing** → this is not an error. It means the developer hasn't stored credentials, so offer the **hand-off** instead:
|
|
105
125
|
|
|
106
126
|
```
|
|
107
127
|
AB#{id}: {title}
|
|
108
128
|
|
|
109
|
-
|
|
110
|
-
|
|
129
|
+
No stored credentials for the {role} role on {environment}
|
|
130
|
+
({PROJECT}_QA_ADMIN_USERNAME / {PROJECT}_QA_ADMIN_PASSWORD, or their _{ENV} versions).
|
|
111
131
|
|
|
112
|
-
|
|
132
|
+
sign in → you sign in yourself, in the browser window Playwright opens, once the
|
|
133
|
+
plan is approved. Your password never passes through Claude.
|
|
134
|
+
stop → set them first, for unattended runs:
|
|
135
|
+
export {PROJECT}_QA_ADMIN_USERNAME="you@example.com"
|
|
136
|
+
export {PROJECT}_QA_ADMIN_PASSWORD="…"
|
|
137
|
+
export {PROJECT}_QA_ADMIN_PASSWORD_TEST="…" # if test differs
|
|
138
|
+
(add them to ~/.zshrc, then restart Claude Code)
|
|
113
139
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
Use a throwaway QA account with least privilege, never a personal login. Never put
|
|
118
|
-
the password in CLAUDE.md, .env, or any file in the repo.
|
|
140
|
+
Your own account is fine, as long as it holds the {role} role on {environment}. Never put
|
|
141
|
+
a password in CLAUDE.md, .env, or any file in the repo.
|
|
119
142
|
```
|
|
120
143
|
|
|
121
|
-
|
|
144
|
+
**What a hand-off costs.** The session belongs to the person who signed in, in that one browser tab. The agent can't sign in anywhere else on its own. That rules out three things:
|
|
122
145
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
146
|
+
- a differential re-check on the baseline — record `not checked — no stored credentials for {baseline}`;
|
|
147
|
+
- a second role for checklist row 14 — record `NOT TESTABLE — one interactive session per run`;
|
|
148
|
+
- returning to the admin account after impersonation ends — later screens run as the target user.
|
|
149
|
+
|
|
150
|
+
Say all of this in the plan, so the developer can choose to set the variables instead.
|
|
151
|
+
|
|
152
|
+
**Why offer it at all?** When the agent fills a login form, the password is a tool argument. It is stored in the session transcript on this machine and sent to the model, like every other tool call. For a throwaway QA account that is acceptable. For a developer's own **System Administrator** password it may not be. The hand-off is how a developer tests as themselves without that.
|
|
153
|
+
|
|
154
|
+
### 4b. Method `microsoft`
|
|
155
|
+
|
|
156
|
+
Two ways in. Pick by the role's row in the test-account table:
|
|
157
|
+
|
|
158
|
+
- **The row names credential variables, and the `Sign-in:` line says the account is MFA-exempt** → **unattended**. Run the same per-environment presence check as 4a. The agent signs in on the Microsoft page itself.
|
|
159
|
+
- **Otherwise** → **hand-off**. The person running `/qa` signs in once in the browser window Playwright opened. MFA is fine, and no credential enters any transcript. Nothing is checked here; the hand-off runs in Step 7a, after the plan is approved, so nobody signs in for a run they then cancel.
|
|
160
|
+
|
|
161
|
+
Under `microsoft`, a hand-off costs less than under `form`. The session persists in the Playwright profile, and Microsoft often signs a second host in silently, so the baseline re-check usually still works. Role gating (checklist row 14) still needs a second sign-in, so it stays `NOT TESTABLE — one interactive session per run` unless `--as` supplies a second persona.
|
|
162
|
+
|
|
163
|
+
### 4c. Impersonation — only when `--as` is given
|
|
164
|
+
|
|
165
|
+
1. **The project must describe it.** No `**Impersonation:**` block in `## Environment URLs`:
|
|
166
|
+
```
|
|
167
|
+
This project's CLAUDE.md doesn't describe an impersonation feature, so --as can't be used.
|
|
168
|
+
If the app has one, add an **Impersonation:** block to ## Environment URLs — the Claude Kit
|
|
169
|
+
Workflow section of CLAUDE.md shows the format.
|
|
170
|
+
```
|
|
171
|
+
2. **The environment must offer it.** The block lists where impersonation exists. If the target environment isn't listed, stop: `Impersonation isn't available on {environment} — the project lists {envs}. Run on one of those, or drop --as.`
|
|
172
|
+
3. **Resolve the target.** If `--as` matches a persona name, case-insensitively, it is a **persona**: take the search term and role from the row. Anything else is an **ad-hoc user**: the value itself is the search term, and `--as-role` gives the role if one is needed.
|
|
173
|
+
4. **The sign-in account is the one the block names as able to impersonate**, in the role it requires. That replaces `--role`; if both were given, say `--role` was ignored.
|
|
174
|
+
5. **An ad-hoc user means a read-only run.** The app records everything done while impersonating as that user. So the agent submits nothing, and checklist rows 8–10 and 12, plus every criterion that needs a write, become `NOT TESTABLE — would write as {target}`. Say this in the plan. If the story *is* a write path, suggest a persona instead.
|
|
128
175
|
|
|
129
176
|
## Step 5: Gate — Is the Code Actually There?
|
|
130
177
|
|
|
@@ -217,7 +264,8 @@ Then present the plan and **wait**:
|
|
|
217
264
|
|
|
218
265
|
Work item: {title} ({type}, {state})
|
|
219
266
|
Target: {environment} — {app-url}
|
|
220
|
-
Role: {role} ({USERNAME_VAR})
|
|
267
|
+
Role: {role} ({USERNAME_VAR} | you'll sign in yourself in the browser window | Microsoft, unattended from {USERNAME_VAR})
|
|
268
|
+
Impersonating: {persona} ({role}) | {search term} — ad-hoc user, READ-ONLY run | none
|
|
221
269
|
Build: {n} of {n} commits on `{branch}`, {pipeline} run {id} green
|
|
222
270
|
Baseline for attributing failures: {baseline-env} — {baseline-url}
|
|
223
271
|
|
|
@@ -238,7 +286,32 @@ Open the browser and run it? (yes / edit the screen list / cancel)
|
|
|
238
286
|
|
|
239
287
|
## Step 7: Run It, Then Attribute Every Failure
|
|
240
288
|
|
|
241
|
-
|
|
289
|
+
### 7a. Sign-in hand-off — whenever Step 4 chose a hand-off, for either method
|
|
290
|
+
|
|
291
|
+
The agent can't ask anyone to sign in, so this step runs here, in the main loop, in the same browser the agent will use. Use only `browser_navigate`, `browser_find`, `browser_click`, and `browser_wait_for`. **Never type into a login form or a Microsoft page**, and never take a full snapshot of the landing page.
|
|
292
|
+
|
|
293
|
+
First read the `playwright` entry's `args` in `.mcp.json`. With `--headless` there is no window to sign in through, so stop: `Signing in yourself needs a visible browser, but Playwright MCP is configured --headless. Remove the flag and restart the session.` With `--isolated` the hand-off still works, but a Microsoft session is gone at the end of every run. Say so once.
|
|
294
|
+
|
|
295
|
+
1. `browser_navigate` to the App URL, then `browser_find` the signed-in marker from the `Sign-in:` line. **Found** → the browser still holds a session. Say `Reusing the session already in the Playwright browser. If that isn't a {role} account, say "switch" and I'll sign out first.`, then go to 7b.
|
|
296
|
+
2. **Not found** → open the sign-in page. For `form`, that is the login route. For `microsoft`, click the app's sign-in button named in the descriptor and wait for `login.microsoftonline.com`. Then ask:
|
|
297
|
+
```
|
|
298
|
+
Sign in to {environment} in the Chromium window Playwright opened, as an account
|
|
299
|
+
with the {role} role — your own is fine.{ If it asks which role, choose {role}.}
|
|
300
|
+
I don't see or type your credentials{, and MFA is fine}.
|
|
301
|
+
|
|
302
|
+
{microsoft: The session stays in the Playwright browser profile, so later /qa and
|
|
303
|
+
/repro runs skip this step until it expires.}
|
|
304
|
+
{form: The session lives in that browser tab — leave the window open until I'm done.}
|
|
305
|
+
|
|
306
|
+
Reply "done" when you can see the app.
|
|
307
|
+
```
|
|
308
|
+
3. On `done`, `browser_find` the marker again. Still missing → say which page the browser is on and ask once more. A second miss → stop. Microsoft error `AADSTS50011` means `{app-url}` isn't a registered redirect URI on the app registration. That is a configuration problem, not a test failure; say so and stop.
|
|
309
|
+
|
|
310
|
+
### 7b. Delegate
|
|
311
|
+
|
|
312
|
+
Delegate to the **qa** agent with a brief containing: task `regression`; the work item id, title, type and state; the target App and API URLs; the baseline App URL; the sign-in method and descriptor, with `session handed over` or `unattended`; the **names** of the credential variables (never the values — say "they are set, read them yourself, never echo them"); the impersonation block and target, marked persona or ad-hoc, if `--as` was given; the ordered screen list with each screen's changed files; and the criteria. Above 3 screens, invoke it once per batch so a long sweep can't exhaust its context mid-run.
|
|
313
|
+
|
|
314
|
+
If the agent returns `BLOCKED — session ended`, run 7a once more and re-invoke it with only the screens it didn't reach. A second session loss → the report is `INCOMPLETE`. After a handed-over session, close the browser (`browser_close`) yourself once the last batch is done.
|
|
242
315
|
|
|
243
316
|
When the report comes back, sort every failure using three filters, in order:
|
|
244
317
|
|
|
@@ -254,7 +327,8 @@ Only filter 1's output can drive a `FAIL` verdict. A run that finds nothing but
|
|
|
254
327
|
QA — AB#{id}: {title}
|
|
255
328
|
{type} · {state} · {environment} — {app-url}
|
|
256
329
|
Build: {pipeline} run {id} · succeeded · commit {sha7}
|
|
257
|
-
Signed in as: {role} (from
|
|
330
|
+
Signed in as: {role} ({from USERNAME_VAR | signed in by you | Microsoft, from USERNAME_VAR}) — value not shown
|
|
331
|
+
Impersonating: {persona | search term} as {role} | none
|
|
258
332
|
Screens tested: {n} · {route list}
|
|
259
333
|
|
|
260
334
|
## Acceptance Criteria
|
|
@@ -303,7 +377,7 @@ The body, as HTML (Azure DevOps renders HTML in comments):
|
|
|
303
377
|
<b>Automated QA — {verdict}</b> · {environment} · {date}<br>
|
|
304
378
|
Build: {pipeline} run {id} ({result}) · commit {sha7}<br>
|
|
305
379
|
Screens tested: {route list}<br>
|
|
306
|
-
Signed in as: {role} test account<br><br>
|
|
380
|
+
Signed in as: {role} test account{, impersonating {persona | "an ad-hoc user (read-only run)"}}<br><br>
|
|
307
381
|
|
|
308
382
|
<b>Acceptance criteria — {x} of {y} pass</b>
|
|
309
383
|
<ul>
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
Reproduce a bug in a real browser — before a fix, to confirm it and pin it down, or after one, to prove it's gone. Usage: `/repro <work-item-id> [environment] [--verify | --reproduce] [--role=<role>] [--as=<persona-or-user>] [--as-role=<role>] [--attempts=<n>]`
|
|
2
|
+
|
|
3
|
+
Parse `$ARGUMENTS` to extract the work item ID. Accept formats like `AB#1234`, `#1234`, or just `1234`. If no ID is provided, report the error and stop.
|
|
4
|
+
|
|
5
|
+
- **Environment**: optional — `local`, `dev`, `test`, or `staging`. Defaults to `dev`.
|
|
6
|
+
- **`--verify` / `--reproduce`**: optional — force the mode. Without either, Step 3 picks it from where the fix is.
|
|
7
|
+
- **`--role`, `--as`, `--as-role`**: exactly as in `/qa`. `--as` is how you reproduce a bug **as the user who reported it**, on an app that supports impersonation.
|
|
8
|
+
- **`--attempts=<n>`**: optional — how many times to run the steps, 1–5. Defaults to 2, or 5 when an earlier `/repro` called the bug intermittent.
|
|
9
|
+
|
|
10
|
+
**`prod` and `production` are refused**, for the same reason as `/qa`: an unattended run signs in and clicks through live member data. Say so and stop.
|
|
11
|
+
|
|
12
|
+
`/repro` and `/qa` share one browser agent (**qa**) and one set of gates. `/qa` asks *"does this story work, and did it break anything?"* `/repro` asks a narrower question: *"does this bug happen?"* It asks it twice — once before the fix, so the developer starts from a confirmed reproduction, and once after, so "fixed" means something.
|
|
13
|
+
|
|
14
|
+
## Step 1: Fetch the Bug
|
|
15
|
+
|
|
16
|
+
Read the work item from Azure DevOps using the project from the current repo's CLAUDE.md configuration. Expand with `relations` so PR links are included. Not found → report the error and stop.
|
|
17
|
+
|
|
18
|
+
**Type gate.** Only a **Bug** or **Hot Fix** has repro steps. For any other type, stop:
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
AB#{id} is a {type}, which has no repro steps. /repro follows a bug's steps.
|
|
22
|
+
To test a story's acceptance criteria, run /qa AB#{id} {environment}.
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Capture: id, title, type, state, created by, linked Pull Request artifacts, and these fields:
|
|
26
|
+
|
|
27
|
+
- `Microsoft.VSTS.TCM.ReproSteps` — the steps, usually followed by *actual* and *expected* results.
|
|
28
|
+
- `Microsoft.VSTS.TCM.SystemInfo` — browser, environment, and account notes, when filled in.
|
|
29
|
+
- `System.Description` — the fallback.
|
|
30
|
+
|
|
31
|
+
Apply `/qa` Step 1's empty-field rules. `""`, whitespace, `<div></div>`, `<p><br></p>` and the `💡 Tip:` placeholder are all empty. Repro Steps empty → use the Description, and note `Repro steps read from Description`. Both empty → stop:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
AB#{id} has no repro steps, so there's nothing to follow.
|
|
35
|
+
|
|
36
|
+
Add them to the work item's Repro Steps and re-run — or give them to me here and I'll
|
|
37
|
+
use them for this run only (they won't be written back to the work item).
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
**Read the comments.** Look for two things:
|
|
41
|
+
|
|
42
|
+
1. **Earlier `/repro` results.** They start with `Automated repro —`. Record each one's mode, verdict, environment, commit, and date. A prior `REPRODUCED` can serve as the control in Step 3, and a prior `INTERMITTENT` raises the default attempts to 5.
|
|
43
|
+
2. **Detail the steps leave out.** Which user hit it, which record, which environment, which browser. A named user feeds the `--as` suggestion in Step 4.
|
|
44
|
+
|
|
45
|
+
Download and view every embedded `<img>` in the steps, description, and comments with WebFetch. A screenshot of the bug is often the clearest statement of its signature.
|
|
46
|
+
|
|
47
|
+
## Step 2: Environment, Browser, Sign-in
|
|
48
|
+
|
|
49
|
+
Read `.claude/commands/qa.md` and run its **Step 2** (Resolve the Environment), **Step 3** (Is a Browser Available?) and **Step 4** (Sign-in Method, Credentials, Impersonation) exactly as written. In their messages, say `/repro` where they say `/qa`. Every stop condition in those steps stops `/repro` too. One refinement applies: under an ad-hoc `--as`, `/qa` makes the whole run read-only, but `/repro` lets the user approve individual writing steps in Step 4.
|
|
50
|
+
|
|
51
|
+
## Step 3: Pick the Mode
|
|
52
|
+
|
|
53
|
+
Find the fix's commits the way `/qa` Step 5a does: linked PRs → `lastMergeCommit`; `git log --all --grep="AB#<id>\b" --grep="#<id>\b" -i --format=%H`; and branches matching `*AB#<id>-*`. Dedupe by SHA. Then check each commit against the target with `git merge-base --is-ancestor <sha> origin/<target-branch>`, or against `HEAD` for `local`.
|
|
54
|
+
|
|
55
|
+
| Fix commits | On the target | Mode |
|
|
56
|
+
|---|---|---|
|
|
57
|
+
| none | — | **reproduce** — no fix exists yet |
|
|
58
|
+
| some | not all | **reproduce** — the fix exists but isn't fully on {environment}, so this run confirms the bug on the old build |
|
|
59
|
+
| some | all | **verify-fix** |
|
|
60
|
+
|
|
61
|
+
For `local`, also count it as verify-fix when the current branch matches `*AB#<id>-*` and the working tree has uncommitted changes — that is a fix in progress. Remind the user that the running dev server must be serving this working tree.
|
|
62
|
+
|
|
63
|
+
**Overrides.** `--reproduce` always wins. `--verify` on a target that lacks some fix commits is refused: verifying a fix that isn't deployed only tests the old build.
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
The fix for AB#{id} isn't fully on {environment} — {x} of {y} commits present.
|
|
67
|
+
/repro --verify here would test the build without it.
|
|
68
|
+
|
|
69
|
+
Run /repro AB#{id} {environment} to confirm the bug on this build, or verify where the
|
|
70
|
+
fix is: {environments whose branch has every commit, or "nowhere yet"}.
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
**Verify-fix on `dev` / `test` / `staging`** → also run `/qa` **Step 5b**. The pipeline that deployed the fix must be green. Being merged is not being deployed. **`local`, either mode** → run `/qa` Step 5's `local` probe. Reproduce mode on a deployed environment needs no pipeline gate, because it tests whatever is there.
|
|
74
|
+
|
|
75
|
+
**Pick the control** — verify-fix only. The control is what tells *fixed* apart from *these steps never triggered it*. Take the first one of these that applies:
|
|
76
|
+
|
|
77
|
+
1. A prior `/repro` comment recorded `REPRODUCED` with the same steps, on a commit without the fix → it is the control. No control run is needed.
|
|
78
|
+
2. Otherwise, walk down the `## Environment URLs` rows below the target, and take the first one whose branch lacks at least one fix commit. That environment still serves the bug. Under `--as`, it must also offer impersonation; skip any row that doesn't.
|
|
79
|
+
3. Neither → no control. Step 6 can then only say `FIXED — UNCONFIRMED`, and the plan says so up front.
|
|
80
|
+
|
|
81
|
+
## Step 4: Turn the Report Into a Script, Then Get Approval
|
|
82
|
+
|
|
83
|
+
Rewrite the repro steps as a numbered script the agent can follow literally: a route, a control's visible label, the value to enter. Keep the report's own wording where it is already concrete. Resolve "go to the bills page" to the route. When a step names something vaguely, like *"a bill"* or *"any provider"*, keep it vague and let the agent pick a suitable record. Never invent a record id.
|
|
84
|
+
|
|
85
|
+
Then define the two things the agent checks for. Without them a run proves nothing:
|
|
86
|
+
|
|
87
|
+
- **Bug signature** — the *actual result*, made observable: a status code, a message, a wrong value, a console error, a control stuck in the wrong state.
|
|
88
|
+
- **Expected behaviour** — the *expected result*: what a correct build shows at the same point.
|
|
89
|
+
|
|
90
|
+
If the report doesn't say what the bug looks like, ask one question and wait: `What do you see when the bug happens? I need something observable — an error message, a wrong value, a failed request.`
|
|
91
|
+
|
|
92
|
+
Mark every step that writes:
|
|
93
|
+
|
|
94
|
+
- `← writes: {what}` — submits, saves, approves, or sends. It runs under the normal write rules: prefixed QA data, and nothing that emails a real person.
|
|
95
|
+
- `← manual only — destructive` — it breaks one of the agent's write-safety rules: payment, settlement, bulk action, tax document, export-all, or a delete. The agent records it as `STEP FAILED`. If the bug sits behind such a step, say plainly that `/repro` can't reach it.
|
|
96
|
+
- `← manual only — protected input` — it requires typing an SSN, TIN, bank account number, or another protected value. The agent never types one.
|
|
97
|
+
- **Under an ad-hoc `--as`,** every writing step needs explicit approval here. Everything done while impersonating is recorded as that user. Approved steps go to the agent as **approved mutating steps**; the rest become `STEP FAILED — would write as {target}`.
|
|
98
|
+
|
|
99
|
+
**Suggest impersonation when it fits.** Step 1 may have found a named user who hit the bug. If the project's `**Impersonation:**` block covers this environment and `--as` wasn't given, add `Tip: the report says this happens to {user} — re-run with --as="{user}" to reproduce it as them.` If the bug is about what a *role* sees, suggest the matching persona.
|
|
100
|
+
|
|
101
|
+
Present the plan and **wait**:
|
|
102
|
+
|
|
103
|
+
```
|
|
104
|
+
## Repro plan — AB#{id} on {environment}
|
|
105
|
+
|
|
106
|
+
Bug: {title} ({type}, {state})
|
|
107
|
+
Mode: {reproduce — no fix exists yet
|
|
108
|
+
| reproduce — fix is on {branch} but not on {environment}
|
|
109
|
+
| verify-fix — {n} of {n} fix commits on `{branch}`, {pipeline} run {id} green}
|
|
110
|
+
Target: {environment} — {app-url}
|
|
111
|
+
Sign-in: {role} ({method})
|
|
112
|
+
Impersonating: {persona} ({role}) | {search term} — ad-hoc user, read-only except approved steps | none
|
|
113
|
+
Control: {baseline-env} — {url}, which doesn't have the fix
|
|
114
|
+
| prior /repro on {date} reproduced it on {env} at {sha7}
|
|
115
|
+
| none — {reason}; a pass can only be FIXED — UNCONFIRMED
|
|
116
|
+
Attempts: {n}
|
|
117
|
+
|
|
118
|
+
Steps:
|
|
119
|
+
1. Go to /bills
|
|
120
|
+
2. Open a bill in "Pending" status
|
|
121
|
+
3. Click "Convert to Check"
|
|
122
|
+
4. Click "Confirm" in the dialog ← writes: converts the bill
|
|
123
|
+
5. Open the Payments tab
|
|
124
|
+
|
|
125
|
+
Bug signature: POST /api/bills/{ref}/convert → 500, and the error toast "Something went wrong"
|
|
126
|
+
Expected behaviour: the bill shows status "Check Issued" and a payment row appears
|
|
127
|
+
|
|
128
|
+
{Tip: …}
|
|
129
|
+
|
|
130
|
+
Run it? (yes / edit the steps / edit the signature / cancel)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Step 5: Run It
|
|
134
|
+
|
|
135
|
+
If `/qa` Step 4 chose a hand-off, for either method, run `/qa` **Step 7a** (the sign-in hand-off) first.
|
|
136
|
+
|
|
137
|
+
Delegate to the **qa** agent with a brief containing:
|
|
138
|
+
|
|
139
|
+
- the task (`reproduce` or `verify-fix`), plus the bug's id, title, type and state;
|
|
140
|
+
- the target App and API URLs;
|
|
141
|
+
- the control's App URL, if the control is an environment, with "run a control" — otherwise `none`;
|
|
142
|
+
- the sign-in method and descriptor, with `session handed over` or `unattended`;
|
|
143
|
+
- the **names** of the credential variables, never the values;
|
|
144
|
+
- the impersonation block and target, marked persona or ad-hoc, if `--as` was given;
|
|
145
|
+
- the approved script, the bug signature, the expected behaviour, the approved mutating steps, and the number of attempts.
|
|
146
|
+
|
|
147
|
+
If the agent returns `BLOCKED — session ended`, run `/qa` Step 7a once more and re-invoke it for the attempts it didn't reach. After a handed-over session, close the browser yourself at the end.
|
|
148
|
+
|
|
149
|
+
**Under a `form` hand-off there is no control run on another environment.** The agent can't sign in there. Step 3's control then falls back to a prior `/repro` comment or none, and the plan must say so.
|
|
150
|
+
|
|
151
|
+
## Step 6: Verdict
|
|
152
|
+
|
|
153
|
+
Assign the verdict from the agent's attempts and the control. Count a `STEP FAILED` at or before the signature step as **not run**, never as *not seen*.
|
|
154
|
+
|
|
155
|
+
### Reproduce mode
|
|
156
|
+
|
|
157
|
+
| Attempts | Verdict |
|
|
158
|
+
|---|---|
|
|
159
|
+
| Signature seen in every attempt | `REPRODUCED` |
|
|
160
|
+
| Seen in some attempts | `REPRODUCED — INTERMITTENT ({k} of {n})` |
|
|
161
|
+
| Seen only under a variation | `REPRODUCED — under V{n}: {what changed}`. The report's steps are incomplete; recommend updating them |
|
|
162
|
+
| Not seen, expected behaviour observed | `NOT REPRODUCED` |
|
|
163
|
+
| Not seen, something else went wrong | `NOT REPRODUCED — DIFFERENT SYMPTOM: {what}` |
|
|
164
|
+
| A step couldn't be performed | `BLOCKED — {reason}` |
|
|
165
|
+
|
|
166
|
+
### Verify-fix mode
|
|
167
|
+
|
|
168
|
+
| Target | Control | Verdict |
|
|
169
|
+
|---|---|---|
|
|
170
|
+
| Not seen in any attempt, expected behaviour observed, screen check clean | seen, or a prior `REPRODUCED` | `FIXED` |
|
|
171
|
+
| Same | not seen, or no control | `FIXED — UNCONFIRMED` — the steps didn't trigger the bug on the old build either, so this run can't tell fixed from never-triggered |
|
|
172
|
+
| Not seen, expected behaviour observed, screen check fails | any | `FIXED — SCREEN REGRESSION: {check}`. Attribute it the way `/qa` Step 7 does: does the failing surface touch the fix's changed files? |
|
|
173
|
+
| Not seen, but expected behaviour **not** observed | any | `NOT FIXED — DIFFERENT SYMPTOM: {what happened instead}` |
|
|
174
|
+
| Seen in any attempt | any | `STILL REPRODUCES ({k} of {n})` |
|
|
175
|
+
| A step couldn't be performed | any | `BLOCKED — {reason}` |
|
|
176
|
+
|
|
177
|
+
A verify-fix with fewer than 3 attempts on a bug a prior run called intermittent can only be `FIXED — UNCONFIRMED`. Two clean runs of a 1-in-3 bug is just luck.
|
|
178
|
+
|
|
179
|
+
## Step 7: Where to Look — whenever the signature was seen
|
|
180
|
+
|
|
181
|
+
The developer's next question is *where*. Answer it from the evidence, reading only:
|
|
182
|
+
|
|
183
|
+
1. **Failing request → handler.** For each failing `METHOD path → status` the agent recorded, grep the API's controllers for the route attributes (`[Route]`, `[HttpPost("…")]`) that match. Report the action by `file:line`.
|
|
184
|
+
2. **Screen → component.** Map the route where the signature appeared to its component through the client router, as `/qa` Step 6.3 does. Then look one hop down to the components that make the failing call.
|
|
185
|
+
3. **Console error → source.** If the first console line names a function or a component, grep for it.
|
|
186
|
+
|
|
187
|
+
Report at most 5 locations, each with the evidence that led there. **Never claim a root cause** — these are places to start reading.
|
|
188
|
+
|
|
189
|
+
In **verify-fix** mode, when the bug still reproduces, run the same mapping. Then compare the locations with the fix commits' changed files. `the fix changed BillService.cs, but the failing call is handled in BillsController.cs:212` is the most useful sentence this command can produce.
|
|
190
|
+
|
|
191
|
+
## Step 8: Report
|
|
192
|
+
|
|
193
|
+
```
|
|
194
|
+
Repro — AB#{id}: {title}
|
|
195
|
+
{type} · {state} · {environment} — {app-url}
|
|
196
|
+
Mode: {reproduce | verify-fix} · Build: {pipeline run {id}, commit {sha7} | local working tree}
|
|
197
|
+
Signed in as: {role} ({method}) — value not shown
|
|
198
|
+
Impersonating: {persona | search term} as {role} | none
|
|
199
|
+
|
|
200
|
+
Verdict: {verdict}
|
|
201
|
+
|
|
202
|
+
## Attempts
|
|
203
|
+
| Attempt | Variation | Result | At step | Evidence |
|
|
204
|
+
|---------|-----------|--------|---------|----------|
|
|
205
|
+
| 1 | — | SIGNATURE SEEN | 4 | network `POST /api/bills/…/convert → 500` |
|
|
206
|
+
| 2 | — | SIGNATURE SEEN | 4 | network `POST /api/bills/…/convert → 500` |
|
|
207
|
+
|
|
208
|
+
## Control — {baseline-env | prior run {date}}
|
|
209
|
+
{SIGNATURE SEEN at step n | NOT SEEN | not run — {reason}}
|
|
210
|
+
|
|
211
|
+
## Screen check — {route} ← verify-fix only
|
|
212
|
+
{rows 1–5 from the agent}
|
|
213
|
+
|
|
214
|
+
## Where to look ← when the bug reproduced
|
|
215
|
+
1. `{file}:{line}` — {why: handles POST /api/bills/{ref}/convert, which returned 500}
|
|
216
|
+
2. …
|
|
217
|
+
|
|
218
|
+
## Not testable
|
|
219
|
+
- {step or check} — {why}
|
|
220
|
+
|
|
221
|
+
## Test data created on {environment}
|
|
222
|
+
- {entity} "QA AB#{id} — …" (id {x}) — left in place for review
|
|
223
|
+
|
|
224
|
+
Screenshot: {local path} — local only, not attached to the work item
|
|
225
|
+
|
|
226
|
+
Next: {REPRODUCED → `/implement AB#{id}`
|
|
227
|
+
| NOT REPRODUCED → ask the reporter for the record, the user, and the environment
|
|
228
|
+
where it happened; or re-run with --as or in another environment
|
|
229
|
+
| FIXED → `Ready for Testing` / `Ready for Staging` is yours to set
|
|
230
|
+
| FIXED — UNCONFIRMED → reproduce it on the old build first: /repro AB#{id} {baseline-env} --reproduce
|
|
231
|
+
| STILL REPRODUCES / NOT FIXED → `/rework AB#{id}`, with the "Where to look" list above}
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
## Step 9: Comment on the Work Item
|
|
235
|
+
|
|
236
|
+
Draft the comment, preview it, and **wait**, exactly as in `/qa` Step 9. Before drafting, check the existing comments for a `/repro` result with the **same mode and commit**. If one exists: `A /repro {mode} comment already exists for commit {sha7} ({date}). Post an updated one anyway? (yes / skip)`.
|
|
237
|
+
|
|
238
|
+
This comment matters more than `/qa`'s, because the next `/repro` reads it. A `REPRODUCED` here becomes the control that lets a later verify-fix say `FIXED` rather than `FIXED — UNCONFIRMED`. So **always include the script and the signature** — they are what make two runs comparable.
|
|
239
|
+
|
|
240
|
+
```html
|
|
241
|
+
<b>Automated repro — {verdict}</b> · {mode} · {environment} · {date}<br>
|
|
242
|
+
Build: {pipeline} run {id} · commit {sha7}{ | local working tree}<br>
|
|
243
|
+
Signed in as: {role} test account{, impersonating {persona | "the reporting user"}}<br><br>
|
|
244
|
+
|
|
245
|
+
<b>Steps run</b>
|
|
246
|
+
<ol>
|
|
247
|
+
<li>{step}</li>
|
|
248
|
+
</ol>
|
|
249
|
+
<b>Bug signature:</b> {signature}<br>
|
|
250
|
+
<b>Expected:</b> {expected behaviour}<br><br>
|
|
251
|
+
|
|
252
|
+
<b>Result — signature seen in {k} of {n} attempts</b>
|
|
253
|
+
<ul>
|
|
254
|
+
<li>Attempt 1: {result} at step {n} — {evidence}</li>
|
|
255
|
+
</ul>
|
|
256
|
+
{<b>Control</b> — {baseline-env}: {result}<br>}
|
|
257
|
+
{<b>Where to look</b><ul><li><code>{file}:{line}</code> — {why}</li></ul>}
|
|
258
|
+
{<b>Test data created</b><ul><li>{entity} "QA AB#{id} — …"</li></ul>}
|
|
259
|
+
|
|
260
|
+
<hr>
|
|
261
|
+
<i>Posted by <code>/repro</code>. Work item state unchanged. No screenshots and no field
|
|
262
|
+
values are attached, per the sensitive-data policy.</i>
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
The comment never names an ad-hoc impersonation target. It says *"the reporting user"*, because the search term is often a real person's email.
|
|
266
|
+
|
|
267
|
+
Then:
|
|
268
|
+
|
|
269
|
+
```
|
|
270
|
+
Repro drafted — nothing has been posted to AB#{id} yet.
|
|
271
|
+
|
|
272
|
+
## Work item comment
|
|
273
|
+
{the full body as it will appear}
|
|
274
|
+
|
|
275
|
+
Reply with one of:
|
|
276
|
+
- "approve" → post the comment exactly as shown
|
|
277
|
+
- "skip" → post nothing (the report above is yours to keep)
|
|
278
|
+
- "edit <section>: <new text>" → revise, then I'll re-preview before posting
|
|
279
|
+
|
|
280
|
+
Wait for my response. Never post to the work item until I reply "approve".
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
On `approve`, post exactly one comment via `mcp__azure-devops__wit_work_item_comment_write`.
|
|
284
|
+
|
|
285
|
+
That's it. `/repro` never changes the work item's state — a `FIXED` doesn't move the bug forward, and a `NOT REPRODUCED` doesn't close it or tag it. Those are human calls. It also never merges, pushes, creates a PR, triggers a pipeline, edits code, attaches a screenshot to the work item, or transcribes a field value from a screen. It reads, it clicks, it reports.
|
|
@@ -8,3 +8,4 @@
|
|
|
8
8
|
echo "Reminder: If you just completed a feature implementation, run UAT before marking it done."
|
|
9
9
|
echo "Ask Claude to generate a UAT checklist from the work item's acceptance criteria."
|
|
10
10
|
echo "Once it is merged and deployed, /qa AB#<id> runs an automated browser regression of the affected screens."
|
|
11
|
+
echo "For a bug fix, /repro AB#<id> re-runs the repro steps and confirms the bug is gone."
|
|
@@ -18,7 +18,7 @@ You (give task)
|
|
|
18
18
|
├── deployer → commits, pushes, triggers CD pipeline
|
|
19
19
|
├── db-admin → queries/fixes MongoDB data
|
|
20
20
|
├── test-runner → runs xUnit, Vitest, Playwright tests
|
|
21
|
-
├── qa → drives a real browser on a
|
|
21
|
+
├── qa → drives a real browser on a running environment: regression-tests a story's screens, reproduces a bug, verifies a fix
|
|
22
22
|
├── build-validator → confirms builds pass
|
|
23
23
|
├── lint-checker → runs ESLint and dotnet format
|
|
24
24
|
├── security-auditor → scans for secrets, vulnerabilities
|
|
@@ -83,7 +83,7 @@ These run automatically — no action needed:
|
|
|
83
83
|
| Server | What It Does |
|
|
84
84
|
|--------|-------------|
|
|
85
85
|
| **Azure DevOps** *(core — drives every slash command)* | Work items, repos, pull requests, pipelines, wiki, test plans, advanced security |
|
|
86
|
-
| **Playwright** | Browser testing (navigate, click, fill, screenshot) — **required by `/qa`** |
|
|
86
|
+
| **Playwright** | Browser testing (navigate, click, fill, screenshot) — **required by `/qa` and `/repro`** |
|
|
87
87
|
| **MongoDB** | Direct database queries and updates |
|
|
88
88
|
| **Microsoft Teams** | Send/read team messages and notifications |
|
|
89
89
|
| **Stripe** | Payment management (when configured) |
|
|
@@ -207,7 +207,7 @@ Every environment in the chain has a work item state, so the board shows where e
|
|
|
207
207
|
| `staging` | Staging | `Ready for Staging` | `Staging` |
|
|
208
208
|
| `prod` | Production | `Ready to Deploy` | `Deployed` |
|
|
209
209
|
|
|
210
|
-
`/qa`
|
|
210
|
+
`/qa` and `/repro` are the exceptions that prove the rule: they test a work item in a browser and post a result comment, but they **never write a state** — not even `Ready for Testing`, and not even when `/repro` finds a bug `FIXED`.
|
|
211
211
|
|
|
212
212
|
Human-only transitions — no slash command ever makes these: `Code Review → Ready for Testing` (the developer confirms the change on Dev), `Testing → Ready for Staging` (QA sign-off), `Staging → Ready to Deploy` (stakeholder / UAT sign-off), `Deployed → Closed` (verified in production).
|
|
213
213
|
|
|
@@ -395,7 +395,8 @@ All deployment and release operations are available as slash commands:
|
|
|
395
395
|
| `/rollback` | `/rollback AB#1234 prod` | Revert commits on an environment |
|
|
396
396
|
| `/track` | `/track release 24` | Check release, pipeline, environment, or work item status; flags work items whose state lags their environment |
|
|
397
397
|
| `/where` | `/where AB#1234` | Show which environment branches contain a work item's commits |
|
|
398
|
-
| `/qa` | `/qa AB#1234 [env]` | Verify the item is fully deployed with a green pipeline → open the app in a real browser → sign in
|
|
398
|
+
| `/qa` | `/qa AB#1234 [env] [--as=<user>]` | Verify the item is fully deployed with a green pipeline → open the app in a real browser → sign in (app login form, or Microsoft) and optionally impersonate a user → full regression of the screens the story touched → pass/fail comment (never a state change) |
|
|
399
|
+
| `/repro` | `/repro AB#1234 [env] [--as=<user>]` | Follow a bug's repro steps in a real browser. Before the fix: confirm it reproduces and point at the code to read. After the fix is deployed: prove the bug is gone, against a control run on a build without the fix → result comment (never a state change) |
|
|
399
400
|
| `/plan-backlog` | `/plan-backlog [project]` | Sweep backlog for Dev Ready stories with points and no tasks → propose child tasks with hours |
|
|
400
401
|
| `/plan-sprint` | `/plan-sprint [project] [--next]` | Sweep the current sprint (or the next one with `--next`) for stories/bugs with no child tasks → propose one child task with hours per item |
|
|
401
402
|
| `/quote-backlog` | `/quote-backlog [project]` | Sweep backlog for unpointed items ready to estimate — stories in `Design Approved`, **bugs in `New`** (bugs have no design states) → review completeness, check for duplicates, suggest rewrites, propose points (+ Investment Category / Impact where unset) + creator comments (10 at a time, approval-gated). Pointed items move to Dev Ready; stories that can't be quoted move back to **Design Review**, bugs that can't be quoted get a **`needs-info`** tag, so the next sweep skips them. Every run also audits the drop-out queue (tagged bugs **and** stories bounced to Design Review) and reports any whose creator answered but which nobody returned to the sweep — neither mechanism expires, so those are otherwise invisible forever |
|
|
@@ -467,7 +468,7 @@ Glasswing and Monarch (not yet migrated — 1 environment):
|
|
|
467
468
|
|
|
468
469
|
### Environment URLs
|
|
469
470
|
|
|
470
|
-
`/qa`
|
|
471
|
+
`/qa` and `/repro` open the app in a real browser. They need to know where each environment is hosted, how sign-in works, and which test account to use. Without this section they report and stop rather than guessing a hostname.
|
|
471
472
|
|
|
472
473
|
Add this to your project's `CLAUDE.md` — **above the `## Claude Kit Workflow` heading.** Everything from that heading to the end of the file is replaced on each kit update, so configuration placed below it is lost.
|
|
473
474
|
|
|
@@ -481,35 +482,92 @@ Add this to your project's `CLAUDE.md` — **above the `## Claude Kit Workflow`
|
|
|
481
482
|
| test | Test | https://test.myapp.example.com | https://test.api.myapp.example.com |
|
|
482
483
|
| staging | Staging | https://staging.myapp.example.com | https://staging.api.myapp.example.com |
|
|
483
484
|
|
|
484
|
-
**Sign-in:** form at `/login` — app-issued JWT, no SSO. Signed-in landing: `/dashboard`.
|
|
485
|
+
**Sign-in:** `form` at `/login` — app-issued JWT, no SSO. Signed-in landing: `/dashboard`.
|
|
485
486
|
|
|
486
|
-
**Test accounts** — values
|
|
487
|
+
**Test accounts** — each developer sets the values on their own machine; never write a password here.
|
|
487
488
|
|
|
488
489
|
| Role | Username variable | Password variable | Default |
|
|
489
490
|
|------|-------------------|-------------------|---------|
|
|
490
491
|
| Admin | `MYAPP_QA_ADMIN_USERNAME` | `MYAPP_QA_ADMIN_PASSWORD` | yes |
|
|
491
492
|
| Provider | `MYAPP_QA_PROVIDER_USERNAME` | `MYAPP_QA_PROVIDER_PASSWORD` | |
|
|
493
|
+
|
|
494
|
+
**Impersonation:** local, dev, test. Sign in as the `Admin` test account first.
|
|
495
|
+
- Start: `/admin/impersonate` → type into "Search users" → "Search" → the row's "Impersonate" button → "Impersonate" in the confirm dialog.
|
|
496
|
+
- You're the target when: the target's landing route loads.
|
|
497
|
+
- End: "Logout" → `/login`. This ends the whole session; sign in again as the admin to continue.
|
|
498
|
+
|
|
499
|
+
| Persona | Search term | Role | Notes |
|
|
500
|
+
|---------|-------------|------|-------|
|
|
501
|
+
| Provider | `qa-provider@example.com` | Provider | profile complete |
|
|
492
502
|
```
|
|
493
503
|
|
|
494
504
|
**Rules for slash commands:**
|
|
495
505
|
- **Branch names must match the `## Pipeline Configuration` table** row for row. The two tables are joined on the branch column — `/qa` reads the URL here and the pipeline ID there. A mismatch is a configuration error worth reporting.
|
|
496
|
-
- **There is no `prod` row, deliberately.** `/qa`
|
|
497
|
-
- **Include the rows below the one you test.** The row immediately below the target serves the pre-change build
|
|
506
|
+
- **There is no `prod` row, deliberately.** `/qa` and `/repro` refuse production: an unattended run signs in and clicks through live member data.
|
|
507
|
+
- **Include the rows below the one you test.** The row immediately below the target serves the pre-change build. That is how `/qa` tells a regression this story caused from a bug that was already there, and how `/repro` runs its *control*: it proves the steps still trigger the bug on a build without the fix, so a clean run on the fixed build means **fixed**, not *never triggered*.
|
|
498
508
|
- **App URL is required; API URL is optional** — it is used to attribute a failing network call to the backend rather than the screen.
|
|
499
|
-
- **`Sign-in:`**
|
|
500
|
-
-
|
|
501
|
-
-
|
|
509
|
+
- **`Sign-in:`** starts with the method, then optional prose: the login route, the landing page, and an element that only exists once you're signed in.
|
|
510
|
+
- `` `form` `` — the app's own login form, with no MFA. If the developer has set the role's credential variables, the `qa` agent fills the form in itself. If not, the developer signs in by hand in the browser window — see Test accounts below.
|
|
511
|
+
- `` `microsoft` `` — Entra ID / MSAL redirect. Name the app's sign-in button and the signed-in marker. **By default the person running the command signs in by hand**, once, in the Chromium window Playwright opens. MFA is fine, and no credential enters any transcript. Playwright MCP keeps a persistent browser profile, so later runs reuse that session until it expires. For a fully unattended run, give the role's row credential variables **and** state on the `Sign-in:` line that the account is a cloud-only, **MFA-exempt** QA account. The agent then types them into the Microsoft page itself.
|
|
512
|
+
- A line naming neither method is read as `microsoft` only if it says sign-in goes through Microsoft / Entra ID, and as `form` otherwise. Name the method explicitly; that is less fragile than relying on this.
|
|
513
|
+
- **Test accounts** are used by `/qa` and `/repro` and nothing else. The table declares the variable **names**. **Each developer sets the values on their own machine** — usually for their own account, which must hold the row's role on the environment being tested.
|
|
514
|
+
- **Per environment:** `{VAR}_{ENV}` overrides `{VAR}`, where `{ENV}` is `LOCAL`, `DEV`, `TEST`, or `STAGING`. Use it when your account's password differs on an environment with its own database — for example `COMPASS_QA_ADMIN_PASSWORD_TEST`.
|
|
515
|
+
- **Not set?** Then `/qa` and `/repro` don't stop. They offer to let you **sign in yourself** in the browser window, and your password never passes through Claude. The cost: the agent can't sign in anywhere else on its own, so on a `form` app there is no re-check on the baseline environment, and no second role.
|
|
516
|
+
- **Which to choose:** when the agent fills a form, the password is a tool argument. It is stored in the session transcript on your machine and sent to the model with every other tool call. For a dedicated QA account that's fine. **For your own administrator account, prefer signing in yourself.**
|
|
517
|
+
- The `Default` column marks the row used when `--role` is not given; if no row is marked, the first row wins. A second role lets `/qa` verify that restricted controls are correctly *absent* — a class of bug a manual checklist rarely catches. Under `microsoft` with a hand-off, the variable columns can be `—`; the role still says which account to sign in with.
|
|
518
|
+
- **`Impersonation:`** is optional. Add it only if the app has a "sign in as another user" feature. It is what makes `--as=<user>` work, and it describes four things:
|
|
519
|
+
- **where** it's available — impersonation is often environment-gated;
|
|
520
|
+
- **which test account** can impersonate, and in which role;
|
|
521
|
+
- **how to start it**, using the visible labels of the controls;
|
|
522
|
+
- **how to end it** — say so if ending it signs you out completely.
|
|
523
|
+
|
|
524
|
+
The persona table names QA test users to impersonate by role. Use dedicated QA users, never a real person. `--as=<persona>` uses the row. Any other `--as` value is searched for literally, as an **ad-hoc user** — typically the person who reported a bug. The app records everything done while impersonating as that user, so an ad-hoc run is **read-only**: `/qa` submits nothing, and `/repro` asks you to approve each writing step.
|
|
525
|
+
- **Never put a credential in these tables.** Only variable names. The values belong in your shell profile — see Environment Setup below.
|
|
502
526
|
|
|
503
|
-
**
|
|
527
|
+
**Examples from actual projects:**
|
|
504
528
|
|
|
505
|
-
COMPASS:
|
|
529
|
+
COMPASS — `form` sign-in, plus impersonation:
|
|
506
530
|
```markdown
|
|
507
531
|
| Branch | Environment | App URL | API URL |
|
|
508
532
|
|--------|------------|---------|---------|
|
|
509
|
-
| (none) | local | http://localhost:
|
|
533
|
+
| (none) | local | http://localhost:3000 | http://localhost:5191 |
|
|
510
534
|
| dev | Dev | https://dev.compass.caresolutions.com | https://dev.api.compass.caresolutions.com |
|
|
511
535
|
| test | Test | https://test.compass.caresolutions.com | https://test.api.compass.caresolutions.com |
|
|
512
536
|
| staging | Staging | https://staging.compass.caresolutions.com | https://staging.api.compass.caresolutions.com |
|
|
537
|
+
|
|
538
|
+
**Sign-in:** `form` at `/login` — app-issued JWT, no SSO, no MFA. If the account holds several roles, the login page asks which one; choose the table's role. Admin lands on `/admin/dashboard`.
|
|
539
|
+
|
|
540
|
+
**Test accounts** — each developer uses **their own System Administrator account**, set on their own machine. Local and dev share a database; test has its own, so set `COMPASS_QA_ADMIN_PASSWORD_TEST` if your test password differs.
|
|
541
|
+
|
|
542
|
+
| Role | Username variable | Password variable | Default |
|
|
543
|
+
|------|-------------------|-------------------|---------|
|
|
544
|
+
| System Administrator | `COMPASS_QA_ADMIN_USERNAME` | `COMPASS_QA_ADMIN_PASSWORD` | yes |
|
|
545
|
+
|
|
546
|
+
**Impersonation:** local, dev, test — **not staging** (the API returns 404 there). Sign in as the `Admin` test account in the **System Administrator** role.
|
|
547
|
+
- Start: `/admin/impersonate` → type into "Search users" → "Search" → on the row, "Impersonate" (or "Impersonate as <Role>" for a multi-role user) → "Impersonate" in the "Impersonate this user?" dialog.
|
|
548
|
+
- You're the target when: the target's landing route loads. There is no impersonation banner.
|
|
549
|
+
- End: "Logout" (drawer bottom row, or the avatar menu) → `/login`. This signs you out completely; sign in again as the admin to continue.
|
|
550
|
+
- The token has no `impersonatedBy` claim — every action is recorded as the target.
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
CSIPay — `microsoft` sign-in, no impersonation:
|
|
554
|
+
```markdown
|
|
555
|
+
| Branch | Environment | App URL | API URL |
|
|
556
|
+
|--------|------------|---------|---------|
|
|
557
|
+
| (none) | local | http://localhost:3001 | https://localhost:5001/api |
|
|
558
|
+
| dev | Dev | https://dev.csipay.caresolutions.com | https://dev.api.csipay.caresolutions.com/api |
|
|
559
|
+
| test | Test | https://test.csipay.caresolutions.com | https://test.api.csipay.caresolutions.com/api |
|
|
560
|
+
| staging | Staging | https://staging.csipay.caresolutions.com | https://staging.api.csipay.caresolutions.com/api |
|
|
561
|
+
|
|
562
|
+
**Sign-in:** `microsoft` — Entra ID redirect via MSAL, single tenant. Any URL shows the "Sign in with Microsoft" button when signed out. Signed-in marker: the "Dashboard" heading at `/`. Roles are Entra app roles (`Admin`, `Developer`).
|
|
563
|
+
|
|
564
|
+
| Role | Username variable | Password variable | Default |
|
|
565
|
+
|------|-------------------|-------------------|---------|
|
|
566
|
+
| Developer | — | — | yes |
|
|
567
|
+
| Admin | — | — | |
|
|
568
|
+
| User (no app role) | — | — | |
|
|
569
|
+
|
|
570
|
+
Each developer signs in with their own Microsoft account. `Admin` and `Developer` see unmasked identifiers on Payees, which the sensitive-data hook blocks on sight. To test Payees content, sign in with an account that has no app role.
|
|
513
571
|
```
|
|
514
572
|
|
|
515
573
|
### Environment Setup
|
|
@@ -529,14 +587,16 @@ az login
|
|
|
529
587
|
# Microsoft Teams MCP — no env vars needed. First invocation prints a
|
|
530
588
|
# device code + URL to sign in via Microsoft Graph (OAuth device flow).
|
|
531
589
|
|
|
532
|
-
# QA
|
|
533
|
-
#
|
|
534
|
-
#
|
|
535
|
-
#
|
|
536
|
-
|
|
590
|
+
# QA sign-in for /qa and /repro — OPTIONAL. Set these only if you want the agent
|
|
591
|
+
# to sign in unattended. Leave them unset and you'll be asked to sign in yourself in
|
|
592
|
+
# the browser window, so your password never passes through Claude.
|
|
593
|
+
# The variable NAMES come from the project's `## Environment URLs` section. The
|
|
594
|
+
# VALUES are yours — usually your own account with the role the table names — and
|
|
595
|
+
# they live only here, never in any file. `_TEST` / `_DEV` / `_LOCAL` / `_STAGING`
|
|
596
|
+
# versions override the plain one on that environment.
|
|
597
|
+
export COMPASS_QA_ADMIN_USERNAME="you@caresolutions.com"
|
|
537
598
|
export COMPASS_QA_ADMIN_PASSWORD="..."
|
|
538
|
-
export
|
|
539
|
-
export COMPASS_QA_PROVIDER_PASSWORD="..."
|
|
599
|
+
export COMPASS_QA_ADMIN_PASSWORD_TEST="..." # only if your test password differs
|
|
540
600
|
```
|
|
541
601
|
|
|
542
|
-
`/qa`
|
|
602
|
+
When these are set, `/qa` and `/repro` read them **inside a subagent** and fill the login form directly. They never print a value, never write one to a file, and never include one in a report or a work item comment. The value is still a tool argument, so it lands in that subagent's local transcript. That is why signing in yourself is the better choice for a personal administrator account. Apps that sign in through Microsoft need no variables at all: you sign in once in the browser window, and the session persists in the Playwright profile.
|