@roopesh.yadava/qa-pack 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -7
- package/bin/postinstall.js +31 -0
- package/claude/skills/SKILLS_CONTEXT.md +61 -5
- package/claude/skills/automation/SKILL.md +53 -6
- package/claude/skills/bug-reporting/SKILL.md +38 -0
- package/claude/skills/k6-framework-scaffold/SKILL.md +58 -0
- package/claude/skills/k6-framework-scaffold/scaffold.sh +388 -0
- package/claude/skills/manual-testing/SKILL.md +67 -2
- package/claude/skills/qa-agent/SKILL.md +30 -0
- package/claude/skills/qa-agent/product_context/README.md +16 -0
- package/claude/skills/qa-agent/toolkit/qa-toolkit.cjs +705 -0
- package/claude/skills/qa-insights/SKILL.md +55 -0
- package/claude/skills/roam-testing/SKILL.md +187 -0
- package/package.json +5 -2
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# k6-framework-scaffold — create a ready-to-use k6 load-testing framework in the CURRENT repo.
|
|
3
|
+
# Usage: bash .claude/skills/k6-framework-scaffold/scaffold.sh [--force]
|
|
4
|
+
# (no flag) aborts if k6-performance-tests/ already exists.
|
|
5
|
+
# --force scaffolds into an existing folder; still never overwrites an existing file.
|
|
6
|
+
set -euo pipefail
|
|
7
|
+
|
|
8
|
+
ROOT="k6-performance-tests"
|
|
9
|
+
FORCE="${1:-}"
|
|
10
|
+
|
|
11
|
+
if [ -d "$ROOT" ] && [ "$FORCE" != "--force" ]; then
|
|
12
|
+
echo "ERROR: $ROOT/ already exists. Re-run with --force to add only missing files." >&2
|
|
13
|
+
exit 1
|
|
14
|
+
fi
|
|
15
|
+
|
|
16
|
+
# w <relative-path> : write heredoc (stdin) to $ROOT/<path>, only if the file doesn't exist.
|
|
17
|
+
w() {
|
|
18
|
+
local path="$1"
|
|
19
|
+
if [ -f "$ROOT/$path" ]; then echo " skip (exists): $path"; cat >/dev/null; return; fi
|
|
20
|
+
mkdir -p "$ROOT/$(dirname "$path")"
|
|
21
|
+
cat > "$ROOT/$path"
|
|
22
|
+
echo " created: $path"
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
echo "Scaffolding $ROOT/ ..."
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------- config/
|
|
28
|
+
w config/env.js <<'EOF'
|
|
29
|
+
// config/env.js — shared configuration. Fill these in for your app + environment.
|
|
30
|
+
|
|
31
|
+
export const BASE = 'https://api.example.com/v1'; // TODO: API base URL
|
|
32
|
+
export const UI_BASE = 'https://app.example.com'; // TODO: web-app URL (browser tests only)
|
|
33
|
+
|
|
34
|
+
export const PROJECT_ID = 0; // TODO: Grafana Cloud project id
|
|
35
|
+
|
|
36
|
+
// Target concurrency — the production peak you reproduce. Rescale by editing these numbers.
|
|
37
|
+
export const CONCURRENT = { userTypeA: 100, userTypeB: 100 };
|
|
38
|
+
export const TOTAL = {
|
|
39
|
+
userTypeAVUs: CONCURRENT.userTypeA,
|
|
40
|
+
userTypeBVUs: CONCURRENT.userTypeB,
|
|
41
|
+
totalVUs: CONCURRENT.userTypeA + CONCURRENT.userTypeB,
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
// Credentials + ids used by auth and parameterised endpoints.
|
|
45
|
+
export const CREDS = { username: 'TODO', password: 'TODO', otp: '000000' };
|
|
46
|
+
export const TEST_DATA = { sampleId: 'TODO' };
|
|
47
|
+
|
|
48
|
+
// Include mutating (write) endpoints in the mix? They modify real data on the target env.
|
|
49
|
+
export const ENABLE_WRITES = false;
|
|
50
|
+
|
|
51
|
+
// Long-lived auth token(s) live in secrets.js (rotate per run).
|
|
52
|
+
export { ACCESS_TOKEN } from './secrets.js';
|
|
53
|
+
EOF
|
|
54
|
+
|
|
55
|
+
w config/secrets.js <<'EOF'
|
|
56
|
+
// config/secrets.js — auth material, refreshed before each run.
|
|
57
|
+
// ⚠️ Keep production tokens OUT of version control. Use short-lived non-prod tokens.
|
|
58
|
+
export const ACCESS_TOKEN = 'Bearer TODO';
|
|
59
|
+
EOF
|
|
60
|
+
|
|
61
|
+
w config/profiles.js <<'EOF'
|
|
62
|
+
// config/profiles.js — reusable LOAD PROFILES (the "how much"). A test wires a flow to one.
|
|
63
|
+
|
|
64
|
+
export function loadProfile({ exec, target, rampUp = '2m', hold = '15m', rampDown = '2m', startVUs = 0 }) {
|
|
65
|
+
return { executor: 'ramping-vus', exec, startVUs, stages: [
|
|
66
|
+
{ duration: rampUp, target }, { duration: hold, target }, { duration: rampDown, target: 0 },
|
|
67
|
+
] };
|
|
68
|
+
}
|
|
69
|
+
export function arrivalRateProfile({ exec, rate, timeUnit = '1m', duration = '15m', preAllocatedVUs = 50, maxVUs = 100 }) {
|
|
70
|
+
return { executor: 'constant-arrival-rate', exec, rate, timeUnit, duration, preAllocatedVUs, maxVUs };
|
|
71
|
+
}
|
|
72
|
+
export function spikeProfile({ exec, target, rampUp = '30s', hold = '1m', rampDown = '30s', startVUs = 0 }) {
|
|
73
|
+
return { executor: 'ramping-vus', exec, startVUs, stages: [
|
|
74
|
+
{ duration: rampUp, target }, { duration: hold, target }, { duration: rampDown, target: 0 },
|
|
75
|
+
] };
|
|
76
|
+
}
|
|
77
|
+
export function breakpointProfile({ exec, maxRate, ramp = '30m', timeUnit = '1m', preAllocatedVUs = 50, maxVUs = 200 }) {
|
|
78
|
+
return { executor: 'ramping-arrival-rate', exec, startRate: 0, timeUnit, preAllocatedVUs, maxVUs,
|
|
79
|
+
stages: [{ duration: ramp, target: maxRate }] };
|
|
80
|
+
}
|
|
81
|
+
export function smokeProfile({ exec, vus = 1, iterations = 1 }) {
|
|
82
|
+
return { executor: 'per-vu-iterations', exec, vus, iterations, maxDuration: '1m' };
|
|
83
|
+
}
|
|
84
|
+
export function enduranceProfile({ exec, target, rampUp = '2m', hold = '2h', rampDown = '2m', startVUs = 0 }) {
|
|
85
|
+
return { executor: 'ramping-vus', exec, startVUs, stages: [
|
|
86
|
+
{ duration: rampUp, target }, { duration: hold, target }, { duration: rampDown, target: 0 },
|
|
87
|
+
] };
|
|
88
|
+
}
|
|
89
|
+
EOF
|
|
90
|
+
|
|
91
|
+
# ---------------------------------------------------------------- lib/
|
|
92
|
+
w lib/http.js <<'EOF'
|
|
93
|
+
// lib/http.js — shared request headers + response handling.
|
|
94
|
+
import { check } from 'k6';
|
|
95
|
+
|
|
96
|
+
// `token` is the full Bearer string. Each flow passes its own token.
|
|
97
|
+
export function authHeaders(token, extra = {}) {
|
|
98
|
+
return { Authorization: token, 'Content-Type': 'application/json', ...extra };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// 2xx check; logs 4xx (client-side) with a short body snippet. 5xx tracked by error-rate metric.
|
|
102
|
+
export function checkResponse(res, name) {
|
|
103
|
+
check(res, { [`${name} ok`]: (r) => r.status >= 200 && r.status < 300 });
|
|
104
|
+
if (res.status >= 400 && res.status < 500) {
|
|
105
|
+
const body = String(res.body || '').replace(/\s+/g, ' ').slice(0, 300);
|
|
106
|
+
console.error(`FAIL ${name} | ${res.status} ${res.request.method} ${res.request.url} | ${body}`);
|
|
107
|
+
}
|
|
108
|
+
return res;
|
|
109
|
+
}
|
|
110
|
+
EOF
|
|
111
|
+
|
|
112
|
+
w lib/rand.js <<'EOF'
|
|
113
|
+
// lib/rand.js — deterministic pseudo-random for load distribution (NOT cryptographic; and
|
|
114
|
+
// intentionally not the JS built-in RNG, which trips security scans). Reproducible runs.
|
|
115
|
+
|
|
116
|
+
// xorshift32 -> fraction in [0,1). seed = any integer (e.g. the iteration index).
|
|
117
|
+
export function frac(seed) {
|
|
118
|
+
let x = (seed >>> 0) || 1;
|
|
119
|
+
x ^= x << 13; x >>>= 0; x ^= x >>> 17; x ^= x << 5; x >>>= 0;
|
|
120
|
+
return (x >>> 0) / 4294967296;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Pick a weighted action given the total weight and a value u in [0,1).
|
|
124
|
+
export function weightedPick(actions, totalWeight, u) {
|
|
125
|
+
const target = u * totalWeight; let acc = 0;
|
|
126
|
+
for (let i = 0; i < actions.length; i++) { acc += actions[i].weight; if (target <= acc) return actions[i]; }
|
|
127
|
+
return actions[actions.length - 1];
|
|
128
|
+
}
|
|
129
|
+
EOF
|
|
130
|
+
|
|
131
|
+
w lib/browser.js <<'EOF'
|
|
132
|
+
// lib/browser.js — checkpoints for browser/ (real-Chromium) tests.
|
|
133
|
+
import { check } from 'k6';
|
|
134
|
+
|
|
135
|
+
// Element visible within timeout ms (a "stuck UI" surfaces here as a timeout -> failure).
|
|
136
|
+
export async function expectVisible(page, selector, name, timeout = 10000) {
|
|
137
|
+
let ok = false;
|
|
138
|
+
try { await page.locator(selector).waitFor({ state: 'visible', timeout }); ok = true; } catch (e) { ok = false; }
|
|
139
|
+
check(ok, { [`${name} visible`]: (v) => v === true });
|
|
140
|
+
return ok;
|
|
141
|
+
}
|
|
142
|
+
// Element gone within timeout ms (e.g. a spinner cleared).
|
|
143
|
+
export async function expectHidden(page, selector, name, timeout = 15000) {
|
|
144
|
+
let ok = false;
|
|
145
|
+
try { await page.locator(selector).waitFor({ state: 'hidden', timeout }); ok = true; } catch (e) { ok = false; }
|
|
146
|
+
check(ok, { [`${name} settled`]: (v) => v === true });
|
|
147
|
+
return ok;
|
|
148
|
+
}
|
|
149
|
+
// Best-effort screenshot (never fails the iteration).
|
|
150
|
+
export async function shot(page, path) { try { await page.screenshot({ path }); } catch (e) {} }
|
|
151
|
+
EOF
|
|
152
|
+
|
|
153
|
+
# ---------------------------------------------------------------- protocol/api/
|
|
154
|
+
w protocol/api/exampleUser/auth.js <<'EOF'
|
|
155
|
+
// protocol/api/exampleUser/auth.js — token-issuing login. Adapt to your auth, or delete if
|
|
156
|
+
// you only use a pre-issued static token (config/secrets.js).
|
|
157
|
+
import http from 'k6/http';
|
|
158
|
+
import { BASE, CREDS } from '../../../config/env.js';
|
|
159
|
+
import { checkResponse } from '../../../lib/http.js';
|
|
160
|
+
|
|
161
|
+
const JSON_HEADERS = { 'Content-Type': 'application/json' };
|
|
162
|
+
|
|
163
|
+
// Example: POST /auth/sign-in -> returns a session (for a two-step OTP flow) or a token.
|
|
164
|
+
export function signIn(tags) {
|
|
165
|
+
const res = http.post(`${BASE}/auth/sign-in`,
|
|
166
|
+
JSON.stringify({ username: CREDS.username, password: CREDS.password }),
|
|
167
|
+
{ headers: JSON_HEADERS, tags: { name: 'auth-sign-in', ...tags } });
|
|
168
|
+
checkResponse(res, 'auth-sign-in');
|
|
169
|
+
try { return res.json('data.session'); } catch (e) { return null; }
|
|
170
|
+
}
|
|
171
|
+
// If login is two-step (OTP): read the session from signIn() and feed it into a verify() here.
|
|
172
|
+
EOF
|
|
173
|
+
|
|
174
|
+
w protocol/api/exampleUser/sample.js <<'EOF'
|
|
175
|
+
// protocol/api/exampleUser/sample.js — ENDPOINTS: one function per API call. Copy this shape
|
|
176
|
+
// for each endpoint (add a unique `name` tag so it shows per-endpoint in Grafana).
|
|
177
|
+
import http from 'k6/http';
|
|
178
|
+
import { BASE, TEST_DATA } from '../../../config/env.js';
|
|
179
|
+
import { authHeaders, checkResponse } from '../../../lib/http.js';
|
|
180
|
+
|
|
181
|
+
export function list(token, tags) {
|
|
182
|
+
const res = http.get(`${BASE}/sample/list?page=1&limit=10`,
|
|
183
|
+
{ headers: authHeaders(token), tags: { name: 'sample-list', ...tags } });
|
|
184
|
+
return checkResponse(res, 'sample-list');
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function get(token, tags) {
|
|
188
|
+
const res = http.get(`${BASE}/sample/get/${TEST_DATA.sampleId}`,
|
|
189
|
+
{ headers: authHeaders(token), tags: { name: 'sample-get', ...tags } });
|
|
190
|
+
return checkResponse(res, 'sample-get');
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// ⚠️ WRITE example — mark write:true in the flow so ENABLE_WRITES can gate it.
|
|
194
|
+
// export function create(token, tags) {
|
|
195
|
+
// const res = http.post(`${BASE}/sample`, JSON.stringify({ /* ... */ }),
|
|
196
|
+
// { headers: authHeaders(token), tags: { name: 'sample-create', ...tags } });
|
|
197
|
+
// return checkResponse(res, 'sample-create');
|
|
198
|
+
// }
|
|
199
|
+
EOF
|
|
200
|
+
|
|
201
|
+
# ---------------------------------------------------------------- protocol/flows/
|
|
202
|
+
w protocol/flows/exampleUserFlow.js <<'EOF'
|
|
203
|
+
// protocol/flows/exampleUserFlow.js — JOURNEY: a weighted API traffic-split for one user
|
|
204
|
+
// population. Each iteration a VU performs ONE weighted action; over the run the request mix
|
|
205
|
+
// converges to these proportions (= the client's feature-usage %).
|
|
206
|
+
import { sleep } from 'k6';
|
|
207
|
+
import exec from 'k6/execution';
|
|
208
|
+
import { ACCESS_TOKEN, ENABLE_WRITES } from '../../config/env.js';
|
|
209
|
+
import { frac, weightedPick } from '../../lib/rand.js';
|
|
210
|
+
import * as sample from '../api/exampleUser/sample.js';
|
|
211
|
+
|
|
212
|
+
// Pass/fail thresholds (from the client's SLA).
|
|
213
|
+
export const THRESHOLDS = {
|
|
214
|
+
http_req_duration: ['p(95)<2000'], // TODO: SLA
|
|
215
|
+
http_req_failed: ['rate<0.01'],
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
// weight = share of total traffic (sum ~ 1.0). write:true = mutating (gated by ENABLE_WRITES).
|
|
219
|
+
const ACTIONS = [
|
|
220
|
+
{ weight: 0.7, run: (t) => sample.list(t) },
|
|
221
|
+
{ weight: 0.3, run: (t) => sample.get(t) },
|
|
222
|
+
// { weight: 0.1, write: true, run: (t) => sample.create(t) },
|
|
223
|
+
];
|
|
224
|
+
const ACTIVE = ENABLE_WRITES ? ACTIONS : ACTIONS.filter((a) => !a.write);
|
|
225
|
+
const TOTAL_WEIGHT = ACTIVE.reduce((s, a) => s + a.weight, 0);
|
|
226
|
+
|
|
227
|
+
export function exampleUserFlow() {
|
|
228
|
+
const token = ACCESS_TOKEN;
|
|
229
|
+
const seed = exec.scenario.iterationInTest + 1;
|
|
230
|
+
weightedPick(ACTIVE, TOTAL_WEIGHT, frac(seed)).run(token);
|
|
231
|
+
sleep(1 + frac(seed ^ 0x5bd1e995) * 2); // think time 1-3s
|
|
232
|
+
}
|
|
233
|
+
EOF
|
|
234
|
+
|
|
235
|
+
# ---------------------------------------------------------------- protocol/tests/
|
|
236
|
+
w protocol/tests/smoke/example.smoke.js <<'EOF'
|
|
237
|
+
// protocol/tests/smoke/example.smoke.js — quick "does it work?" before a big run.
|
|
238
|
+
import { PROJECT_ID } from '../../../config/env.js';
|
|
239
|
+
import { smokeProfile } from '../../../config/profiles.js';
|
|
240
|
+
import { THRESHOLDS } from '../../flows/exampleUserFlow.js';
|
|
241
|
+
export { exampleUserFlow } from '../../flows/exampleUserFlow.js';
|
|
242
|
+
|
|
243
|
+
export const options = {
|
|
244
|
+
cloud: { projectID: PROJECT_ID, name: 'MyApp API — Smoke' },
|
|
245
|
+
scenarios: { smoke: smokeProfile({ exec: 'exampleUserFlow', vus: 1, iterations: 1 }) },
|
|
246
|
+
thresholds: THRESHOLDS,
|
|
247
|
+
};
|
|
248
|
+
EOF
|
|
249
|
+
|
|
250
|
+
w protocol/tests/load/example.load.js <<'EOF'
|
|
251
|
+
// protocol/tests/load/example.load.js — the expected-peak run.
|
|
252
|
+
import { PROJECT_ID, TOTAL } from '../../../config/env.js';
|
|
253
|
+
import { loadProfile } from '../../../config/profiles.js';
|
|
254
|
+
import { THRESHOLDS } from '../../flows/exampleUserFlow.js';
|
|
255
|
+
export { exampleUserFlow } from '../../flows/exampleUserFlow.js';
|
|
256
|
+
|
|
257
|
+
export const options = {
|
|
258
|
+
cloud: { projectID: PROJECT_ID, name: 'MyApp API — Load' },
|
|
259
|
+
scenarios: {
|
|
260
|
+
users: loadProfile({ exec: 'exampleUserFlow', target: TOTAL.userTypeAVUs, hold: '15m' }),
|
|
261
|
+
},
|
|
262
|
+
thresholds: THRESHOLDS,
|
|
263
|
+
};
|
|
264
|
+
EOF
|
|
265
|
+
|
|
266
|
+
w protocol/tests/spike/README.md <<'EOF'
|
|
267
|
+
# Spike tests
|
|
268
|
+
Copy a load test here and swap the profile to `spikeProfile` (config/profiles.js) — a sudden
|
|
269
|
+
burst then a quick drop, to check the system survives/recovers from a surge.
|
|
270
|
+
EOF
|
|
271
|
+
|
|
272
|
+
w protocol/tests/breakpoint/README.md <<'EOF'
|
|
273
|
+
# Breakpoint tests
|
|
274
|
+
Ramp load upward until it breaks (find capacity). Use `breakpointProfile` and pair with an
|
|
275
|
+
`abortOnFail` threshold so the run stops at the breaking point.
|
|
276
|
+
EOF
|
|
277
|
+
|
|
278
|
+
w protocol/tests/endurance/README.md <<'EOF'
|
|
279
|
+
# Endurance (soak) tests
|
|
280
|
+
Moderate, sustained load for hours (memory leaks, resource growth). Use `enduranceProfile`.
|
|
281
|
+
EOF
|
|
282
|
+
|
|
283
|
+
# ---------------------------------------------------------------- browser/ (OPTIONAL)
|
|
284
|
+
w browser/pages/exampleApp/loginPage.js <<'EOF'
|
|
285
|
+
// browser/pages/exampleApp/loginPage.js — PAGE OBJECT (selectors + actions). Fill selectors
|
|
286
|
+
// from the real DOM/source. SPA-safe: wait on the next screen's element, not full navigation.
|
|
287
|
+
import { UI_BASE, CREDS } from '../../../config/env.js';
|
|
288
|
+
|
|
289
|
+
const SEL = {
|
|
290
|
+
username: '#username', // TODO
|
|
291
|
+
password: '#password', // TODO
|
|
292
|
+
submit: '[data-testid="submit"]', // TODO
|
|
293
|
+
};
|
|
294
|
+
|
|
295
|
+
export async function login(page) {
|
|
296
|
+
await page.goto(`${UI_BASE}/login`, { waitUntil: 'networkidle' });
|
|
297
|
+
await page.locator(SEL.username).fill(CREDS.username);
|
|
298
|
+
await page.locator(SEL.password).fill(CREDS.password);
|
|
299
|
+
await page.locator(SEL.submit).click();
|
|
300
|
+
}
|
|
301
|
+
EOF
|
|
302
|
+
|
|
303
|
+
w browser/pages/exampleApp/samplePage.js <<'EOF'
|
|
304
|
+
// browser/pages/exampleApp/samplePage.js — checkpoints for a screen.
|
|
305
|
+
import { expectVisible } from '../../../lib/browser.js';
|
|
306
|
+
const SEL = { root: '[data-testid="page-root"]' }; // TODO
|
|
307
|
+
export async function assertLoaded(page) {
|
|
308
|
+
return expectVisible(page, SEL.root, 'sample-page', 15000);
|
|
309
|
+
}
|
|
310
|
+
EOF
|
|
311
|
+
|
|
312
|
+
w browser/flows/exampleUiFlow.js <<'EOF'
|
|
313
|
+
// browser/flows/exampleUiFlow.js — UI JOURNEY (real browser): login -> a screen renders.
|
|
314
|
+
import { browser } from 'k6/browser';
|
|
315
|
+
import * as loginPage from '../pages/exampleApp/loginPage.js';
|
|
316
|
+
import * as samplePage from '../pages/exampleApp/samplePage.js';
|
|
317
|
+
import { shot } from '../../lib/browser.js';
|
|
318
|
+
|
|
319
|
+
export const UI_THRESHOLDS = {
|
|
320
|
+
checks: ['rate>0.95'],
|
|
321
|
+
browser_web_vital_lcp: ['p(90)<2500'], // TODO: tune to UX SLA
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
export async function exampleUiJourney() {
|
|
325
|
+
const page = await browser.newPage();
|
|
326
|
+
try {
|
|
327
|
+
await loginPage.login(page);
|
|
328
|
+
const ok = await samplePage.assertLoaded(page);
|
|
329
|
+
await shot(page, `screenshots/example-${ok ? 'ok' : 'FAIL'}.png`);
|
|
330
|
+
} finally {
|
|
331
|
+
await page.close();
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
EOF
|
|
335
|
+
|
|
336
|
+
w browser/tests/exampleUi.browser.js <<'EOF'
|
|
337
|
+
// browser/tests/exampleUi.browser.js — run a FEW real-browser VUs (each is a real Chromium,
|
|
338
|
+
// so keep counts small). Headless by default; K6_BROWSER_HEADLESS=false to watch locally.
|
|
339
|
+
import { PROJECT_ID } from '../../config/env.js';
|
|
340
|
+
import { UI_THRESHOLDS } from '../flows/exampleUiFlow.js';
|
|
341
|
+
export { exampleUiJourney } from '../flows/exampleUiFlow.js';
|
|
342
|
+
|
|
343
|
+
export const options = {
|
|
344
|
+
cloud: { projectID: PROJECT_ID, name: 'MyApp UI — Example Journey (browser)' },
|
|
345
|
+
scenarios: {
|
|
346
|
+
ui: {
|
|
347
|
+
executor: 'per-vu-iterations', exec: 'exampleUiJourney', vus: 1, iterations: 1,
|
|
348
|
+
maxDuration: '3m', options: { browser: { type: 'chromium' } },
|
|
349
|
+
},
|
|
350
|
+
},
|
|
351
|
+
thresholds: UI_THRESHOLDS,
|
|
352
|
+
};
|
|
353
|
+
EOF
|
|
354
|
+
|
|
355
|
+
# ---------------------------------------------------------------- root files
|
|
356
|
+
w .gitignore <<'EOF'
|
|
357
|
+
config/secrets.local.js
|
|
358
|
+
.env
|
|
359
|
+
*.k6-summary.json
|
|
360
|
+
summary.json
|
|
361
|
+
screenshots/
|
|
362
|
+
EOF
|
|
363
|
+
|
|
364
|
+
w README.md <<'EOF'
|
|
365
|
+
# k6 Performance & UX Test Framework
|
|
366
|
+
|
|
367
|
+
Two layers, sharing `config/` + `lib/`:
|
|
368
|
+
- **protocol/** — API / HTTP load (the CORE, every app). api/ (endpoints) -> flows/ (weighted
|
|
369
|
+
traffic-split + thresholds) -> tests/ (wire a flow to a load profile).
|
|
370
|
+
- **browser/** — real-browser UX (OPTIONAL; UI-heavy apps only). Delete this folder if unneeded.
|
|
371
|
+
|
|
372
|
+
## Fill in
|
|
373
|
+
1. `config/env.js` — BASE / UI_BASE / PROJECT_ID / scale numbers / test data.
|
|
374
|
+
2. `config/secrets.js` — auth token(s) (keep out of VCS).
|
|
375
|
+
3. `protocol/api/**` — one function per real endpoint.
|
|
376
|
+
4. `protocol/flows/**` — weighted action table (feature-usage %) + thresholds (SLA).
|
|
377
|
+
|
|
378
|
+
## Run (from inside this folder)
|
|
379
|
+
```
|
|
380
|
+
k6 cloud login --token <GRAFANA_CLOUD_TOKEN>
|
|
381
|
+
k6 run protocol/tests/smoke/example.smoke.js # local sanity
|
|
382
|
+
k6 cloud run protocol/tests/load/example.load.js # on Grafana Cloud
|
|
383
|
+
# browser (optional): k6 run browser/tests/exampleUi.browser.js
|
|
384
|
+
```
|
|
385
|
+
EOF
|
|
386
|
+
|
|
387
|
+
echo "Done. Structure created under $ROOT/"
|
|
388
|
+
echo "Next: fill config/env.js + config/secrets.js, then replace the example endpoints/flows."
|
|
@@ -95,6 +95,10 @@ Use `getJiraIssue` to fetch `CARD_ID`. Extract and store:
|
|
|
95
95
|
- `FIGMA_URL_FROM_CARD` — any Figma link found in description or comments
|
|
96
96
|
- `PROJECT_KEY` — for bug filing later
|
|
97
97
|
|
|
98
|
+
Derive `PRODUCT_FOLDER` now (uppercase `PROJECT_KEY`'s product name, spaces → `_` — same
|
|
99
|
+
normalisation qa-agent Step 6a uses) so Phases 3 and 4 below can call the toolkit without
|
|
100
|
+
re-deriving it. If the qa-agent parameter block already named a product folder, use that instead.
|
|
101
|
+
|
|
98
102
|
Run token tracking `jira_fetch` checkpoint.
|
|
99
103
|
|
|
100
104
|
### 1b — Truncate Jira data if card is verbose
|
|
@@ -170,7 +174,18 @@ From the AC, generate numbered test ideas:
|
|
|
170
174
|
- 2+ negative/edge case tests
|
|
171
175
|
- 1+ error state test
|
|
172
176
|
|
|
173
|
-
|
|
177
|
+
**Risk-based ordering** — pull 2–5 module/page keywords straight out of the AC/card title
|
|
178
|
+
you already read (e.g. "login", "checkout"; no extra fetching), then run one script call:
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
node .claude/skills/qa-agent/toolkit/qa-toolkit.cjs risk-score --product {PRODUCT_FOLDER} --modules "kw1,kw2,kw3"
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Order the T-01, T-02... list so tests touching the highest-risk module come first. On a
|
|
185
|
+
product's first run (no git history / no known bugs yet) every module scores 0 — keep the
|
|
186
|
+
natural AC order in that case, no need to mention it.
|
|
187
|
+
|
|
188
|
+
Show as a compact table (T-01, T-02 ... with name and expected outcome), risk-ordered.
|
|
174
189
|
Ask: `"Ready to run these tests? (yes / no or edit)"` — include any missing navigation
|
|
175
190
|
questions in this same message.
|
|
176
191
|
Proceed on confirmation.
|
|
@@ -241,6 +256,21 @@ For each test T-01, T-02, ...:
|
|
|
241
256
|
1. Navigate to the feature area via Playwright MCP (preserves session)
|
|
242
257
|
2. Add current URL to `HINTS.pages` if not already present
|
|
243
258
|
|
|
259
|
+
2a. **DOM fingerprint check — once per distinct URL, the first time you land on it this run:**
|
|
260
|
+
```javascript
|
|
261
|
+
browser_evaluate({ expression: `
|
|
262
|
+
Array.from(document.querySelectorAll('[data-testid]')).map(el => el.getAttribute('data-testid')).join(',')
|
|
263
|
+
` })
|
|
264
|
+
```
|
|
265
|
+
```bash
|
|
266
|
+
node .claude/skills/qa-agent/toolkit/qa-toolkit.cjs fingerprint --product {PRODUCT_FOLDER} --url "CURRENT_URL" --testids "RESULT_FROM_ABOVE"
|
|
267
|
+
```
|
|
268
|
+
- `UNCHANGED` → this page's selectors haven't moved since a prior run. Skip step 3's
|
|
269
|
+
per-interaction element capture for this page entirely (still perform the actual test
|
|
270
|
+
interactions and assertions — only the *hint-recording* is skipped). Add one line to
|
|
271
|
+
`HINTS.notes`: `"Elements on {url}: unchanged — reused prior fingerprint, capture skipped."`
|
|
272
|
+
- `NEW` or `CHANGED` → proceed with full per-interaction capture in step 3 below, same as always.
|
|
273
|
+
|
|
244
274
|
3. For each browser interaction (`browser_fill`, `browser_click`, `browser_select_option`):
|
|
245
275
|
- Execute the interaction
|
|
246
276
|
- Immediately run element capture (zero-token, targeted JS):
|
|
@@ -346,7 +376,7 @@ Run token tracking `test_execution` checkpoint.
|
|
|
346
376
|
|
|
347
377
|
After saving the automation hints file, silently update the product context.
|
|
348
378
|
|
|
349
|
-
|
|
379
|
+
`PRODUCT_FOLDER` was already derived in Phase 1 — reuse it, don't re-derive.
|
|
350
380
|
|
|
351
381
|
```
|
|
352
382
|
CONTEXT_FILE = .claude/skills/qa-agent/product_context/{PRODUCT_FOLDER}/context.md
|
|
@@ -386,6 +416,41 @@ Derive all fields from test execution data — no user input needed:
|
|
|
386
416
|
| Severity | AC explicitly failed → High · Assertion failed → Medium · Observation → Low |
|
|
387
417
|
| Screenshot | Match `outputs/screenshots/T-{N}-*.png` by test ID — use exact filename |
|
|
388
418
|
|
|
419
|
+
**Step 4a.1 — Duplicate check (one script call per failure, no user input needed)**
|
|
420
|
+
|
|
421
|
+
Test names are derived from the AC, not typed by a person, but AC text copied from Jira can
|
|
422
|
+
still contain quotes or symbols — never inline free text into a shell command. Write it with
|
|
423
|
+
the Write tool first, then reference the file:
|
|
424
|
+
|
|
425
|
+
```
|
|
426
|
+
Write outputs/.dupcheck-tmp.txt containing: T-{N}: {test name}
|
|
427
|
+
```
|
|
428
|
+
```bash
|
|
429
|
+
node .claude/skills/qa-agent/toolkit/qa-toolkit.cjs dup-bug --product {PRODUCT_FOLDER} --summary-file outputs/.dupcheck-tmp.txt
|
|
430
|
+
```
|
|
431
|
+
|
|
432
|
+
`NO_DUPLICATE_FOUND` → proceed normally. `POSSIBLE_DUPLICATE: {BUG-ID} (...) — "{title}"` →
|
|
433
|
+
still file the bug (a regression is a real, separately-trackable failure) but prepend one
|
|
434
|
+
line to the Description: `"⚠ Possibly related to {BUG-ID}: {title}"` for the triager.
|
|
435
|
+
|
|
436
|
+
**Step 4a.2 — PII / secrets scan (one script call per bug, before it leaves the machine)**
|
|
437
|
+
|
|
438
|
+
Same rule — the Description includes live-app text (error messages, field values) that can
|
|
439
|
+
contain anything. Write it to a file, never interpolate it into the command:
|
|
440
|
+
|
|
441
|
+
```
|
|
442
|
+
Write outputs/.piicheck-tmp.txt containing the composed Summary + Description text
|
|
443
|
+
```
|
|
444
|
+
```bash
|
|
445
|
+
node .claude/skills/qa-agent/toolkit/qa-toolkit.cjs pii-scan --file outputs/.piicheck-tmp.txt
|
|
446
|
+
```
|
|
447
|
+
|
|
448
|
+
`CLEAN` → proceed silently. `FLAGGED: ...` → this is rare (test data occasionally captures a
|
|
449
|
+
real value) — pause only this one bug, show the flagged pattern types (never the raw match),
|
|
450
|
+
and ask once: `"This bug's description may contain real {types}. Redact and continue, or file as-is? (redact / as-is)"`.
|
|
451
|
+
Apply the user's choice, then continue to the next failure — do not stop the whole batch.
|
|
452
|
+
Delete both `outputs/.dupcheck-tmp.txt` and `outputs/.piicheck-tmp.txt` once the batch is done.
|
|
453
|
+
|
|
389
454
|
**Step 4b — File each bug via Atlassian MCP (parallelise where possible)**
|
|
390
455
|
|
|
391
456
|
For each failure:
|
|
@@ -76,6 +76,8 @@ If the user's message contains `--reset-context`:
|
|
|
76
76
|
Extract the project key prefix from the card ID (`QE-89` → `QE`). Check
|
|
77
77
|
`.claude/skills/qa-agent/product_context/{PREFIX}/context.md`; if not found by exact
|
|
78
78
|
prefix, check whether any folder under `product_context/` starts with that prefix.
|
|
79
|
+
Whichever folder name matches, store it as `PRODUCT_FOLDER` for the rest of this run (Steps
|
|
80
|
+
1d and 3 below use it) — it will be re-derived/confirmed from the Jira project name in Step 6a.
|
|
79
81
|
|
|
80
82
|
**If found:** read it once and store `CTX_APP_URL`, `CTX_LOGIN_URL`, `CTX_OTP`,
|
|
81
83
|
`CTX_ENVIRONMENT`, plus the Covered Flows and Known Bugs tables (needed for Steps 2 and 5).
|
|
@@ -112,6 +114,17 @@ only — qa-pack has no way to force a mid-run model switch, so proceed on whate
|
|
|
112
114
|
active regardless of the user's choice. See `SKILLS_CONTEXT.md` → "Model Routing" for the
|
|
113
115
|
opt-in subagent-override pattern some environments can wire up instead.
|
|
114
116
|
|
|
117
|
+
### 1d — Pre-run Cost Estimate
|
|
118
|
+
|
|
119
|
+
**Only if Step 1a found an existing product folder.** This is a plain script call, not a
|
|
120
|
+
reasoning step — run it and print its one line verbatim, prefixed `"Cost estimate: "`:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
node .claude/skills/qa-agent/toolkit/qa-toolkit.cjs cost-estimate --product {PRODUCT_FOLDER}
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Skip entirely on a product's first run (no context file yet — nothing to estimate from).
|
|
127
|
+
|
|
115
128
|
---
|
|
116
129
|
|
|
117
130
|
## Step 2 — Ask Relevant Questions (gaps only)
|
|
@@ -176,6 +189,23 @@ Set `AUTO_APPROVE = true` only when ALL hold:
|
|
|
176
189
|
Set `AUTO_APPROVE = false` otherwise, or when the user said "manual gates" / "confirm each
|
|
177
190
|
step" / "don't auto-approve", or when `--reset-context` was passed.
|
|
178
191
|
|
|
192
|
+
### Trust Ratchet (only when AUTO_APPROVE would otherwise be false, and a product folder exists)
|
|
193
|
+
|
|
194
|
+
One script call, before showing the phase table:
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
node .claude/skills/qa-agent/toolkit/qa-toolkit.cjs trust-status --product {PRODUCT_FOLDER}
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
- Prints `ELIGIBLE`: add one line to the phase-selection message — "This product has a clean
|
|
201
|
+
approval streak — reply **auto** to skip manual gates for this run." If the user replies
|
|
202
|
+
`auto`, set `AUTO_APPROVE = true` for this run only (does not change future runs or other
|
|
203
|
+
products).
|
|
204
|
+
- Prints `NOT_ELIGIBLE`: say nothing — do not mention trust status at all.
|
|
205
|
+
|
|
206
|
+
The streak itself is recorded by the `automation` skill at each gate (see its SKILL.md) —
|
|
207
|
+
qa-agent only reads the status here.
|
|
208
|
+
|
|
179
209
|
---
|
|
180
210
|
|
|
181
211
|
## Step 4 — Pre-flight Check + Dispatch
|
|
@@ -17,3 +17,19 @@ The QA Agent writes to `{PRODUCT}/context.md` automatically at the end of every
|
|
|
17
17
|
- Known bugs accumulated across all runs
|
|
18
18
|
- Covered test flows
|
|
19
19
|
- Environment notes (URLs, quirks — never credentials; those live only in `.env`)
|
|
20
|
+
|
|
21
|
+
## Other files that may appear in a product folder
|
|
22
|
+
|
|
23
|
+
These are written and read exclusively by `.claude/skills/qa-agent/toolkit/qa-toolkit.cjs`
|
|
24
|
+
— never edited by hand, never read in full by a skill (only queried through the toolkit).
|
|
25
|
+
They exist so features like duplicate-bug detection, DOM-change skipping, and locator
|
|
26
|
+
learning don't have to re-derive their state from `context.md` every run.
|
|
27
|
+
|
|
28
|
+
| File | Written by | Purpose |
|
|
29
|
+
|------|-----------|---------|
|
|
30
|
+
| `dom-fingerprints.json` | `fingerprint` command | One hash per page URL — lets a run skip DOM re-discovery when nothing changed |
|
|
31
|
+
| `trust.json` | `trust-record` command | Gate 1 / Gate 2 clean-approval streaks — feeds the auto-approve trust ratchet |
|
|
32
|
+
| `locator-learnings.md` | `locator-record` command | Locators that broke before, and what fixed them — read before writing new POM code |
|
|
33
|
+
|
|
34
|
+
All three are covered by the same `.claude/skills/qa-agent/` gitignore entry as
|
|
35
|
+
`context.md` — they never leave the machine that generated them.
|