@activate-core/test-forge 1.0.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +27 -2
- package/dist/core/docs/doc-guidance.js +25 -5
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +17 -1
- package/dist/core/generate/ai-generator.js +208 -283
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +166 -236
- package/dist/core/generate/pending.js +122 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +63 -68
- package/dist/core/generate/suite-planner.js +52 -3
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
|
@@ -77,11 +77,6 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
77
77
|
const framework = isApi
|
|
78
78
|
? 'Jest for an API/backend test'
|
|
79
79
|
: 'Playwright for a web/UI test';
|
|
80
|
-
// Pending-test syntax differs by runner: Jest has test.todo/it.skip;
|
|
81
|
-
// Playwright has NEITHER (test.todo throws) — it uses test.fixme/skip.
|
|
82
|
-
const pending = isApi
|
|
83
|
-
? 'test.todo(...) or it.skip(...)'
|
|
84
|
-
: 'test.fixme(title, async () => {}) or test.skip(title, async () => {}) — Playwright has NO test.todo/it.*';
|
|
85
80
|
const ext = config.testExtension ?? '.test.ts';
|
|
86
81
|
const isJs = ext.endsWith('.js');
|
|
87
82
|
const language = isJs
|
|
@@ -101,248 +96,170 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
101
96
|
`useful). For payment/webhook journeys, test idempotency, amount/`,
|
|
102
97
|
`price tampering, and signature/verification where relevant.`,
|
|
103
98
|
``,
|
|
104
|
-
`IMPORTANT — do not presume the app is vulnerable. For every security`,
|
|
105
|
-
`probe, assert the SECURE expectation (e.g. auth bypass returns 401/403,`,
|
|
106
|
-
`a tampered amount is rejected, a duplicate charge is idempotent). Only`,
|
|
107
|
-
`assert that a vulnerability exists when the journey doc EXPLICITLY`,
|
|
108
|
-
`documents it as a known/accepted behavior. If you cannot tell from the`,
|
|
109
|
-
`journey or map whether a defense exists, write the check as`,
|
|
110
|
-
`${pending} with a "// TODO:" note — never assume a`,
|
|
111
|
-
`guard is missing and produce a failing test that falsely brands the`,
|
|
112
|
-
`product as vulnerable.`,
|
|
113
|
-
``,
|
|
114
99
|
`PRIME DIRECTIVE — A FAILING TEST MUST MEAN A REAL PRODUCT DEFECT.`,
|
|
115
|
-
`Teams hold releases on what these tests report. A test that fails for`,
|
|
116
|
-
`
|
|
117
|
-
`
|
|
118
|
-
`
|
|
119
|
-
`
|
|
120
|
-
|
|
121
|
-
`
|
|
122
|
-
`
|
|
123
|
-
`
|
|
124
|
-
`
|
|
125
|
-
`
|
|
126
|
-
`
|
|
127
|
-
`
|
|
128
|
-
|
|
129
|
-
`
|
|
130
|
-
`
|
|
131
|
-
`
|
|
132
|
-
`
|
|
133
|
-
`
|
|
134
|
-
`
|
|
135
|
-
`
|
|
136
|
-
`
|
|
137
|
-
`
|
|
138
|
-
`
|
|
139
|
-
`
|
|
140
|
-
`
|
|
141
|
-
`
|
|
142
|
-
`
|
|
143
|
-
`
|
|
144
|
-
`
|
|
145
|
-
`
|
|
146
|
-
`
|
|
147
|
-
`
|
|
148
|
-
`
|
|
149
|
-
`
|
|
150
|
-
`
|
|
151
|
-
`
|
|
152
|
-
`
|
|
153
|
-
`
|
|
154
|
-
`
|
|
155
|
-
`
|
|
156
|
-
`
|
|
157
|
-
`
|
|
158
|
-
`
|
|
159
|
-
`
|
|
160
|
-
`
|
|
161
|
-
`
|
|
162
|
-
`
|
|
163
|
-
`
|
|
164
|
-
`
|
|
165
|
-
`
|
|
166
|
-
`
|
|
167
|
-
`
|
|
168
|
-
`
|
|
169
|
-
`
|
|
170
|
-
`
|
|
171
|
-
`
|
|
172
|
-
`
|
|
173
|
-
`
|
|
174
|
-
`
|
|
175
|
-
`
|
|
176
|
-
`
|
|
177
|
-
`
|
|
178
|
-
`
|
|
179
|
-
`
|
|
180
|
-
`
|
|
181
|
-
`
|
|
182
|
-
`
|
|
183
|
-
`
|
|
184
|
-
`
|
|
185
|
-
`
|
|
186
|
-
`
|
|
187
|
-
`
|
|
188
|
-
`
|
|
189
|
-
`
|
|
190
|
-
`
|
|
191
|
-
`
|
|
192
|
-
`
|
|
193
|
-
`
|
|
194
|
-
`
|
|
195
|
-
`
|
|
196
|
-
`
|
|
197
|
-
`
|
|
198
|
-
`
|
|
199
|
-
`
|
|
200
|
-
`
|
|
201
|
-
`
|
|
202
|
-
`
|
|
203
|
-
`
|
|
204
|
-
`
|
|
205
|
-
`
|
|
206
|
-
`
|
|
207
|
-
`
|
|
208
|
-
`
|
|
209
|
-
`
|
|
210
|
-
`
|
|
211
|
-
`
|
|
212
|
-
`
|
|
213
|
-
`
|
|
214
|
-
`
|
|
215
|
-
`
|
|
216
|
-
`
|
|
217
|
-
`
|
|
218
|
-
`
|
|
219
|
-
`
|
|
220
|
-
`
|
|
221
|
-
`
|
|
222
|
-
`
|
|
223
|
-
`
|
|
224
|
-
`
|
|
225
|
-
`
|
|
226
|
-
`
|
|
227
|
-
`
|
|
228
|
-
`
|
|
229
|
-
`
|
|
230
|
-
`
|
|
231
|
-
`
|
|
232
|
-
`
|
|
233
|
-
`
|
|
234
|
-
`
|
|
235
|
-
`
|
|
236
|
-
`
|
|
237
|
-
`
|
|
238
|
-
`
|
|
239
|
-
`
|
|
240
|
-
`
|
|
241
|
-
`
|
|
242
|
-
`
|
|
243
|
-
`
|
|
244
|
-
`
|
|
245
|
-
`
|
|
246
|
-
`
|
|
247
|
-
`
|
|
248
|
-
`
|
|
249
|
-
`
|
|
250
|
-
`
|
|
251
|
-
|
|
252
|
-
`
|
|
253
|
-
`
|
|
254
|
-
`
|
|
255
|
-
`
|
|
256
|
-
`
|
|
257
|
-
`
|
|
258
|
-
`
|
|
259
|
-
`
|
|
260
|
-
`
|
|
261
|
-
` The same
|
|
262
|
-
`
|
|
263
|
-
`
|
|
264
|
-
`
|
|
265
|
-
`
|
|
266
|
-
`
|
|
267
|
-
`
|
|
268
|
-
`
|
|
269
|
-
`
|
|
270
|
-
`
|
|
271
|
-
`
|
|
272
|
-
|
|
273
|
-
`
|
|
274
|
-
`
|
|
275
|
-
`
|
|
276
|
-
`
|
|
277
|
-
`
|
|
278
|
-
` CLEAN UP BY A HANDLE THAT CANNOT GO MISSING. If cleanup matches a`,
|
|
279
|
-
` name prefix, check that every record the file creates can actually`,
|
|
280
|
-
` carry that prefix — validation minimums, enum values and generated`,
|
|
281
|
-
` identifiers routinely forbid it. One record the filter cannot see`,
|
|
282
|
-
` survives, and the next run collides with its own leftovers and`,
|
|
283
|
-
` reports the collision as a product error. Prefer an owning field`,
|
|
284
|
-
` that is always set (a parent, an owner id, a namespace).`,
|
|
285
|
-
`(12) A ROLE'S CREDENTIAL MUST BE THAT ROLE'S — NEVER FALL THROUGH TO`,
|
|
286
|
-
` ANOTHER ONE. When a test needs a specific actor (an admin, a staff`,
|
|
287
|
-
` user, a second customer) and that actor's credential is missing or`,
|
|
288
|
-
` expired, the answer is to SKIP, not to reach for whichever token`,
|
|
289
|
-
` the harness happens to hold. A live token for the wrong actor`,
|
|
290
|
-
` passes every "is it present / is it in date" check and then gets`,
|
|
291
|
-
` 401/403 from every route the real actor is allowed to use — which`,
|
|
292
|
-
` reads exactly like the product refusing legitimate access. This is`,
|
|
293
|
-
` the single most expensive mistake available to you: one fallback in`,
|
|
294
|
-
` a shared helper turns a whole area's suite into false defects, and`,
|
|
295
|
-
` nothing in the output says the wrong identity was used. So: resolve`,
|
|
296
|
-
` each role from its OWN source, never chain roles in a fallback`,
|
|
297
|
-
` list, and treat a template placeholder (DUMMY, replace_me, an empty`,
|
|
298
|
-
` value, a bracketed name) as absent rather than as a credential.`,
|
|
299
|
-
` Say which actor was missing in the skip reason.`,
|
|
300
|
-
`(13) A [baseline] CHECK PROVES ROBUSTNESS, NOT A VALUE. "Opens without a`,
|
|
301
|
-
` crash" is proved by the page loading with no error and no 5xx — not by`,
|
|
302
|
-
` an exact URL, trailing slash, heading text or element the document`,
|
|
303
|
-
` does not name. Never add an extra condition the criterion does not`,
|
|
304
|
-
` state (e.g. "double-click opens exactly one pop-up" when the document`,
|
|
305
|
-
` only says the button opens it), and never assert data in a place the`,
|
|
306
|
-
` page does not render it — check where the app shows it first.`,
|
|
307
|
-
`(14) SCOPE EVERY LOCATOR TO THE JOURNEY'S OWN UI. Global components`,
|
|
308
|
-
` (a login pop-up, a header, a cookie bar) share ids and labels with`,
|
|
309
|
-
` the page under test — "#password" can match the login modal's field.`,
|
|
310
|
-
` Locate inside the journey's form/container, never page-wide by a`,
|
|
311
|
-
` bare id or label that another always-mounted component also uses.`,
|
|
312
|
-
`(15) NOTHING YOU ARE NOT TESTING MAY BLOCK YOU. Apps open their own`,
|
|
313
|
-
` pop-ups for a signed-in or new user (promotions, cookie consent,`,
|
|
314
|
-
` onboarding tips, notification prompts). Use the harness's login /`,
|
|
315
|
-
` storage helpers that pre-suppress them; if one still opens, close it`,
|
|
316
|
-
` before you continue. A click intercepted by an overlay the test is`,
|
|
317
|
-
` not about is your bug, not the product's.`,
|
|
318
|
-
`(16) "INSIDE" MEANS THE MIDDLE. To click inside a panel, click its centre`,
|
|
319
|
-
` or a non-interactive child — never a corner offset like (5,5): with a`,
|
|
320
|
-
` border-radius that point is outside the element and lands on the`,
|
|
321
|
-
` overlay behind it.`,
|
|
322
|
-
`(17) LET THE APP SETTLE BEFORE YOU CHANGE THE SESSION. After a login,`,
|
|
323
|
-
` sign-out or navigation the app sends its own requests and may`,
|
|
324
|
-
` redirect by itself. Wait for the network to go idle before rotating`,
|
|
325
|
-
` a session, logging in elsewhere or navigating away, and assert the`,
|
|
326
|
-
` URL the APP chooses — never force a navigation that can cut off the`,
|
|
327
|
-
` app's own redirect.`,
|
|
328
|
-
`(18) ONE SOURCE FOR SEEDED IDS. Read every seeded id and handle from the`,
|
|
329
|
-
` harness (TEST_IDS / the values setup() publishes). If a test needs a`,
|
|
330
|
-
` private account, override EVERY id that belongs to it, never a`,
|
|
331
|
-
` subset — a leftover shared id makes you compare against another`,
|
|
332
|
-
` customer's row.`,
|
|
333
|
-
`(19) VERIFICATION CODES THE TEST CANNOT READ (SMS/email OTP, magic`,
|
|
334
|
-
` links, OAuth). Drive the real UI up to the code step, then use the`,
|
|
335
|
-
` harness's bypassVerification() so the final verify/register request`,
|
|
336
|
-
` is answered with a REAL account and a REAL token; everything after it`,
|
|
337
|
-
` (logged-in state, redirect, payment) is then tested for real. Put`,
|
|
338
|
-
` "(verification bypassed)" in that test's title. Wrong-code and`,
|
|
339
|
-
` expired-code tests do NOT bypass: the real server must refuse them.`,
|
|
340
|
-
`(20) CHECK STATE THROUGH WHAT THE HARNESS OFFERS. If the harness has`,
|
|
341
|
-
` no database connection (the data lives in another repository), prove`,
|
|
342
|
-
` persisted state through the app's own API — the list or detail`,
|
|
343
|
-
` endpoint the client reads — never by importing a database client.`,
|
|
344
|
-
`When rules conflict, prefer reporting LESS: a pending test costs a line`,
|
|
345
|
-
`in the report; a false failure costs the team a release.`,
|
|
100
|
+
`Teams hold releases on what these tests report. A test that fails for any`,
|
|
101
|
+
`reason other than the product being wrong is worse than no test at all: it`,
|
|
102
|
+
`burns the team's trust, and the next real defect gets waved through as`,
|
|
103
|
+
`"probably the tool again". So a test may FAIL only when the product provably`,
|
|
104
|
+
`contradicts an expectation the journey document states.`,
|
|
105
|
+
``,
|
|
106
|
+
`THE METHOD — EVERY LINE OF A TEST NEEDS EVIDENCE. A test is a chain of`,
|
|
107
|
+
`claims: "this is what should happen", "this is the element", "this account`,
|
|
108
|
+
`exists", "now is the moment to look". Each claim is either proven by evidence`,
|
|
109
|
+
`you can point to — the journey document, this project's source, the harness,`,
|
|
110
|
+
`a response you actually received — or it is a guess, and a guess is how a`,
|
|
111
|
+
`test fails on a correct product. Before you write each line, find its`,
|
|
112
|
+
`evidence. These five kinds of evidence cover every line you will write.`,
|
|
113
|
+
``,
|
|
114
|
+
`E1. WHAT YOU ASSERT — evidence: a line in the journey document.`,
|
|
115
|
+
` - Find the exact line that fixes the expected outcome and cite it in a`,
|
|
116
|
+
` short comment (e.g. "// oracle: §4 total = fee + shipping"). Criteria`,
|
|
117
|
+
` tagged [spec], [baseline] or [observed] are stated oracles. Look properly`,
|
|
118
|
+
` before you conclude one is absent: it may sit in another section, in a §`,
|
|
119
|
+
` the text cross-references, or in an acceptance list. "Test setup" says`,
|
|
120
|
+
` how to establish preconditions — follow it.`,
|
|
121
|
+
` - Assert what the line states, at the loosest form that still proves it.`,
|
|
122
|
+
` Add no condition it does not state, and no exact copy, ordering, id,`,
|
|
123
|
+
` timestamp or count it does not fix. A [baseline] robustness line ("opens`,
|
|
124
|
+
` without a crash") is proved by the page loading with no error and no`,
|
|
125
|
+
` 5xx, not by a URL, heading or element it never names. "The customer is`,
|
|
126
|
+
` told why" is proved by a non-empty message VISIBLE where the app shows`,
|
|
127
|
+
` it, unless the line fixes the words. Check where the page actually`,
|
|
128
|
+
` renders a value before asserting it there.`,
|
|
129
|
+
` - Assert the real end state (the response body and status, the`,
|
|
130
|
+
` persisted record, the screen) — not merely that nothing threw, and`,
|
|
131
|
+
` never a value your own stub hard-coded.`,
|
|
132
|
+
` - No line, no assertion: a case the document fixes no outcome for gets`,
|
|
133
|
+
` NO test — not a placeholder either. Neither does an item under`,
|
|
134
|
+
` "Questions for product (not asserted)" or "Suggested", or one the`,
|
|
135
|
+
` document says is not tested through the screen: it is the product's`,
|
|
136
|
+
` to answer.`,
|
|
137
|
+
` - A security probe asserts the SECURE outcome (refused, rejected,`,
|
|
138
|
+
` idempotent) unless the document records the weakness as accepted; when`,
|
|
139
|
+
` you cannot tell whether a defence exists, the case is left out.`,
|
|
140
|
+
`E2. WHAT YOU TOUCH — evidence: this project's source.`,
|
|
141
|
+
` - Every locator, route, request field, response field, storage key and id`,
|
|
142
|
+
` is something you found in THIS project's code (or the CODE FACTS), never`,
|
|
143
|
+
` something that usually exists.`,
|
|
144
|
+
` - A locator resolves to exactly ONE node — the one you mean. Components`,
|
|
145
|
+
` often label a control twice, and always-mounted components (a login`,
|
|
146
|
+
` pop-up, a header) share ids and labels with the page. Scope to the`,
|
|
147
|
+
` journey's own container, after confirming in the component source that`,
|
|
148
|
+
` the element is rendered INSIDE it: a footer link or a resend control`,
|
|
149
|
+
` often sits beside a form, not in it — then scope to the parent of both.`,
|
|
150
|
+
` "Inside" a panel means its centre or a non-interactive child, never a`,
|
|
151
|
+
` corner offset that a rounded border puts outside it. "Outside" means a`,
|
|
152
|
+
` point on the backdrop itself, clear of the panel and away from the`,
|
|
153
|
+
` window's edges, where a scrollbar or other chrome may sit.`,
|
|
154
|
+
` - A response field is one the server's serializer for THAT endpoint`,
|
|
155
|
+
` sends, not a name from the client's types for another endpoint.`,
|
|
156
|
+
` - Ids come from the harness (TEST_IDS and what setup() publishes); a`,
|
|
157
|
+
` private account overrides every id that belongs to it, never a subset.`,
|
|
158
|
+
` A dynamic page is keyed on an id the served build can render — some`,
|
|
159
|
+
` builds serve only a listed set of parameters.`,
|
|
160
|
+
` - Read a harness helper before calling it, to see what it really returns.`,
|
|
161
|
+
` - Other test files are NOT evidence. They were written the same way`,
|
|
162
|
+
` as yours and can carry the same mistake, so copying one spreads it.`,
|
|
163
|
+
` Where the harness offers a helper, use it. Take anything from a`,
|
|
164
|
+
` sibling file only after checking it against the source exactly as`,
|
|
165
|
+
` you would check your own line.`,
|
|
166
|
+
`E3. WHAT YOU NEED FIRST — evidence: the harness, or the product's own way of`,
|
|
167
|
+
` making it.`,
|
|
168
|
+
` - Accounts, records, settings and codes are established through the`,
|
|
169
|
+
` harness, or the way the product would: follow the transition that`,
|
|
170
|
+
` produces a state and copy what it writes; satisfy the schema (every NOT`,
|
|
171
|
+
` NULL column, the real type of each id). A record no code path could`,
|
|
172
|
+
` produce proves nothing.`,
|
|
173
|
+
` - Look for a precondition the way you would look for a bug before calling`,
|
|
174
|
+
` it missing: another env name, an identity the harness already has, one`,
|
|
175
|
+
` seed call away, a service on another port.`,
|
|
176
|
+
` - A credential that exists is not one that works — confirm it is`,
|
|
177
|
+
` accepted. Each role uses its OWN credential; never fall through to`,
|
|
178
|
+
` another role's token, and treat a placeholder (DUMMY, replace_me, empty,`,
|
|
179
|
+
` <bracketed>) as absent. Bind fixtures to the identity the screen is`,
|
|
180
|
+
` scoped by — the key the server scopes on, not an identity provider's`,
|
|
181
|
+
` handle.`,
|
|
182
|
+
` - A code the test cannot read (message or email code, magic link,`,
|
|
183
|
+
` third-party sign-in): drive the real UI to the code step, then let the`,
|
|
184
|
+
` harness's bypassVerification() answer only that step with a real account`,
|
|
185
|
+
` and a real token; put "(verification bypassed)" in the title. Wrong- and`,
|
|
186
|
+
` expired-code tests do not bypass: the real server must refuse them.`,
|
|
187
|
+
` - Where the harness has no database client, prove persisted state through`,
|
|
188
|
+
` the app's own API.`,
|
|
189
|
+
` - If it genuinely cannot be established, SKIP at the top of the test with`,
|
|
190
|
+
` a reason stating what you observed ("GET /x answered 404", "TEST_X is not`,
|
|
191
|
+
` set", "no answer after 20 s") — never a guessed cause, and never let a`,
|
|
192
|
+
` missing precondition reach an assertion.`,
|
|
193
|
+
`E4. WHAT YOU WAIT FOR — evidence: the event that actually happens.`,
|
|
194
|
+
` - Wait for a specific condition, never a fixed sleep, and prefer a`,
|
|
195
|
+
` retrying assertion: a value written after hydration or a round-trip is`,
|
|
196
|
+
` not there at the first look. A page-side read that dies because a`,
|
|
197
|
+
` redirect landed mid-poll is retried, not reported.`,
|
|
198
|
+
` - A page whose job is to redirect never fires load on the URL you asked`,
|
|
199
|
+
` for: wait for the response or the outcome. A control hidden from the`,
|
|
200
|
+
` accessibility tree is not there for a role query. Hold a response/request`,
|
|
201
|
+
` wait BEFORE the action that triggers it.`,
|
|
202
|
+
` - After a login, sign-out or navigation, let the app's own requests and`,
|
|
203
|
+
` redirects settle before you change the session, and assert the URL the`,
|
|
204
|
+
` APP chooses.`,
|
|
205
|
+
` - The first render after a navigation is a precondition: wait for the`,
|
|
206
|
+
` control you will use, with a timeout sized to what the app loads first.`,
|
|
207
|
+
` A page that never renders is reported as that, never as the behaviour`,
|
|
208
|
+
` under test failing. "It timed out" is never the finding.`,
|
|
209
|
+
`E5. WHAT ELSE IS RUNNING — evidence: you checked, and your test does not`,
|
|
210
|
+
` depend on it.`,
|
|
211
|
+
` - The app's own pop-ups (promotions, consent, onboarding): use the`,
|
|
212
|
+
` harness helpers that suppress them on every login path, and close any`,
|
|
213
|
+
` that still opens. A click intercepted by something the test is not`,
|
|
214
|
+
` about is the test's bug.`,
|
|
215
|
+
` - Other tests: no dependence on order, or on data another test creates;`,
|
|
216
|
+
` assert only on records YOU set up, by id. A serial block only when a`,
|
|
217
|
+
` test consumes state an earlier one in it created — otherwise one`,
|
|
218
|
+
` failure silently skips the rest.`,
|
|
219
|
+
` - Shared state (a shared account, the one seeded record): read it first`,
|
|
220
|
+
` and restore it in a finally or after-EACH hook, never after-all. Clean`,
|
|
221
|
+
` up by a handle that cannot go missing (an owning field always set), not`,
|
|
222
|
+
` a name prefix validation may forbid.`,
|
|
223
|
+
` - A stub intercepts exactly the dependency — a different host; the same`,
|
|
224
|
+
` host on another port; a path prefix; or a predicate on the request —`,
|
|
225
|
+
` never a URL substring that also matches the app's own pages; then`,
|
|
226
|
+
` confirm the page under test still loads. A dependency you truly cannot`,
|
|
227
|
+
` separate is left untested; say which of these you checked in a comment.`,
|
|
228
|
+
` - Real side effects (a message, an email, a charge) sit behind the opt-in`,
|
|
229
|
+
` gate the sibling files use, with one identity per rate-limited send.`,
|
|
230
|
+
` - A login elsewhere can end a cached session: get tokens through the`,
|
|
231
|
+
` harness accessors that re-confirm them.`,
|
|
232
|
+
` - Negative input must really be negative: a "bad" credential, signature,`,
|
|
233
|
+
` id or payload differs from the valid one at the level the product checks`,
|
|
234
|
+
` (change decoded bytes, not an encoded character), and the valid control`,
|
|
235
|
+
` is asserted too where you can.`,
|
|
236
|
+
``,
|
|
237
|
+
`COVER EACH OUTCOME ONCE. A suite is judged by the defects it can catch, not`,
|
|
238
|
+
`by its size; every redundant test costs run time and buries the ones that`,
|
|
239
|
+
`matter.`,
|
|
240
|
+
` - One test per distinct outcome: each criterion, and each branch or`,
|
|
241
|
+
` boundary the code takes for it (just inside, just outside). Inputs that`,
|
|
242
|
+
` reach the same branch and give the same outcome are ONE case: thirty`,
|
|
243
|
+
` attack strings against a field that only checks "not empty" prove`,
|
|
244
|
+
` nothing the first did not. A parametrised test lists only rows that`,
|
|
245
|
+
` reach different branches, each with a short comment naming its branch.`,
|
|
246
|
+
` The same goes for repeating one check per field, per page, per opener`,
|
|
247
|
+
` or per error code when the same code handles them all.`,
|
|
248
|
+
` - A behaviour the "covers" line of this file says another journey owns is`,
|
|
249
|
+
` not re-tested here: check only how the customer reaches it from this`,
|
|
250
|
+
` journey, and the outcome specific to this journey.`,
|
|
251
|
+
` - Do not assert implementation details — storage key names, CSS classes,`,
|
|
252
|
+
` internal flags, request field names — unless the criterion itself is`,
|
|
253
|
+
` about them (for example, a token must never appear in a URL).`,
|
|
254
|
+
` - Never write a placeholder: no test.fixme / test.skip / test.todo`,
|
|
255
|
+
` declarations. A precondition the environment lacks is a run-time skip`,
|
|
256
|
+
` inside a real test, stating what was observed.`,
|
|
257
|
+
``,
|
|
258
|
+
`SELF-CHECK BEFORE YOU OUTPUT. Go through every test you wrote and name, for`,
|
|
259
|
+
`each line, its evidence under E1–E5, and the distinct outcome each test`,
|
|
260
|
+
`adds. A line with no evidence is fixed, or its test is removed; a test that`,
|
|
261
|
+
`adds no new outcome is removed. When in doubt, report LESS: a missing test`,
|
|
262
|
+
`costs one line of coverage; a false failure costs the team a release.`,
|
|
346
263
|
``,
|
|
347
264
|
...(grounding
|
|
348
265
|
? [
|
|
@@ -377,9 +294,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
377
294
|
`and let the test FAIL — a real bug surfaced is the goal, never`,
|
|
378
295
|
`something to paper over. NEVER reshape an assertion to match observed`,
|
|
379
296
|
`code behaviour just to make it green; that rubber-stamps the bug. If`,
|
|
380
|
-
`the document does not state the expected outcome for a case,
|
|
381
|
-
`
|
|
382
|
-
`expectation from the code.`,
|
|
297
|
+
`the document does not state the expected outcome for a case, write no`,
|
|
298
|
+
`test for it — do not invent the expectation from the code.`,
|
|
383
299
|
`REQUEST URLS: build every request path as the FULL mounted route the`,
|
|
384
300
|
`router actually registers — INCLUDING the version/area prefix (e.g.`,
|
|
385
301
|
`/v{VERSION}/<area>/<action>, reading the real version from its env var).`,
|
|
@@ -393,18 +309,12 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
393
309
|
`fallback overrides the token for an AUTHENTICATED call: the testkit`,
|
|
394
310
|
`seeds the owner fixtures under the authenticated identity, so expect`,
|
|
395
311
|
`the token owner's rows. A second, non-owner identity (for IDOR/scope`,
|
|
396
|
-
`tests) needs a distinct token; if unavailable,
|
|
312
|
+
`tests) needs a distinct token; if unavailable, the test skips at run time`,
|
|
313
|
+
`naming the missing identity.`,
|
|
397
314
|
`Do NOT write or edit any file; output only the test code to stdout.`,
|
|
398
|
-
`EVERY
|
|
399
|
-
`
|
|
400
|
-
|
|
401
|
-
`testing anything.`,
|
|
402
|
-
`ASSERTION DEPTH: assert the REAL end-state — persisted DB/state, the`,
|
|
403
|
-
`actual response body/status, or observable UI — not merely that "no`,
|
|
404
|
-
`error was thrown". CRITICAL: if you mock an external/API response, do`,
|
|
405
|
-
`NOT then "verify" a value your own mock hard-coded (asserting the mock`,
|
|
406
|
-
`returned what you told it to proves nothing). Read back the real effect`,
|
|
407
|
-
`(DB row, downstream call, UI change) the behaviour was supposed to have.`,
|
|
315
|
+
`EVERY test MUST contain at least one real assertion. If you genuinely`,
|
|
316
|
+
`cannot assert an outcome, write no test for that case — never a test body`,
|
|
317
|
+
`with no assertion, which passes without testing anything.`,
|
|
408
318
|
``,
|
|
409
319
|
isApi
|
|
410
320
|
? [
|
|
@@ -453,9 +363,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
453
363
|
`ground on role/label/text/name instead of inventing a test-id.`,
|
|
454
364
|
`NEVER invent a data-testid/data-cy: an unverified test-id is`,
|
|
455
365
|
`REJECTED by the selector-grounding gate and wastes the whole file.`,
|
|
456
|
-
`Only when an element cannot be identified by ANY real handle
|
|
457
|
-
`
|
|
458
|
-
`"// TODO: confirm selector" note — do not default to TODO, never guess.`,
|
|
366
|
+
`Only when an element cannot be identified by ANY real handle, leave`,
|
|
367
|
+
`that single case out and name the control in a comment — never guess.`,
|
|
459
368
|
`NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
|
|
460
369
|
`never a description/comment, and never a route-file placeholder`,
|
|
461
370
|
`like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
|
|
@@ -465,8 +374,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
465
374
|
`CRITICAL Playwright API: import { test, expect } from`,
|
|
466
375
|
`'@playwright/test'. There is NO test.todo / it.* / describe.skip`,
|
|
467
376
|
`in Playwright — using them throws at collection and breaks the`,
|
|
468
|
-
`WHOLE file.
|
|
469
|
-
`{}) or test.skip(...); group with test.describe(...).`,
|
|
377
|
+
`WHOLE file. Group with test.describe(...).`,
|
|
470
378
|
`CRITICAL API-request API: the \`request\` FIXTURE (test args`,
|
|
471
379
|
`\`async ({ request }) => …\`) IS already an APIRequestContext —`,
|
|
472
380
|
`call it DIRECTLY: \`await request.post(url, { data })\`,`,
|
|
@@ -522,8 +430,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
522
430
|
`- Do NOT ask for approval or describe what you are doing.`,
|
|
523
431
|
`- Do NOT wrap the code in markdown fences.`,
|
|
524
432
|
`- Tag the top with a comment: // @generated by testforge`,
|
|
525
|
-
`-
|
|
526
|
-
`
|
|
433
|
+
`- Never write "TODO" or "FIXME" in the file. Something you cannot`,
|
|
434
|
+
` establish is left out, never a note to finish later.`,
|
|
527
435
|
``,
|
|
528
436
|
`--- PROJECT MAP (data layer) ---`,
|
|
529
437
|
map,
|
|
@@ -548,7 +456,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
548
456
|
`--- JOURNEY ---`,
|
|
549
457
|
`It may include sections beyond fields — actions, expected results,`,
|
|
550
458
|
`negative cases, race conditions, webhooks, security notes, and a`,
|
|
551
|
-
`Data section. Use ALL of it
|
|
459
|
+
`Data section. Use ALL of it.`,
|
|
552
460
|
``,
|
|
553
461
|
journey.rawMarkdown,
|
|
554
462
|
// ---- PER-FILE TASK (kept LAST so the big context above stays a stable,
|
|
@@ -612,6 +520,8 @@ export function generateWithAI(journey, testPath, focus, grounding) {
|
|
|
612
520
|
* journey's suite can be generated concurrently (each spawns its own
|
|
613
521
|
* `claude` process). Same prompt, same read-only exploration.
|
|
614
522
|
*/
|
|
523
|
+
/** Marks a test whose failure the repair judged to be the application's. */
|
|
524
|
+
export const DEFECT_MARK = 'testforge-defect:';
|
|
615
525
|
/**
|
|
616
526
|
* Repair a generated test from the runner's own failure output.
|
|
617
527
|
*
|
|
@@ -628,26 +538,41 @@ export async function repairWithAIAsync(journey, testPath, current, failure, har
|
|
|
628
538
|
'',
|
|
629
539
|
'=== REPAIR ===',
|
|
630
540
|
`The file below was generated for ${testPath} and RUN against the real`,
|
|
631
|
-
'application.
|
|
632
|
-
'you what the application actually
|
|
541
|
+
'application. The tests listed under RUN RESULT did not pass. The runner',
|
|
542
|
+
'output is the ground truth: it tells you what the application actually',
|
|
543
|
+
'renders, routes and returns.',
|
|
633
544
|
'',
|
|
634
|
-
'
|
|
545
|
+
'Decide, for EACH listed test, which of these it is — from the evidence,',
|
|
546
|
+
'not from what you hope:',
|
|
547
|
+
' TEST FAULT — the test never reached the behaviour it checks, or checked',
|
|
548
|
+
' it wrongly: a locator that matched nothing or the wrong',
|
|
549
|
+
' element, a container that does not hold the element, a wait',
|
|
550
|
+
' for the wrong event, a response field the server does not',
|
|
551
|
+
' send, a precondition its own setup got wrong, an assertion',
|
|
552
|
+
' the journey document does not state. FIX the test.',
|
|
553
|
+
' APP DEFECT — the test reached the behaviour (the element or response it',
|
|
554
|
+
' checks is there) and the application contradicts a criterion',
|
|
555
|
+
' the journey document states. Change NOTHING in that test.',
|
|
556
|
+
` Put one comment line directly above its declaration:`,
|
|
557
|
+
` // ${DEFECT_MARK} <the criterion, quoted with its tag> — <what the app did instead>`,
|
|
558
|
+
' SKIPPED — a test listed under SKIPPED skipped itself. If its own',
|
|
559
|
+
' setup looked in the wrong place (wrong field, wrong route,',
|
|
560
|
+
' wrong id), fix that so it runs. If what it needs truly does',
|
|
561
|
+
' not exist in this environment, leave it exactly as it is.',
|
|
635
562
|
'',
|
|
636
|
-
'Rules
|
|
637
|
-
' 1.
|
|
638
|
-
'
|
|
639
|
-
'
|
|
640
|
-
'
|
|
641
|
-
'
|
|
642
|
-
'
|
|
643
|
-
'
|
|
644
|
-
'
|
|
645
|
-
' the journey document), keep the assertion exactly as it is and change',
|
|
646
|
-
' nothing — a real defect must stay caught.',
|
|
563
|
+
'Rules, in addition to every rule above:',
|
|
564
|
+
' 1. Never delete, rename, weaken or invert an assertion, and never delete',
|
|
565
|
+
' or rename a test.',
|
|
566
|
+
' 2. Never add test.skip / test.fixme / try-catch to make a failure go away.',
|
|
567
|
+
' 3. Do not touch a test that is not listed below.',
|
|
568
|
+
' 4. When in doubt between TEST FAULT and APP DEFECT, look again at the',
|
|
569
|
+
' component and server source: an element that is in the DOM but hidden,',
|
|
570
|
+
' a message the server sends but the screen does not show, a value the',
|
|
571
|
+
' screen accepts but the server refuses — those are the application.',
|
|
647
572
|
'',
|
|
648
573
|
'--- CURRENT FILE ---',
|
|
649
574
|
current,
|
|
650
|
-
'---
|
|
575
|
+
'--- RUN RESULT ---',
|
|
651
576
|
failure,
|
|
652
577
|
'--- END ---',
|
|
653
578
|
'',
|