@activate-core/test-forge 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +29 -11
- package/dist/core/docs/doc-guidance.js +10 -0
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +12 -1
- package/dist/core/generate/ai-generator.js +181 -267
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +181 -226
- package/dist/core/generate/pending.js +118 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +70 -73
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
|
@@ -81,7 +81,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
81
81
|
// Playwright has NEITHER (test.todo throws) — it uses test.fixme/skip.
|
|
82
82
|
const pending = isApi
|
|
83
83
|
? 'test.todo(...) or it.skip(...)'
|
|
84
|
-
:
|
|
84
|
+
: "test.fixme(title, { annotation: { type: 'fixme', description: '<why it is pending>' } }, async () => {}) — the reason goes in the annotation, where the report shows it; Playwright has NO test.todo/it.*";
|
|
85
85
|
const ext = config.testExtension ?? '.test.ts';
|
|
86
86
|
const isJs = ext.endsWith('.js');
|
|
87
87
|
const language = isJs
|
|
@@ -101,244 +101,147 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
101
101
|
`useful). For payment/webhook journeys, test idempotency, amount/`,
|
|
102
102
|
`price tampering, and signature/verification where relevant.`,
|
|
103
103
|
``,
|
|
104
|
-
`IMPORTANT — do not presume the app is vulnerable. For every security`,
|
|
105
|
-
`probe, assert the SECURE expectation (e.g. auth bypass returns 401/403,`,
|
|
106
|
-
`a tampered amount is rejected, a duplicate charge is idempotent). Only`,
|
|
107
|
-
`assert that a vulnerability exists when the journey doc EXPLICITLY`,
|
|
108
|
-
`documents it as a known/accepted behavior. If you cannot tell from the`,
|
|
109
|
-
`journey or map whether a defense exists, write the check as`,
|
|
110
|
-
`${pending} with a "// TODO:" note — never assume a`,
|
|
111
|
-
`guard is missing and produce a failing test that falsely brands the`,
|
|
112
|
-
`product as vulnerable.`,
|
|
113
|
-
``,
|
|
114
104
|
`PRIME DIRECTIVE — A FAILING TEST MUST MEAN A REAL PRODUCT DEFECT.`,
|
|
115
|
-
`Teams hold releases on what these tests report. A test that fails for`,
|
|
116
|
-
`
|
|
117
|
-
`
|
|
118
|
-
`
|
|
119
|
-
`
|
|
120
|
-
|
|
121
|
-
`
|
|
122
|
-
`
|
|
123
|
-
`
|
|
124
|
-
`
|
|
125
|
-
`
|
|
126
|
-
`
|
|
127
|
-
`
|
|
128
|
-
|
|
129
|
-
`
|
|
130
|
-
`
|
|
131
|
-
`
|
|
132
|
-
`
|
|
133
|
-
`
|
|
134
|
-
`
|
|
135
|
-
`
|
|
136
|
-
`
|
|
137
|
-
`
|
|
138
|
-
`
|
|
139
|
-
`
|
|
140
|
-
`
|
|
141
|
-
`
|
|
142
|
-
`
|
|
143
|
-
`
|
|
144
|
-
`
|
|
145
|
-
`
|
|
146
|
-
`
|
|
147
|
-
`
|
|
148
|
-
`
|
|
149
|
-
`
|
|
150
|
-
`
|
|
151
|
-
`
|
|
152
|
-
`
|
|
153
|
-
`
|
|
154
|
-
`
|
|
155
|
-
`
|
|
156
|
-
`
|
|
157
|
-
`
|
|
158
|
-
`
|
|
159
|
-
`
|
|
160
|
-
`
|
|
161
|
-
`
|
|
162
|
-
`
|
|
163
|
-
`
|
|
164
|
-
`
|
|
165
|
-
`
|
|
166
|
-
` on
|
|
167
|
-
`
|
|
168
|
-
`
|
|
169
|
-
`
|
|
170
|
-
`
|
|
171
|
-
`
|
|
172
|
-
`
|
|
173
|
-
`
|
|
174
|
-
`
|
|
175
|
-
`
|
|
176
|
-
`
|
|
177
|
-
`
|
|
178
|
-
`
|
|
179
|
-
`
|
|
180
|
-
`
|
|
181
|
-
`
|
|
182
|
-
`
|
|
183
|
-
`
|
|
184
|
-
`
|
|
185
|
-
`
|
|
186
|
-
`
|
|
187
|
-
`
|
|
188
|
-
`
|
|
189
|
-
`
|
|
190
|
-
`
|
|
191
|
-
` never
|
|
192
|
-
`
|
|
193
|
-
`
|
|
194
|
-
`
|
|
195
|
-
`
|
|
196
|
-
`
|
|
197
|
-
`
|
|
198
|
-
`
|
|
199
|
-
`
|
|
200
|
-
`
|
|
201
|
-
`
|
|
202
|
-
`
|
|
203
|
-
`
|
|
204
|
-
`
|
|
205
|
-
`
|
|
206
|
-
`
|
|
207
|
-
`
|
|
208
|
-
`
|
|
209
|
-
`
|
|
210
|
-
`
|
|
211
|
-
`
|
|
212
|
-
`
|
|
213
|
-
`
|
|
214
|
-
`
|
|
215
|
-
`
|
|
216
|
-
`
|
|
217
|
-
`
|
|
218
|
-
`
|
|
219
|
-
`
|
|
220
|
-
`
|
|
221
|
-
`
|
|
222
|
-
`
|
|
223
|
-
`
|
|
224
|
-
`
|
|
225
|
-
`
|
|
226
|
-
`
|
|
227
|
-
`
|
|
228
|
-
`
|
|
229
|
-
`
|
|
230
|
-
`
|
|
231
|
-
`
|
|
232
|
-
`
|
|
233
|
-
`
|
|
234
|
-
`
|
|
235
|
-
`
|
|
236
|
-
`
|
|
237
|
-
`
|
|
238
|
-
`
|
|
239
|
-
`
|
|
240
|
-
`
|
|
241
|
-
`
|
|
242
|
-
`
|
|
243
|
-
`
|
|
244
|
-
`
|
|
245
|
-
`
|
|
246
|
-
`
|
|
247
|
-
`
|
|
248
|
-
`
|
|
249
|
-
`
|
|
250
|
-
|
|
251
|
-
`
|
|
252
|
-
`
|
|
253
|
-
`
|
|
254
|
-
`
|
|
255
|
-
` "it timed out" is never the finding — find what it was waiting for.`,
|
|
256
|
-
`(10) THE HARNESS IS SOURCE TOO — READ IT BEFORE YOU CALL IT. Open the`,
|
|
257
|
-
` helper you are about to use and check what it actually RETURNS: a`,
|
|
258
|
-
` setup routine that publishes env and returns nothing hands you`,
|
|
259
|
-
` undefined, and every later reference through it dies in beforeAll,`,
|
|
260
|
-
` taking the whole file down as "failures" the product never caused.`,
|
|
261
|
-
` The same applies to locators: a query must resolve to exactly ONE`,
|
|
262
|
-
` node. Component libraries label a control twice (the <label> and the`,
|
|
263
|
-
` widget both answer getByLabel), so prefer the role the widget`,
|
|
264
|
-
` actually exposes. Where a sibling test in this journey already`,
|
|
265
|
-
` drives the same control, reuse ITS helper verbatim rather than`,
|
|
266
|
-
` writing a second one that has to rediscover the same trap.`,
|
|
267
|
-
`(11) LEAVE SHARED STATE AS YOU FOUND IT, EVEN WHEN YOU FAIL. A fixture`,
|
|
268
|
-
` every other suite reads — a shared account, the one seeded record`,
|
|
269
|
-
` the whole journey keys on — must be put back the moment you are`,
|
|
270
|
-
` done with it. Read the value first, restore it in a finally or an`,
|
|
271
|
-
` after-EACH hook, never an after-ALL: the specs that mutate shared`,
|
|
272
|
-
` state are usually the ones asserting the write is REFUSED, so they`,
|
|
273
|
-
` throw by design the day the product is wrong, and a worker that dies`,
|
|
274
|
-
` on that throw never reaches after-all. The forged value then sits in`,
|
|
275
|
-
` the fixture and every later suite reads it as real — one defect`,
|
|
276
|
-
` becomes a hundred failures in journeys that were never broken, and`,
|
|
277
|
-
` the run cannot be trusted until someone repairs the data by hand.`,
|
|
278
|
-
` CLEAN UP BY A HANDLE THAT CANNOT GO MISSING. If cleanup matches a`,
|
|
279
|
-
` name prefix, check that every record the file creates can actually`,
|
|
280
|
-
` carry that prefix — validation minimums, enum values and generated`,
|
|
281
|
-
` identifiers routinely forbid it. One record the filter cannot see`,
|
|
282
|
-
` survives, and the next run collides with its own leftovers and`,
|
|
283
|
-
` reports the collision as a product error. Prefer an owning field`,
|
|
284
|
-
` that is always set (a parent, an owner id, a namespace).`,
|
|
285
|
-
`(12) A ROLE'S CREDENTIAL MUST BE THAT ROLE'S — NEVER FALL THROUGH TO`,
|
|
286
|
-
` ANOTHER ONE. When a test needs a specific actor (an admin, a staff`,
|
|
287
|
-
` user, a second customer) and that actor's credential is missing or`,
|
|
288
|
-
` expired, the answer is to SKIP, not to reach for whichever token`,
|
|
289
|
-
` the harness happens to hold. A live token for the wrong actor`,
|
|
290
|
-
` passes every "is it present / is it in date" check and then gets`,
|
|
291
|
-
` 401/403 from every route the real actor is allowed to use — which`,
|
|
292
|
-
` reads exactly like the product refusing legitimate access. This is`,
|
|
293
|
-
` the single most expensive mistake available to you: one fallback in`,
|
|
294
|
-
` a shared helper turns a whole area's suite into false defects, and`,
|
|
295
|
-
` nothing in the output says the wrong identity was used. So: resolve`,
|
|
296
|
-
` each role from its OWN source, never chain roles in a fallback`,
|
|
297
|
-
` list, and treat a template placeholder (DUMMY, replace_me, an empty`,
|
|
298
|
-
` value, a bracketed name) as absent rather than as a credential.`,
|
|
299
|
-
` Say which actor was missing in the skip reason.`,
|
|
300
|
-
`(13) A [baseline] CHECK PROVES ROBUSTNESS, NOT A VALUE. "Opens without a`,
|
|
301
|
-
` crash" is proved by the page loading with no error and no 5xx — not by`,
|
|
302
|
-
` an exact URL, trailing slash, heading text or element the document`,
|
|
303
|
-
` does not name. Never add an extra condition the criterion does not`,
|
|
304
|
-
` state (e.g. "double-click opens exactly one pop-up" when the document`,
|
|
305
|
-
` only says the button opens it), and never assert data in a place the`,
|
|
306
|
-
` page does not render it — check where the app shows it first.`,
|
|
307
|
-
`(14) SCOPE EVERY LOCATOR TO THE JOURNEY'S OWN UI. Global components`,
|
|
308
|
-
` (a login pop-up, a header, a cookie bar) share ids and labels with`,
|
|
309
|
-
` the page under test — "#password" can match the login modal's field.`,
|
|
310
|
-
` Locate inside the journey's form/container, never page-wide by a`,
|
|
311
|
-
` bare id or label that another always-mounted component also uses.`,
|
|
312
|
-
`(15) NOTHING YOU ARE NOT TESTING MAY BLOCK YOU. Apps open their own`,
|
|
313
|
-
` pop-ups for a signed-in or new user (promotions, cookie consent,`,
|
|
314
|
-
` onboarding tips, notification prompts). Use the harness's login /`,
|
|
315
|
-
` storage helpers that pre-suppress them; if one still opens, close it`,
|
|
316
|
-
` before you continue. A click intercepted by an overlay the test is`,
|
|
317
|
-
` not about is your bug, not the product's.`,
|
|
318
|
-
`(16) "INSIDE" MEANS THE MIDDLE. To click inside a panel, click its centre`,
|
|
319
|
-
` or a non-interactive child — never a corner offset like (5,5): with a`,
|
|
320
|
-
` border-radius that point is outside the element and lands on the`,
|
|
321
|
-
` overlay behind it.`,
|
|
322
|
-
`(17) LET THE APP SETTLE BEFORE YOU CHANGE THE SESSION. After a login,`,
|
|
323
|
-
` sign-out or navigation the app sends its own requests and may`,
|
|
324
|
-
` redirect by itself. Wait for the network to go idle before rotating`,
|
|
325
|
-
` a session, logging in elsewhere or navigating away, and assert the`,
|
|
326
|
-
` URL the APP chooses — never force a navigation that can cut off the`,
|
|
327
|
-
` app's own redirect.`,
|
|
328
|
-
`(18) ONE SOURCE FOR SEEDED IDS. Read every seeded id and handle from the`,
|
|
329
|
-
` harness (TEST_IDS / the values setup() publishes). If a test needs a`,
|
|
330
|
-
` private account, override EVERY id that belongs to it, never a`,
|
|
331
|
-
` subset — a leftover shared id makes you compare against another`,
|
|
332
|
-
` customer's row.`,
|
|
333
|
-
`(19) VERIFICATION CODES THE TEST CANNOT READ (SMS/email OTP, magic`,
|
|
334
|
-
` links, OAuth). Drive the real UI up to the code step, then use the`,
|
|
335
|
-
` harness's bypassVerification() so the final verify/register request`,
|
|
336
|
-
` is answered with a REAL account and a REAL token; everything after it`,
|
|
337
|
-
` (logged-in state, redirect, payment) is then tested for real. Put`,
|
|
338
|
-
` "(verification bypassed)" in that test's title. Wrong-code and`,
|
|
339
|
-
` expired-code tests do NOT bypass: the real server must refuse them.`,
|
|
340
|
-
`When rules conflict, prefer reporting LESS: a pending test costs a line`,
|
|
341
|
-
`in the report; a false failure costs the team a release.`,
|
|
105
|
+
`Teams hold releases on what these tests report. A test that fails for any`,
|
|
106
|
+
`reason other than the product being wrong is worse than no test at all: it`,
|
|
107
|
+
`burns the team's trust, and the next real defect gets waved through as`,
|
|
108
|
+
`"probably the tool again". So a test may FAIL only when the product provably`,
|
|
109
|
+
`contradicts an expectation the journey document states.`,
|
|
110
|
+
``,
|
|
111
|
+
`THE METHOD — EVERY LINE OF A TEST NEEDS EVIDENCE. A test is a chain of`,
|
|
112
|
+
`claims: "this is what should happen", "this is the element", "this account`,
|
|
113
|
+
`exists", "now is the moment to look". Each claim is either proven by evidence`,
|
|
114
|
+
`you can point to — the journey document, this project's source, the harness,`,
|
|
115
|
+
`a response you actually received — or it is a guess, and a guess is how a`,
|
|
116
|
+
`test fails on a correct product. Before you write each line, find its`,
|
|
117
|
+
`evidence. These five kinds of evidence cover every line you will write.`,
|
|
118
|
+
``,
|
|
119
|
+
`E1. WHAT YOU ASSERT — evidence: a line in the journey document.`,
|
|
120
|
+
` - Find the exact line that fixes the expected outcome and cite it in a`,
|
|
121
|
+
` short comment (e.g. "// oracle: §4 total = fee + shipping"). Criteria`,
|
|
122
|
+
` tagged [spec], [baseline] or [observed] are stated oracles. Look properly`,
|
|
123
|
+
` before you conclude one is absent: it may sit in another section, in a §`,
|
|
124
|
+
` the text cross-references, or in an acceptance list. "Test setup" says`,
|
|
125
|
+
` how to establish preconditions — follow it.`,
|
|
126
|
+
` - Assert what the line states, at the loosest form that still proves it.`,
|
|
127
|
+
` Add no condition it does not state, and no exact copy, ordering, id,`,
|
|
128
|
+
` timestamp or count it does not fix. A [baseline] robustness line ("opens`,
|
|
129
|
+
` without a crash") is proved by the page loading with no error and no`,
|
|
130
|
+
` 5xx, not by a URL, heading or element it never names. "The customer is`,
|
|
131
|
+
` told why" is proved by a non-empty message VISIBLE where the app shows`,
|
|
132
|
+
` it, unless the line fixes the words. Check where the page actually`,
|
|
133
|
+
` renders a value before asserting it there.`,
|
|
134
|
+
` - Assert the real end state (the response body and status, the`,
|
|
135
|
+
` persisted record, the screen) — not merely that nothing threw, and`,
|
|
136
|
+
` never a value your own stub hard-coded.`,
|
|
137
|
+
` - No line, no assertion: write the case pending (${pending}). An item under`,
|
|
138
|
+
` "Questions for product (not asserted)" or "Suggested", or one the`,
|
|
139
|
+
` document says is not tested through the screen, gets NO test at all —`,
|
|
140
|
+
` not even a pending one: it is the product's to answer.`,
|
|
141
|
+
` - A security probe asserts the SECURE outcome (refused, rejected,`,
|
|
142
|
+
` idempotent) unless the document records the weakness as accepted; when`,
|
|
143
|
+
` you cannot tell whether a defence exists, the case is pending.`,
|
|
144
|
+
`E2. WHAT YOU TOUCH — evidence: this project's source.`,
|
|
145
|
+
` - Every locator, route, request field, response field, storage key and id`,
|
|
146
|
+
` is something you found in THIS project's code (or the CODE FACTS), never`,
|
|
147
|
+
` something that usually exists.`,
|
|
148
|
+
` - A locator resolves to exactly ONE node — the one you mean. Components`,
|
|
149
|
+
` often label a control twice, and always-mounted components (a login`,
|
|
150
|
+
` pop-up, a header) share ids and labels with the page. Scope to the`,
|
|
151
|
+
` journey's own container, after confirming in the component source that`,
|
|
152
|
+
` the element is rendered INSIDE it: a footer link or a resend control`,
|
|
153
|
+
` often sits beside a form, not in it — then scope to the parent of both.`,
|
|
154
|
+
` "Inside" a panel means its centre or a non-interactive child, never a`,
|
|
155
|
+
` corner offset that a rounded border puts outside it. "Outside" means a`,
|
|
156
|
+
` point on the backdrop itself, clear of the panel and away from the`,
|
|
157
|
+
` window's edges, where a scrollbar or other chrome may sit.`,
|
|
158
|
+
` - A response field is one the server's serializer for THAT endpoint`,
|
|
159
|
+
` sends, not a name from the client's types for another endpoint.`,
|
|
160
|
+
` - Ids come from the harness (TEST_IDS and what setup() publishes); a`,
|
|
161
|
+
` private account overrides every id that belongs to it, never a subset.`,
|
|
162
|
+
` A dynamic page is keyed on an id the served build can render — some`,
|
|
163
|
+
` builds serve only a listed set of parameters.`,
|
|
164
|
+
` - Read a harness helper before calling it, to see what it really returns.`,
|
|
165
|
+
` - Other test files are NOT evidence. They were written the same way`,
|
|
166
|
+
` as yours and can carry the same mistake, so copying one spreads it.`,
|
|
167
|
+
` Where the harness offers a helper, use it. Take anything from a`,
|
|
168
|
+
` sibling file only after checking it against the source exactly as`,
|
|
169
|
+
` you would check your own line.`,
|
|
170
|
+
`E3. WHAT YOU NEED FIRST — evidence: the harness, or the product's own way of`,
|
|
171
|
+
` making it.`,
|
|
172
|
+
` - Accounts, records, settings and codes are established through the`,
|
|
173
|
+
` harness, or the way the product would: follow the transition that`,
|
|
174
|
+
` produces a state and copy what it writes; satisfy the schema (every NOT`,
|
|
175
|
+
` NULL column, the real type of each id). A record no code path could`,
|
|
176
|
+
` produce proves nothing.`,
|
|
177
|
+
` - Look for a precondition the way you would look for a bug before calling`,
|
|
178
|
+
` it missing: another env name, an identity the harness already has, one`,
|
|
179
|
+
` seed call away, a service on another port.`,
|
|
180
|
+
` - A credential that exists is not one that works — confirm it is`,
|
|
181
|
+
` accepted. Each role uses its OWN credential; never fall through to`,
|
|
182
|
+
` another role's token, and treat a placeholder (DUMMY, replace_me, empty,`,
|
|
183
|
+
` <bracketed>) as absent. Bind fixtures to the identity the screen is`,
|
|
184
|
+
` scoped by — the key the server scopes on, not an identity provider's`,
|
|
185
|
+
` handle.`,
|
|
186
|
+
` - A code the test cannot read (message or email code, magic link,`,
|
|
187
|
+
` third-party sign-in): drive the real UI to the code step, then let the`,
|
|
188
|
+
` harness's bypassVerification() answer only that step with a real account`,
|
|
189
|
+
` and a real token; put "(verification bypassed)" in the title. Wrong- and`,
|
|
190
|
+
` expired-code tests do not bypass: the real server must refuse them.`,
|
|
191
|
+
` - Where the harness has no database client, prove persisted state through`,
|
|
192
|
+
` the app's own API.`,
|
|
193
|
+
` - If it genuinely cannot be established, SKIP at the top of the test with`,
|
|
194
|
+
` a reason stating what you observed ("GET /x answered 404", "TEST_X is not`,
|
|
195
|
+
` set", "no answer after 20 s") — never a guessed cause, and never let a`,
|
|
196
|
+
` missing precondition reach an assertion.`,
|
|
197
|
+
`E4. WHAT YOU WAIT FOR — evidence: the event that actually happens.`,
|
|
198
|
+
` - Wait for a specific condition, never a fixed sleep, and prefer a`,
|
|
199
|
+
` retrying assertion: a value written after hydration or a round-trip is`,
|
|
200
|
+
` not there at the first look. A page-side read that dies because a`,
|
|
201
|
+
` redirect landed mid-poll is retried, not reported.`,
|
|
202
|
+
` - A page whose job is to redirect never fires load on the URL you asked`,
|
|
203
|
+
` for: wait for the response or the outcome. A control hidden from the`,
|
|
204
|
+
` accessibility tree is not there for a role query. Hold a response/request`,
|
|
205
|
+
` wait BEFORE the action that triggers it.`,
|
|
206
|
+
` - After a login, sign-out or navigation, let the app's own requests and`,
|
|
207
|
+
` redirects settle before you change the session, and assert the URL the`,
|
|
208
|
+
` APP chooses.`,
|
|
209
|
+
` - The first render after a navigation is a precondition: wait for the`,
|
|
210
|
+
` control you will use, with a timeout sized to what the app loads first.`,
|
|
211
|
+
` A page that never renders is reported as that, never as the behaviour`,
|
|
212
|
+
` under test failing. "It timed out" is never the finding.`,
|
|
213
|
+
`E5. WHAT ELSE IS RUNNING — evidence: you checked, and your test does not`,
|
|
214
|
+
` depend on it.`,
|
|
215
|
+
` - The app's own pop-ups (promotions, consent, onboarding): use the`,
|
|
216
|
+
` harness helpers that suppress them on every login path, and close any`,
|
|
217
|
+
` that still opens. A click intercepted by something the test is not`,
|
|
218
|
+
` about is the test's bug.`,
|
|
219
|
+
` - Other tests: no dependence on order, or on data another test creates;`,
|
|
220
|
+
` assert only on records YOU set up, by id. A serial block only when a`,
|
|
221
|
+
` test consumes state an earlier one in it created — otherwise one`,
|
|
222
|
+
` failure silently skips the rest.`,
|
|
223
|
+
` - Shared state (a shared account, the one seeded record): read it first`,
|
|
224
|
+
` and restore it in a finally or after-EACH hook, never after-all. Clean`,
|
|
225
|
+
` up by a handle that cannot go missing (an owning field always set), not`,
|
|
226
|
+
` a name prefix validation may forbid.`,
|
|
227
|
+
` - A stub intercepts exactly the dependency — a different host; the same`,
|
|
228
|
+
` host on another port; a path prefix; or a predicate on the request —`,
|
|
229
|
+
` never a URL substring that also matches the app's own pages; then`,
|
|
230
|
+
` confirm the page under test still loads. Only a dependency you truly`,
|
|
231
|
+
` cannot separate stays pending, naming what you checked.`,
|
|
232
|
+
` - Real side effects (a message, an email, a charge) sit behind the opt-in`,
|
|
233
|
+
` gate the sibling files use, with one identity per rate-limited send.`,
|
|
234
|
+
` - A login elsewhere can end a cached session: get tokens through the`,
|
|
235
|
+
` harness accessors that re-confirm them.`,
|
|
236
|
+
` - Negative input must really be negative: a "bad" credential, signature,`,
|
|
237
|
+
` id or payload differs from the valid one at the level the product checks`,
|
|
238
|
+
` (change decoded bytes, not an encoded character), and the valid control`,
|
|
239
|
+
` is asserted too where you can.`,
|
|
240
|
+
``,
|
|
241
|
+
`SELF-CHECK BEFORE YOU OUTPUT. Go through every test you wrote and name, for`,
|
|
242
|
+
`each line, its evidence under E1–E5. A line with none is fixed, or its test`,
|
|
243
|
+
`becomes pending with the reason. When in doubt, report LESS: a pending test`,
|
|
244
|
+
`costs a line in the report; a false failure costs the team a release.`,
|
|
342
245
|
``,
|
|
343
246
|
...(grounding
|
|
344
247
|
? [
|
|
@@ -374,7 +277,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
374
277
|
`something to paper over. NEVER reshape an assertion to match observed`,
|
|
375
278
|
`code behaviour just to make it green; that rubber-stamps the bug. If`,
|
|
376
279
|
`the document does not state the expected outcome for a case, make it`,
|
|
377
|
-
`pending (${pending})
|
|
280
|
+
`pending (${pending}) — do not invent the`,
|
|
378
281
|
`expectation from the code.`,
|
|
379
282
|
`REQUEST URLS: build every request path as the FULL mounted route the`,
|
|
380
283
|
`router actually registers — INCLUDING the version/area prefix (e.g.`,
|
|
@@ -395,12 +298,6 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
395
298
|
`genuinely cannot assert an outcome, make the case pending (${pending})`,
|
|
396
299
|
`— never write a test body with no assertion, which passes without`,
|
|
397
300
|
`testing anything.`,
|
|
398
|
-
`ASSERTION DEPTH: assert the REAL end-state — persisted DB/state, the`,
|
|
399
|
-
`actual response body/status, or observable UI — not merely that "no`,
|
|
400
|
-
`error was thrown". CRITICAL: if you mock an external/API response, do`,
|
|
401
|
-
`NOT then "verify" a value your own mock hard-coded (asserting the mock`,
|
|
402
|
-
`returned what you told it to proves nothing). Read back the real effect`,
|
|
403
|
-
`(DB row, downstream call, UI change) the behaviour was supposed to have.`,
|
|
404
301
|
``,
|
|
405
302
|
isApi
|
|
406
303
|
? [
|
|
@@ -450,8 +347,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
450
347
|
`NEVER invent a data-testid/data-cy: an unverified test-id is`,
|
|
451
348
|
`REJECTED by the selector-grounding gate and wastes the whole file.`,
|
|
452
349
|
`Only when an element cannot be identified by ANY real handle,`,
|
|
453
|
-
`defer that single case with
|
|
454
|
-
`
|
|
350
|
+
`defer that single case with ${pending} naming the control you could`,
|
|
351
|
+
`not identify — do not default to pending, never guess.`,
|
|
455
352
|
`NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
|
|
456
353
|
`never a description/comment, and never a route-file placeholder`,
|
|
457
354
|
`like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
|
|
@@ -461,8 +358,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
461
358
|
`CRITICAL Playwright API: import { test, expect } from`,
|
|
462
359
|
`'@playwright/test'. There is NO test.todo / it.* / describe.skip`,
|
|
463
360
|
`in Playwright — using them throws at collection and breaks the`,
|
|
464
|
-
`WHOLE file. For a pending case use
|
|
465
|
-
`
|
|
361
|
+
`WHOLE file. For a pending case use ${pending}; group with`,
|
|
362
|
+
`test.describe(...).`,
|
|
466
363
|
`CRITICAL API-request API: the \`request\` FIXTURE (test args`,
|
|
467
364
|
`\`async ({ request }) => …\`) IS already an APIRequestContext —`,
|
|
468
365
|
`call it DIRECTLY: \`await request.post(url, { data })\`,`,
|
|
@@ -518,8 +415,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
518
415
|
`- Do NOT ask for approval or describe what you are doing.`,
|
|
519
416
|
`- Do NOT wrap the code in markdown fences.`,
|
|
520
417
|
`- Tag the top with a comment: // @generated by testforge`,
|
|
521
|
-
`-
|
|
522
|
-
`
|
|
418
|
+
`- Never write "TODO" or "FIXME" in the file. Something you cannot`,
|
|
419
|
+
` establish is a pending test with its reason, never a note to finish later.`,
|
|
523
420
|
``,
|
|
524
421
|
`--- PROJECT MAP (data layer) ---`,
|
|
525
422
|
map,
|
|
@@ -544,7 +441,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
544
441
|
`--- JOURNEY ---`,
|
|
545
442
|
`It may include sections beyond fields — actions, expected results,`,
|
|
546
443
|
`negative cases, race conditions, webhooks, security notes, and a`,
|
|
547
|
-
`Data section. Use ALL of it
|
|
444
|
+
`Data section. Use ALL of it.`,
|
|
548
445
|
``,
|
|
549
446
|
journey.rawMarkdown,
|
|
550
447
|
// ---- PER-FILE TASK (kept LAST so the big context above stays a stable,
|
|
@@ -608,6 +505,8 @@ export function generateWithAI(journey, testPath, focus, grounding) {
|
|
|
608
505
|
* journey's suite can be generated concurrently (each spawns its own
|
|
609
506
|
* `claude` process). Same prompt, same read-only exploration.
|
|
610
507
|
*/
|
|
508
|
+
/** Marks a test whose failure the repair judged to be the application's. */
|
|
509
|
+
export const DEFECT_MARK = 'testforge-defect:';
|
|
611
510
|
/**
|
|
612
511
|
* Repair a generated test from the runner's own failure output.
|
|
613
512
|
*
|
|
@@ -624,26 +523,41 @@ export async function repairWithAIAsync(journey, testPath, current, failure, har
|
|
|
624
523
|
'',
|
|
625
524
|
'=== REPAIR ===',
|
|
626
525
|
`The file below was generated for ${testPath} and RUN against the real`,
|
|
627
|
-
'application.
|
|
628
|
-
'you what the application actually
|
|
526
|
+
'application. The tests listed under RUN RESULT did not pass. The runner',
|
|
527
|
+
'output is the ground truth: it tells you what the application actually',
|
|
528
|
+
'renders, routes and returns.',
|
|
629
529
|
'',
|
|
630
|
-
'
|
|
530
|
+
'Decide, for EACH listed test, which of these it is — from the evidence,',
|
|
531
|
+
'not from what you hope:',
|
|
532
|
+
' TEST FAULT — the test never reached the behaviour it checks, or checked',
|
|
533
|
+
' it wrongly: a locator that matched nothing or the wrong',
|
|
534
|
+
' element, a container that does not hold the element, a wait',
|
|
535
|
+
' for the wrong event, a response field the server does not',
|
|
536
|
+
' send, a precondition its own setup got wrong, an assertion',
|
|
537
|
+
' the journey document does not state. FIX the test.',
|
|
538
|
+
' APP DEFECT — the test reached the behaviour (the element or response it',
|
|
539
|
+
' checks is there) and the application contradicts a criterion',
|
|
540
|
+
' the journey document states. Change NOTHING in that test.',
|
|
541
|
+
` Put one comment line directly above its declaration:`,
|
|
542
|
+
` // ${DEFECT_MARK} <the criterion, quoted with its tag> — <what the app did instead>`,
|
|
543
|
+
' SKIPPED — a test listed under SKIPPED skipped itself. If its own',
|
|
544
|
+
' setup looked in the wrong place (wrong field, wrong route,',
|
|
545
|
+
' wrong id), fix that so it runs. If what it needs truly does',
|
|
546
|
+
' not exist in this environment, leave it exactly as it is.',
|
|
631
547
|
'',
|
|
632
|
-
'Rules
|
|
633
|
-
' 1.
|
|
634
|
-
'
|
|
635
|
-
'
|
|
636
|
-
'
|
|
637
|
-
'
|
|
638
|
-
'
|
|
639
|
-
'
|
|
640
|
-
'
|
|
641
|
-
' the journey document), keep the assertion exactly as it is and change',
|
|
642
|
-
' nothing — a real defect must stay caught.',
|
|
548
|
+
'Rules, in addition to every rule above:',
|
|
549
|
+
' 1. Never delete, rename, weaken or invert an assertion, and never delete',
|
|
550
|
+
' or rename a test.',
|
|
551
|
+
' 2. Never add test.skip / test.fixme / try-catch to make a failure go away.',
|
|
552
|
+
' 3. Do not touch a test that is not listed below.',
|
|
553
|
+
' 4. When in doubt between TEST FAULT and APP DEFECT, look again at the',
|
|
554
|
+
' component and server source: an element that is in the DOM but hidden,',
|
|
555
|
+
' a message the server sends but the screen does not show, a value the',
|
|
556
|
+
' screen accepts but the server refuses — those are the application.',
|
|
643
557
|
'',
|
|
644
558
|
'--- CURRENT FILE ---',
|
|
645
559
|
current,
|
|
646
|
-
'---
|
|
560
|
+
'--- RUN RESULT ---',
|
|
647
561
|
failure,
|
|
648
562
|
'--- END ---',
|
|
649
563
|
'',
|