@activate-core/test-forge 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +29 -11
- package/dist/core/docs/doc-guidance.js +10 -0
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +12 -1
- package/dist/core/generate/ai-generator.js +181 -267
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +181 -226
- package/dist/core/generate/pending.js +118 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +70 -73
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { execFileSync, spawn } from 'node:child_process';
|
|
15
15
|
import { existsSync, readFileSync } from 'node:fs';
|
|
16
|
+
import { createHash } from 'node:crypto';
|
|
16
17
|
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
17
18
|
import { cleanOutput } from './ai-generator.js';
|
|
18
19
|
const MAP_PATH = 'testforge.md';
|
|
@@ -22,226 +23,144 @@ const MAP_PATH = 'testforge.md';
|
|
|
22
23
|
* are given this exact shape, so imports and usage line up.
|
|
23
24
|
*/
|
|
24
25
|
export const HARNESS_CONTRACT = [
|
|
25
|
-
`The testkit module MUST export exactly this stable API (CommonJS`,
|
|
26
|
-
`module.exports)
|
|
27
|
-
` - connect(): Promise<void> open the
|
|
28
|
-
`
|
|
29
|
-
`
|
|
30
|
-
`
|
|
31
|
-
`
|
|
32
|
-
`
|
|
33
|
-
`
|
|
34
|
-
` -
|
|
35
|
-
` -
|
|
36
|
-
`
|
|
37
|
-
`
|
|
38
|
-
`
|
|
39
|
-
` -
|
|
40
|
-
`
|
|
41
|
-
`
|
|
42
|
-
`
|
|
43
|
-
`
|
|
44
|
-
`
|
|
45
|
-
`
|
|
46
|
-
`
|
|
47
|
-
`
|
|
48
|
-
` replaceOne, never clobber creation`,
|
|
49
|
-
` metadata or a pre-existing richer record.`,
|
|
50
|
-
` Seeds are TRUSTED fixtures: do NOT let the`,
|
|
51
|
-
` app's own schema quirks block them — if an`,
|
|
52
|
-
` update-validator rejects a value the app`,
|
|
53
|
-
` itself defaults (e.g. a required String`,
|
|
54
|
-
` whose default is ''), write WITHOUT update`,
|
|
55
|
-
` validators (runValidators:false) rather`,
|
|
56
|
-
` than failing the whole seed.`,
|
|
57
|
-
` READ THE SCHEMA BEFORE WRITING A SEED.`,
|
|
58
|
-
` A relational database refuses a row that`,
|
|
59
|
-
` omits a NOT NULL column with no default,`,
|
|
60
|
-
` so every such column must be supplied —`,
|
|
61
|
-
` take the list from the migrations or the`,
|
|
62
|
-
` schema itself, not from the fields the`,
|
|
63
|
-
` journey happens to mention. And take each`,
|
|
64
|
-
` id's TYPE from the schema too: one table`,
|
|
65
|
-
` keyed by uuid does not make them all uuid,`,
|
|
66
|
-
` and an id of the wrong type is rejected`,
|
|
67
|
-
` outright rather than simply not matching.`,
|
|
68
|
-
` - TEST_IDS: object the stable well-known test ids/handles.`,
|
|
69
|
-
` EVERY id MUST be env-overridable: resolve`,
|
|
70
|
-
` each as process.env.TEST_<NAME>_ID ||`,
|
|
71
|
-
` '<stable default>' (e.g. TEST_PROJECT_ID,`,
|
|
72
|
-
` TEST_OWNER_ID). This lets a caller point`,
|
|
73
|
-
` the suite at data created BY HAND on`,
|
|
74
|
-
` staging/prod (with SEED_MODE=existing)`,
|
|
75
|
-
` without editing code. Resolve non-id`,
|
|
76
|
-
` handles (phone, email, auth subject) the`,
|
|
77
|
-
` same env-overridable way.`,
|
|
78
|
-
` - setup(): Promise<void> THE one-call entry point (idempotent,`,
|
|
79
|
-
` safe to call from every beforeAll AND`,
|
|
80
|
-
` from a Playwright globalSetup). It:`,
|
|
81
|
-
` 1. connect()s; then seed()s — UNLESS`,
|
|
82
|
-
` process.env.SEED_MODE === 'existing',`,
|
|
83
|
-
` in which case it SKIPS seed() and`,
|
|
84
|
-
` reset() and runs against data already`,
|
|
85
|
-
` in the DB (staging/prod, where the`,
|
|
86
|
-
` fixtures were created by hand and`,
|
|
87
|
-
` addressed via the env-overridable`,
|
|
88
|
-
` TEST_IDS);`,
|
|
89
|
-
` 2. MINTS a fresh auth access token at`,
|
|
90
|
-
` runtime — read the project's REAL`,
|
|
91
|
-
` login/refresh auth (controllers/`,
|
|
92
|
-
` middleware) and either exchange a`,
|
|
93
|
-
` seed token (env, e.g. a refresh`,
|
|
94
|
-
` token) or log in the seeded qa-`,
|
|
95
|
-
` user to obtain it;`,
|
|
96
|
-
` 3. CACHES the minted token to a temp`,
|
|
97
|
-
` file so parallel workers reuse it`,
|
|
98
|
-
` (one mint per run, no churn);`,
|
|
99
|
-
` 4. POPULATES process.env for EVERY`,
|
|
100
|
-
` token/id env var the suite reads`,
|
|
101
|
-
` — GREP the sibling test files for`,
|
|
102
|
-
` process.env.*TOKEN*/*ID* names and`,
|
|
103
|
-
` set each to the minted/seeded`,
|
|
104
|
-
` value, so tests that read`,
|
|
105
|
-
` process.env work with NO hand-`,
|
|
106
|
-
` pasted tokens.`,
|
|
107
|
-
` - token(role?): string the current access token accessor`,
|
|
108
|
-
` - REQUIRED_PACKAGES, resetRateLimits(), bypassVerification(): see`,
|
|
109
|
-
` RUN ENVIRONMENT below.`,
|
|
110
|
-
`Plus, for Playwright suites, ALSO export:`,
|
|
111
|
-
` - apiContext(request): a helper that returns an authed API request`,
|
|
112
|
-
` context (Bearer = the minted token — NEVER hardcode a secret).`,
|
|
113
|
-
` CRITICAL: create it from the imported request MODULE, NOT the`,
|
|
114
|
-
` fixture — \`const { request: apiRequest } = require('@playwright/`,
|
|
115
|
-
` test'); return apiRequest.newContext({ baseURL, extraHTTPHeaders })\`.`,
|
|
116
|
-
` The \`request\` FIXTURE has NO .newContext(); calling it throws.`,
|
|
117
|
-
` - login(page): Promise<void> establish an authenticated UI session`,
|
|
118
|
-
` the webapp actually accepts. Inject the`,
|
|
119
|
-
` SAME real minted access token (a valid,`,
|
|
120
|
-
` decodable JWT — the app gate typically`,
|
|
121
|
-
` jwtDecode()s it and checks exp) into the`,
|
|
122
|
-
` EXACT localStorage/cookie keys the app`,
|
|
123
|
-
` reads (GREP the auth reducer/guard for the`,
|
|
124
|
-
` key names). A fabricated/opaque token that`,
|
|
125
|
-
` fails the app's validity check leaves the`,
|
|
126
|
-
` user logged-out (redirect loop). The token`,
|
|
127
|
-
` owner MUST be the identity that owns the`,
|
|
128
|
-
` seeded fixtures (see IDENTITY BINDING), so`,
|
|
129
|
-
` the UI shows the seeded data.`,
|
|
130
|
-
`Use only env var NAMES for secrets/connection — never literal values.`,
|
|
131
|
-
`The ONLY credentials a developer should have to provide are the DB`,
|
|
132
|
-
`connection, a test key for any paid third party, and ONE seed/refresh`,
|
|
133
|
-
`token (or seeded login) — everything else setup() derives at runtime.`,
|
|
26
|
+
`THE API. The testkit module MUST export exactly this stable API (CommonJS`,
|
|
27
|
+
`module.exports); every generated test is written against it:`,
|
|
28
|
+
` - connect(): Promise<void> open the connection(s) to every data store`,
|
|
29
|
+
` this project's data lives in (there may be`,
|
|
30
|
+
` several — read the config).`,
|
|
31
|
+
` - disconnect(): Promise<void> close them.`,
|
|
32
|
+
` - reset(): Promise<void> remove ONLY test data. Gated — see H5.`,
|
|
33
|
+
` - seed(spec?): Promise<object> idempotently create the records the tests`,
|
|
34
|
+
` need and return their ids/handles.`,
|
|
35
|
+
` - TEST_IDS: object the stable test ids and handles.`,
|
|
36
|
+
` - setup(): Promise<void> the one-call entry point (see H4).`,
|
|
37
|
+
` - token(role?): string the current access token.`,
|
|
38
|
+
` - ensureToken(role?): Promise<string> that token, confirmed still accepted.`,
|
|
39
|
+
` - REQUIRED_PACKAGES: string[] packages the testkit needs (see H6).`,
|
|
40
|
+
` - resetRateLimits(), bypassVerification(), suppressPopups(), and one`,
|
|
41
|
+
` resolver per seeded entity — see H7 and H3.`,
|
|
42
|
+
`Plus, for Playwright suites:`,
|
|
43
|
+
` - apiContext(request): an authenticated API request context, created from`,
|
|
44
|
+
` the imported request MODULE (\`const { request: apiRequest } =`,
|
|
45
|
+
` require('@playwright/test'); apiRequest.newContext({ baseURL,`,
|
|
46
|
+
` extraHTTPHeaders })\`) — the \`request\` FIXTURE has no .newContext().`,
|
|
47
|
+
` - login(page): Promise<void> an authenticated UI session the app really`,
|
|
48
|
+
` accepts (see H2).`,
|
|
134
49
|
``,
|
|
135
|
-
`
|
|
136
|
-
`
|
|
137
|
-
`
|
|
138
|
-
`
|
|
139
|
-
`
|
|
140
|
-
`
|
|
141
|
-
`
|
|
142
|
-
` - When process.env.SEED_MODE === 'existing', reset() and all seed()`,
|
|
143
|
-
` WRITES are skipped entirely — the suite runs against data already`,
|
|
144
|
-
` present (hand-created on staging/prod), addressed via TEST_IDS. Do`,
|
|
145
|
-
` not mutate that data destructively.`,
|
|
146
|
-
` - Never widen a delete beyond the "qa-"/"QA-TEST" prefix + TEST_IDS`,
|
|
147
|
-
` scope; always cascade children by projectId, never a bare drop.`,
|
|
50
|
+
`THE METHOD — EVERY PART OF THE TESTKIT IS PROVEN FROM THIS PROJECT. Each`,
|
|
51
|
+
`function is a claim about how this product stores data, signs people in and`,
|
|
52
|
+
`behaves at run time. Read the evidence first — the data layer and its`,
|
|
53
|
+
`schema, the auth flow, the client's storage keys, the server's response`,
|
|
54
|
+
`serializers — and decide everything from what you read in THIS project.`,
|
|
55
|
+
`Nothing here names a stack. A testkit built on a guess fails every test that`,
|
|
56
|
+
`uses it, and every one of those failures reads as the product being broken.`,
|
|
148
57
|
``,
|
|
149
|
-
`
|
|
150
|
-
`
|
|
151
|
-
`
|
|
152
|
-
`
|
|
153
|
-
`
|
|
154
|
-
`
|
|
155
|
-
`
|
|
156
|
-
`
|
|
157
|
-
`
|
|
158
|
-
`
|
|
159
|
-
`
|
|
160
|
-
`
|
|
161
|
-
`
|
|
162
|
-
`
|
|
163
|
-
`
|
|
164
|
-
` app can never cast: owner reads come back empty and writes answer`,
|
|
165
|
-
` 500, both of which read as the product being broken.`,
|
|
166
|
-
` - A seeded BROWSER session carries the same id. Whatever the app`,
|
|
167
|
-
` stores about the signed-in user (its web storage keys) must hold`,
|
|
168
|
-
` the resolved app id, because the client copies it into the`,
|
|
169
|
-
` payloads it sends. And resolve it FOR THE ROLE being seeded — a`,
|
|
170
|
-
` maker session built from the wearer's identity tests nothing.`,
|
|
171
|
-
` - Each role resolves from its OWN credential. Never chain roles in a`,
|
|
172
|
-
` fallback list: a live token for the wrong actor passes every`,
|
|
173
|
-
` presence and expiry check and then collects 401/403 from every`,
|
|
174
|
-
` route the real actor is allowed to use. Treat a template`,
|
|
175
|
-
` placeholder (DUMMY, replace_me, empty, <bracketed>) as absent, and`,
|
|
176
|
-
` when a role's credential is missing, SKIP with its name in the`,
|
|
177
|
-
` reason.`,
|
|
178
|
-
` - Do NOT hardcode an owner id the token does not own: the seeded rows`,
|
|
179
|
-
` would never be returned to the authenticated caller, and every`,
|
|
180
|
-
` owner-scoped test would fail for a fake reason.`,
|
|
181
|
-
` - A scope/IDOR test needing a SECOND, non-owner identity needs a`,
|
|
182
|
-
` distinct token (env). If only one identity is available, make those`,
|
|
183
|
-
` cases pending — never assert a scope the token cannot actually prove.`,
|
|
184
|
-
``,
|
|
185
|
-
`RUN ENVIRONMENT — the suite must run on a fresh machine given only the`,
|
|
186
|
-
`env file, in WHATEVER stack this project uses. Decide everything below`,
|
|
187
|
-
`from what you read in THIS project; nothing here names a stack.`,
|
|
188
|
-
` - PACKAGES: use Node built-ins, the test runner, and what the project`,
|
|
189
|
-
` already has installed. If reaching the project's data layer needs a`,
|
|
190
|
-
` client the project lacks (a driver for its database), require it`,
|
|
191
|
-
` lazily, throw "testkit: missing package <name> — run: npm i -D <name>"`,
|
|
192
|
-
` when absent, and list it in REQUIRED_PACKAGES (an array literal) so`,
|
|
193
|
-
` the runner can check before the suite starts. Never add a package`,
|
|
194
|
-
` for anything else.`,
|
|
195
|
-
` - NO OTHER RUNTIMES: never shell out to another language runtime, a`,
|
|
196
|
-
` framework CLI or a container tool — the test machine need not have`,
|
|
197
|
-
` them.`,
|
|
198
|
-
` - CREDENTIALS COME FROM THE ENV FILE: every role the suite logs in as`,
|
|
199
|
-
` (customer, staff, admin, second customer, …) is an account whose`,
|
|
200
|
-
` login identifier and password are read from the env file`,
|
|
58
|
+
`H1. WHERE THE DATA LIVES. Read the data configuration: how many stores,`,
|
|
59
|
+
` which env var names each one's connection, which records live in which.`,
|
|
60
|
+
` Seed each record into the store it belongs to. When this repository`,
|
|
61
|
+
` does not own its data (the map or a related repository shows the data`,
|
|
62
|
+
` layer belongs to another codebase, e.g. a client talking to a separate`,
|
|
63
|
+
` API), connect to no store at all: seed and read state through the app's`,
|
|
64
|
+
` own API, and a state only a direct write could create is not seeded —`,
|
|
65
|
+
` the tests needing it skip with that reason.`,
|
|
66
|
+
`H2. HOW PEOPLE SIGN IN. Read the real login, refresh and verification flow`,
|
|
67
|
+
` (handlers and middleware) and the client's guard: which storage keys or`,
|
|
68
|
+
` cookies it reads, and what it checks a token for. login(page) injects a`,
|
|
69
|
+
` real token the app accepts into exactly those keys.`,
|
|
70
|
+
`H3. EVERY RECORD THE PRODUCT'S WAY.`,
|
|
71
|
+
` - Credentials come from the env file: every role the suite signs in as`,
|
|
72
|
+
` is an account whose identifier and password are read from it`,
|
|
201
73
|
` (TEST_<ROLE>_EMAIL / TEST_<ROLE>_PASSWORD, or the identifier the app`,
|
|
202
|
-
`
|
|
203
|
-
`
|
|
204
|
-
`
|
|
205
|
-
`
|
|
206
|
-
`
|
|
207
|
-
` through its API where one exists
|
|
208
|
-
`
|
|
209
|
-
`
|
|
210
|
-
`
|
|
211
|
-
`
|
|
212
|
-
`
|
|
213
|
-
`
|
|
214
|
-
`
|
|
215
|
-
`
|
|
216
|
-
`
|
|
217
|
-
`
|
|
218
|
-
`
|
|
219
|
-
` -
|
|
220
|
-
`
|
|
221
|
-
`
|
|
222
|
-
`
|
|
223
|
-
` -
|
|
224
|
-
`
|
|
225
|
-
`
|
|
226
|
-
`
|
|
227
|
-
`
|
|
228
|
-
`
|
|
229
|
-
`
|
|
230
|
-
`
|
|
231
|
-
`
|
|
232
|
-
|
|
233
|
-
`
|
|
234
|
-
`
|
|
235
|
-
`
|
|
236
|
-
`
|
|
237
|
-
`
|
|
238
|
-
`
|
|
239
|
-
`
|
|
240
|
-
`
|
|
241
|
-
`
|
|
242
|
-
`
|
|
243
|
-
` the
|
|
74
|
+
` signs in with). The testkit never hashes or invents a credential. Each`,
|
|
75
|
+
` role resolves from its OWN credential, never a fallback to another`,
|
|
76
|
+
` role's; a placeholder (DUMMY, replace_me, empty, <bracketed>) is absent,`,
|
|
77
|
+
` and a role whose credential is missing makes its tests skip naming it.`,
|
|
78
|
+
` - Other records and states are created the way the product creates them:`,
|
|
79
|
+
` through its API where one exists; otherwise through the data layer by`,
|
|
80
|
+
` following the transition that produces the state and copying what it`,
|
|
81
|
+
` writes — every NOT NULL column supplied, every id of the column's real`,
|
|
82
|
+
` type, upserts that never clobber creation metadata. Extra account`,
|
|
83
|
+
` states (unverified, banned, deleted, another role) copy the stored`,
|
|
84
|
+
` credential of an env-provided account, then set the state.`,
|
|
85
|
+
` - Owner-scoped data is seeded under the id the PRODUCT reports for the`,
|
|
86
|
+
` signed-in identity (ask its profile/whoami endpoint) — an identity`,
|
|
87
|
+
` provider's subject is a different value. Browser storage for a role`,
|
|
88
|
+
` carries that same id.`,
|
|
89
|
+
` - Every id and handle is env-overridable (process.env.TEST_<NAME>_ID ||`,
|
|
90
|
+
` '<stable default>'), so the suite can run against hand-made data.`,
|
|
91
|
+
` - One resolver per seeded entity the tests read back through the API: it`,
|
|
92
|
+
` fetches the entity and returns it, reading field names from the`,
|
|
93
|
+
` server's response serializer for that endpoint. Tests call it instead of`,
|
|
94
|
+
` re-reading the response.`,
|
|
95
|
+
` - When a dynamic page is served only for a listed set of parameters,`,
|
|
96
|
+
` expose a seeded id from that list; if no listed id has the data,`,
|
|
97
|
+
` setup() says which route and which ids.`,
|
|
98
|
+
`H4. EVERY ACCESSOR STAYS TRUE FOR THE WHOLE RUN.`,
|
|
99
|
+
` - setup() is idempotent and safe from every beforeAll and a global setup:`,
|
|
100
|
+
` connect; seed (unless SEED_MODE=existing); mint each role's token once`,
|
|
101
|
+
` through the real login; cache it to a temp file for other workers; set`,
|
|
102
|
+
` process.env for every token/id variable the sibling tests read (grep`,
|
|
103
|
+
` them), so no token is ever pasted by hand.`,
|
|
104
|
+
` - A cached token can be revoked mid-run (some backends keep only the`,
|
|
105
|
+
` newest session per account): ensureToken() confirms it with one cheap`,
|
|
106
|
+
` authenticated call — once per process and again after any refusal —`,
|
|
107
|
+
` and mints a fresh one when needed; apiContext() and login() go through`,
|
|
108
|
+
` it. Never claim in a comment that something is re-checked unless the`,
|
|
109
|
+
` accessor the tests call does it.`,
|
|
110
|
+
` - Connections are checked by their live state before use and reopened if`,
|
|
111
|
+
` closed — never a memoised "already set up" flag. Tests never disconnect`,
|
|
112
|
+
` the shared connection; teardown runs once, globally.`,
|
|
113
|
+
`H5. NOTHING DESTRUCTIVE OUTSIDE TEST DATA.`,
|
|
114
|
+
` - reset() and any delete refuse unless BOTH hold: TESTFORGE_ALLOW_RESET`,
|
|
115
|
+
` === 'true', and the target is clearly not production (a local host, or a`,
|
|
116
|
+
` name containing "test"). Otherwise throw naming the target.`,
|
|
117
|
+
` - With SEED_MODE=existing, reset() and every seed write are skipped.`,
|
|
118
|
+
` - Deletes never reach beyond the test-data prefix and TEST_IDS, and`,
|
|
119
|
+
` children go by their owning key, never a bare drop.`,
|
|
120
|
+
`H6. RUNS ON A FRESH MACHINE given only the env file.`,
|
|
121
|
+
` - Use Node built-ins, the test runner and what the project already has.`,
|
|
122
|
+
` A data-store client the project lacks is required lazily, throws`,
|
|
123
|
+
` "testkit: missing package <name> — run: npm i -D <name>" when absent,`,
|
|
124
|
+
` and is listed in REQUIRED_PACKAGES (an array literal).`,
|
|
125
|
+
` - Never shell out to another language runtime, a framework CLI or a`,
|
|
126
|
+
` container tool.`,
|
|
127
|
+
` - Only env var NAMES for secrets and connections, never literal values.`,
|
|
128
|
+
` A developer provides the connection, any paid third party's test key`,
|
|
129
|
+
` and the accounts' credentials; setup() derives the rest.`,
|
|
130
|
+
`H7. THE APP'S OWN INTERRUPTIONS.`,
|
|
131
|
+
` - Pop-ups the app opens by itself for a signed-in or new user`,
|
|
132
|
+
` (promotions, consent, onboarding, permission prompts): suppressPopups(page)`,
|
|
133
|
+
` pre-sets the flags that suppress them with an init script, so they hold`,
|
|
134
|
+
` before the app's code runs and across navigations. login(page) calls`,
|
|
135
|
+
` it; a test that signs in through the UI calls it first.`,
|
|
136
|
+
` - Rate limits: if the limiter's state lives in a store the testkit can`,
|
|
137
|
+
` reach in local mode, resetRateLimits() clears ONLY the limiter entries,`,
|
|
138
|
+
` gated like reset(), and setup() calls it.`,
|
|
139
|
+
` - Codes the test cannot read (message or email code, magic link,`,
|
|
140
|
+
` third-party sign-in): bypassVerification(page, { url, role, sendUrl? })`,
|
|
141
|
+
` answers the final verify/register request at \`url\` with the app's REAL`,
|
|
142
|
+
` success shape (read the handler and the client that consumes it)`,
|
|
143
|
+
` carrying a REAL token for an env-provided account of that role, from the`,
|
|
144
|
+
` real login. When the environment cannot SEND the code, \`sendUrl\` is`,
|
|
145
|
+
` answered with the real success shape too. Scope each to exactly that`,
|
|
146
|
+
` request; everything after it runs against the real backend.`,
|
|
244
147
|
].join('\n');
|
|
148
|
+
/**
|
|
149
|
+
* Stamp written into every generated testkit: the contract it was built
|
|
150
|
+
* against. When TestForge's contract changes (a fix to how harnesses log in,
|
|
151
|
+
* seed or bypass verification), an existing testkit no longer matches and
|
|
152
|
+
* `generate` rebuilds it by itself — nobody has to remember a command.
|
|
153
|
+
*/
|
|
154
|
+
export const HARNESS_STAMP = `testforge-harness-contract: ${createHash('sha1').update(HARNESS_CONTRACT).digest('hex').slice(0, 12)}`;
|
|
155
|
+
/** Was this testkit built against the current contract? */
|
|
156
|
+
export function harnessIsCurrent(source) {
|
|
157
|
+
return source.includes(HARNESS_STAMP);
|
|
158
|
+
}
|
|
159
|
+
/** Prefix a freshly generated testkit with the stamp. */
|
|
160
|
+
export function stampHarness(source) {
|
|
161
|
+
// A rebuild is handed the old kit and may echo its stamp; keep only the current one.
|
|
162
|
+
return `// ${HARNESS_STAMP}\n${source.replace(/^\/\/ testforge-harness-contract: \w+\n/gm, '')}`;
|
|
163
|
+
}
|
|
245
164
|
/** Where an area's shared testkit lives, and how tests import it. */
|
|
246
165
|
export function harnessInfo(areaDir, framework, ext) {
|
|
247
166
|
// Jest tests are flat inside <areaDir> (e.g. .../test/service), so the
|
|
@@ -260,21 +179,21 @@ function dirnameOf(p) {
|
|
|
260
179
|
/** Build the prompt that generates one area's testkit. */
|
|
261
180
|
function buildHarnessPrompt(harnessPath, framework) {
|
|
262
181
|
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
182
|
+
// A rebuild must not break the suite already written against the old kit.
|
|
183
|
+
let previous = '';
|
|
184
|
+
try {
|
|
185
|
+
previous = readFileSync(harnessPath, 'utf-8');
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
// first build
|
|
189
|
+
}
|
|
263
190
|
return [
|
|
264
191
|
`You are a senior SDET writing the SHARED test harness ("testkit")`,
|
|
265
192
|
`that every test in this area will import. It must be correct against`,
|
|
266
193
|
`the project's REAL data layer and auth.`,
|
|
267
194
|
``,
|
|
268
195
|
`You are inside the repository with read-only tools (Read, Grep, Glob,`,
|
|
269
|
-
`LS).
|
|
270
|
-
`flow (login / refresh-token / OTP controllers + middleware) so setup()`,
|
|
271
|
-
`mints tokens the way the app actually does. Determine whether the app`,
|
|
272
|
-
`uses MORE THAN ONE database (e.g. a main DB, a chat/conversation DB, an`,
|
|
273
|
-
`identity/IM DB); if so, connect to and seed EACH via its own env-driven`,
|
|
274
|
-
`connection — do not collapse them into one. Also GREP the sibling test`,
|
|
275
|
-
`files next to this testkit for every process.env.<NAME> they read`,
|
|
276
|
-
`(especially *TOKEN*, *_ID, base URLs) and make setup() POPULATE those`,
|
|
277
|
-
`exact names — the tests must run without any hand-pasted token.`,
|
|
196
|
+
`LS). Follow THE METHOD below: read before you write.`,
|
|
278
197
|
``,
|
|
279
198
|
`This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area.`,
|
|
280
199
|
...(framework === 'jest'
|
|
@@ -293,6 +212,21 @@ function buildHarnessPrompt(harnessPath, framework) {
|
|
|
293
212
|
``,
|
|
294
213
|
`The file will be saved at EXACTLY: ${harnessPath}`,
|
|
295
214
|
`Compute any require() paths to app source RELATIVE to that location.`,
|
|
215
|
+
...(previous
|
|
216
|
+
? [
|
|
217
|
+
``,
|
|
218
|
+
`A TESTKIT ALREADY EXISTS at that path (below) and a suite of tests`,
|
|
219
|
+
`imports it. Rebuild it against the contract above WITHOUT breaking`,
|
|
220
|
+
`them: keep EVERY existing export, with the same name, parameters and`,
|
|
221
|
+
`return shape (GREP the sibling test files for how each one is used);`,
|
|
222
|
+
`change an implementation only where the contract requires it, and add`,
|
|
223
|
+
`the exports it requires. A removed or reshaped export fails every`,
|
|
224
|
+
`test that uses it.`,
|
|
225
|
+
`--- EXISTING TESTKIT ---`,
|
|
226
|
+
previous,
|
|
227
|
+
`--- END EXISTING TESTKIT ---`,
|
|
228
|
+
]
|
|
229
|
+
: []),
|
|
296
230
|
``,
|
|
297
231
|
`Output ONLY the testkit code (CommonJS). No prose, no markdown fences.`,
|
|
298
232
|
`The very first characters must be code. Do NOT write any file.`,
|
|
@@ -359,6 +293,24 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
|
|
|
359
293
|
`// Minimal Playwright config so testforge's generated specs run.`,
|
|
360
294
|
`// Install once: npm i -D @playwright/test && npx playwright install`,
|
|
361
295
|
`const { defineConfig } = require('@playwright/test');`,
|
|
296
|
+
`const fs = require('fs');`,
|
|
297
|
+
`const path = require('path');`,
|
|
298
|
+
``,
|
|
299
|
+
`// Load the env file named in testforge.config.json, so a plain`,
|
|
300
|
+
`// \`npx playwright test\` (or an editor's test runner) gets the same env as`,
|
|
301
|
+
`// \`testforge run\`. A variable already set in the shell wins.`,
|
|
302
|
+
`(() => {`,
|
|
303
|
+
` const home = [__dirname, process.cwd()].find((d) => fs.existsSync(path.join(d, 'testforge.config.json'))) || __dirname;`,
|
|
304
|
+
` let file = '.env.testing';`,
|
|
305
|
+
` try { file = JSON.parse(fs.readFileSync(path.join(home, 'testforge.config.json'), 'utf8')).envFile || file; } catch (_) {}`,
|
|
306
|
+
` let text = '';`,
|
|
307
|
+
` try { text = fs.readFileSync(path.resolve(home, file), 'utf8'); } catch (_) { return; }`,
|
|
308
|
+
` for (const line of text.split('\\n')) {`,
|
|
309
|
+
` const m = line.match(/^\\s*(?:export\\s+)?([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*(.*)$/);`,
|
|
310
|
+
` if (!m || process.env[m[1]] !== undefined) continue;`,
|
|
311
|
+
` process.env[m[1]] = m[2].trim().replace(/^(['"])(.*)\\1$/, '$2');`,
|
|
312
|
+
` }`,
|
|
313
|
+
`})();`,
|
|
362
314
|
``,
|
|
363
315
|
`module.exports = defineConfig({`,
|
|
364
316
|
` testDir: '${testDirRelToPackage}',`,
|
|
@@ -380,7 +332,8 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
|
|
|
380
332
|
` reporter: [['line'], [require.resolve('@activate-core/test-forge/reporters/playwright-reporter.cjs'), { category: ${JSON.stringify(category)} }]],`,
|
|
381
333
|
` use: {`,
|
|
382
334
|
` baseURL: process.env.BASE_URL || 'http://localhost:3000',`,
|
|
383
|
-
`
|
|
335
|
+
` // Kept for every failure: with no retries, 'on-first-retry' never records one.`,
|
|
336
|
+
` trace: 'retain-on-failure',`,
|
|
384
337
|
` screenshot: 'only-on-failure',`,
|
|
385
338
|
` },`,
|
|
386
339
|
`});`,
|
|
@@ -406,6 +359,8 @@ export function jestConfig(testDirRelToPackage, setupRelToPackage, category = 'a
|
|
|
406
359
|
` // Serial: the suites share ONE test DB and reset it between tests.`,
|
|
407
360
|
` maxWorkers: 1,`,
|
|
408
361
|
` forceExit: true,`,
|
|
362
|
+
` // Lets the reporter show what each test checks (it reads the test's oracle comments).`,
|
|
363
|
+
` testLocationInResults: true,`,
|
|
409
364
|
` // 'default' keeps console output; the second reporter auto-publishes the`,
|
|
410
365
|
` // run to the TestForge dashboard (needs TESTFORGE_KEY in env +`,
|
|
411
366
|
` // project/dashboardUrl in testforge.config.json).`,
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — pending tests carry their reason where the report can see it.
|
|
3
|
+
*
|
|
4
|
+
* A Playwright test declared `test.fixme(title, async () => {})` is reported
|
|
5
|
+
* as "fixme" with NO description: the reason the generator had (an open
|
|
6
|
+
* product question, a missing second identity) lived only in a comment, so
|
|
7
|
+
* every report showed a wall of reasonless skips that read like the tool
|
|
8
|
+
* giving up. The reason belongs in the test's annotation:
|
|
9
|
+
*
|
|
10
|
+
* test.fixme(title, { annotation: { type: 'fixme', description: reason } }, async () => {})
|
|
11
|
+
*
|
|
12
|
+
* normalisePending() rewrites the declaration form into that, taking the
|
|
13
|
+
* reason from the comment directly above the call or inside its body. It is
|
|
14
|
+
* deterministic (no AI), so a suite generated before this rule is fixed in
|
|
15
|
+
* place without regenerating anything. reasonlessPending() lists what is
|
|
16
|
+
* still without a reason, for the gate.
|
|
17
|
+
*/
|
|
18
|
+
import { matchParen } from '../gates/validate.js';
|
|
19
|
+
const DECL = /\b(test|it)\.(fixme|skip)\s*\(/g;
|
|
20
|
+
/** The title literal at `i` (quote, double quote or a backtick with no ${}), or null. */
|
|
21
|
+
function titleAt(s, i) {
|
|
22
|
+
const q = s[i];
|
|
23
|
+
if (q !== "'" && q !== '"' && q !== '`')
|
|
24
|
+
return null;
|
|
25
|
+
for (let j = i + 1; j < s.length; j++) {
|
|
26
|
+
if (s[j] === '\\') {
|
|
27
|
+
j++;
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
if (q === '`' && s[j] === '$' && s[j + 1] === '{')
|
|
31
|
+
return null;
|
|
32
|
+
if (s[j] === q)
|
|
33
|
+
return { end: j };
|
|
34
|
+
}
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
/** Comment text from the contiguous `//` lines right above `at`. */
|
|
38
|
+
function commentAbove(s, at) {
|
|
39
|
+
const lines = s.slice(0, at).split('\n');
|
|
40
|
+
lines.pop(); // the call's own line (only indentation precedes the call)
|
|
41
|
+
// One comment often explains a run of pending tests written one per line;
|
|
42
|
+
// each of them takes the reason from the comment above the run.
|
|
43
|
+
while (lines.length && /^\s*(?:test|it)\.(?:fixme|skip)\(.*\);?\s*$/.test(lines[lines.length - 1]))
|
|
44
|
+
lines.pop();
|
|
45
|
+
const out = [];
|
|
46
|
+
while (lines.length) {
|
|
47
|
+
const t = lines[lines.length - 1].trim();
|
|
48
|
+
if (!t.startsWith('//'))
|
|
49
|
+
break;
|
|
50
|
+
out.unshift(t.replace(/^\/\/\s?/, ''));
|
|
51
|
+
lines.pop();
|
|
52
|
+
}
|
|
53
|
+
return out.join(' ');
|
|
54
|
+
}
|
|
55
|
+
/** Comment text inside a body that holds nothing but comments. */
|
|
56
|
+
function commentInside(body) {
|
|
57
|
+
const inner = body.replace(/^[^{]*\{/, '').replace(/\}\s*$/, '');
|
|
58
|
+
const lines = inner.split('\n').map((l) => l.trim()).filter(Boolean);
|
|
59
|
+
if (!lines.length || !lines.every((l) => l.startsWith('//')))
|
|
60
|
+
return '';
|
|
61
|
+
return lines.map((l) => l.replace(/^\/\/\s?/, '')).join(' ');
|
|
62
|
+
}
|
|
63
|
+
function cleanReason(text) {
|
|
64
|
+
return text
|
|
65
|
+
.replace(/\b(TODO|FIXME)\s*:?\s*/g, '')
|
|
66
|
+
.replace(/\s+/g, ' ')
|
|
67
|
+
.trim();
|
|
68
|
+
}
|
|
69
|
+
/** Every declaration-form pending test (`test.fixme('title', ...)`); runtime `test.skip(cond, why)` is not one. */
|
|
70
|
+
function pendingDecls(s) {
|
|
71
|
+
const out = [];
|
|
72
|
+
for (const m of s.matchAll(DECL)) {
|
|
73
|
+
const open = m.index + m[0].length - 1;
|
|
74
|
+
let i = open + 1;
|
|
75
|
+
while (/\s/.test(s[i]))
|
|
76
|
+
i++;
|
|
77
|
+
const title = titleAt(s, i);
|
|
78
|
+
if (!title)
|
|
79
|
+
continue;
|
|
80
|
+
const close = matchParen(s, open);
|
|
81
|
+
if (close === -1)
|
|
82
|
+
continue;
|
|
83
|
+
let j = title.end + 1;
|
|
84
|
+
while (/\s/.test(s[j]))
|
|
85
|
+
j++;
|
|
86
|
+
if (s[j] !== ',' && s[j] !== ')')
|
|
87
|
+
continue; // a computed title ('a' + b) — leave it alone
|
|
88
|
+
while (/[\s,]/.test(s[j]))
|
|
89
|
+
j++;
|
|
90
|
+
out.push({ start: m.index, open, close, kind: m[2], titleEnd: title.end, hasDetails: s[j] === '{' });
|
|
91
|
+
}
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
/** Rewrite reasonless declaration-form pending tests so their reason is in the annotation. */
|
|
95
|
+
export function normalisePending(content) {
|
|
96
|
+
let s = content;
|
|
97
|
+
// Right to left, so earlier indices stay valid.
|
|
98
|
+
for (const d of pendingDecls(s).reverse()) {
|
|
99
|
+
if (d.hasDetails)
|
|
100
|
+
continue;
|
|
101
|
+
const body = s.slice(d.titleEnd + 1, d.close);
|
|
102
|
+
const reason = cleanReason(commentInside(body) || commentAbove(s, d.start));
|
|
103
|
+
if (!reason)
|
|
104
|
+
continue;
|
|
105
|
+
const details = `{ annotation: { type: '${d.kind}', description: ${JSON.stringify(reason)} } }`;
|
|
106
|
+
const rest = body.replace(/^\s*,?\s*/, '');
|
|
107
|
+
s = `${s.slice(0, d.titleEnd + 1)}, ${details}${rest ? `, ${rest}` : ''}${s.slice(d.close)}`;
|
|
108
|
+
}
|
|
109
|
+
// The reason now lives in the annotation; a "TODO" left in a test reads as
|
|
110
|
+
// unfinished work, which these comments are not.
|
|
111
|
+
return s.replace(/\/\/(\s*)(?:TODO|FIXME)\s*:?\s*/g, '//$1Pending: ');
|
|
112
|
+
}
|
|
113
|
+
/** Titles of declaration-form pending tests that still carry no reason. */
|
|
114
|
+
export function reasonlessPending(content) {
|
|
115
|
+
return pendingDecls(content)
|
|
116
|
+
.filter((d) => !d.hasDetails)
|
|
117
|
+
.map((d) => content.slice(d.open + 1, d.titleEnd + 1).trim().slice(1, 61));
|
|
118
|
+
}
|
|
@@ -19,14 +19,14 @@ const BANNER = [
|
|
|
19
19
|
`// NOT pass silently. Regenerate with AI once available, or implement`,
|
|
20
20
|
`// them by hand.`,
|
|
21
21
|
].join('\n');
|
|
22
|
+
const STUB_REASON = 'not generated: the AI was unavailable when this file was written — re-run `testforge generate` to fill it in';
|
|
22
23
|
/** One pending entry, phrased for the area's runner. */
|
|
23
24
|
function pendingLine(testCase, framework) {
|
|
24
25
|
const outcome = testCase.expect === 'pass' ? 'succeed' : 'be rejected';
|
|
25
|
-
const title = `${testCase.title}
|
|
26
|
-
const safe = title.replace(/"/g, '\\"');
|
|
26
|
+
const title = `${testCase.title} (expected to ${outcome})`;
|
|
27
27
|
return framework === 'jest'
|
|
28
|
-
? ` test.todo(
|
|
29
|
-
: ` test.fixme(
|
|
28
|
+
? ` test.todo(${JSON.stringify(`${title} — not generated yet`)});`
|
|
29
|
+
: ` test.fixme(${JSON.stringify(title)}, { annotation: { type: 'fixme', description: ${JSON.stringify(STUB_REASON)} } }, async () => {});`;
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
32
|
* Build the full deferred-stub file for a journey, correct for the area's
|