@activate-core/test-forge 2.3.1 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,6 +16,7 @@ import { createHash } from 'node:crypto';
16
16
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
17
17
  import { cleanOutput, runWithStdin } from './ai-generator.js';
18
18
  import { runAiSync } from './ai-usage.js';
19
+ import { transportHint } from '../project/detect.js';
19
20
  const MAP_PATH = 'testforge.md';
20
21
  /**
21
22
  * The fixed contract every testkit exposes and every generated test
@@ -23,182 +24,155 @@ const MAP_PATH = 'testforge.md';
23
24
  * are given this exact shape, so imports and usage line up.
24
25
  */
25
26
  export const HARNESS_CONTRACT = [
26
- `THE API. The testkit module MUST export exactly this stable API (CommonJS`,
27
- `module.exports); every generated test is written against it:`,
28
- ` - connect(): Promise<void> open the connection(s) to every data store`,
29
- ` this project's data lives in (there may be`,
30
- ` several — read the config).`,
31
- ` - disconnect(): Promise<void> close them.`,
32
- ` - reset(): Promise<void> remove ONLY test data. Gated — see H5.`,
33
- ` - seed(spec?): Promise<object> idempotently create the records the tests`,
34
- ` need and return their ids/handles.`,
35
- ` - TEST_IDS: object the stable test ids and handles.`,
36
- ` - setup(): Promise<void> the one-call entry point (see H4).`,
37
- ` - token(role?): string the current access token.`,
38
- ` - ensureToken(role?): Promise<string> that token, confirmed still accepted.`,
39
- ` - REQUIRED_PACKAGES: string[] packages the testkit needs (see H6).`,
40
- ` - resetRateLimits(), bypassVerification(), suppressPopups(), and one`,
41
- ` resolver per seeded entity — see H7 and H3.`,
42
- ` - unique(label?): string a value no other run produces (H8).`,
43
- ` - created(kind, id, cleanup?, role?): void record what the run made (H8).`,
44
- ` - the "run" role: token('run'), ensureToken('run'), login(page, 'run'),`,
45
- ` apiContext(request, 'run') — the account tests that CREATE data use (H8).`,
46
- ` - distinctRunAccount(): boolean true only when the "run" role is a`,
47
- ` different account from the fixture role (H8), so a test that needs two`,
48
- ` different people at once can tell whether it has them.`,
49
- `Plus, for Playwright suites:`,
50
- ` - openStep(page, step, input?): Promise<true | string> open a later step`,
51
- ` of a multi-step flow with what the earlier steps would have left (H9).`,
52
- ` - STEPS: object the step names openStep accepts (H9).`,
53
- ` - settledClick, fillFrameInputs, watchDialogs: re-exported unchanged from`,
54
- ` require('@activate-core/test-forge/reporters/ui-helpers.cjs') — never`,
55
- ` rewritten here.`,
56
- ` - apiContext(request): an authenticated API request context, created from`,
57
- ` the imported request MODULE (\`const { request: apiRequest } =`,
58
- ` require('@playwright/test'); apiRequest.newContext({ baseURL,`,
59
- ` extraHTTPHeaders })\`) — the \`request\` FIXTURE has no .newContext().`,
60
- ` - login(page): Promise<void> an authenticated UI session the app really`,
61
- ` accepts (see H2).`,
27
+ `EXPORTS (CommonJS). Export at least these; generated tests are written`,
28
+ `against them.`,
29
+ `All areas`,
30
+ `- setup(): Promise<void>, the one entry point (H4)`,
31
+ `- connect(), disconnect(): Promise<void>, for every data store (H1)`,
32
+ `- reset(): Promise<void>, which removes only test data and is gated (H5)`,
33
+ `- seed(spec?): Promise<object>, idempotent, returns ids and handles (H3)`,
34
+ `- TEST_IDS: object, stable and env-overridable ids (H3)`,
35
+ `- token(role?): string, and ensureToken(role?): Promise<string>, which`,
36
+ ` confirms the token is still accepted (H4)`,
37
+ `- one resolver per seeded entity that tests read back (H3)`,
38
+ `- unique(label?): string, and created(kind, id, cleanup?, role?): void (H8)`,
39
+ `- distinctRunAccount(): boolean (H8)`,
40
+ `- resetRateLimits(): Promise<void> (H7)`,
41
+ `- REQUIRED_PACKAGES: string[] (H6)`,
42
+ `- a 'run' role alongside the fixture roles (H8)`,
43
+ `Jest areas only`,
44
+ `- call(method, path, { body?, headers?, role? }):`,
45
+ ` Promise<{ status, headers, body }>`,
46
+ ` Every API test sends its requests through it, with the role's token. It`,
47
+ ` drives the server in-process with the framework's own inject when the`,
48
+ ` server is a Node app this package can require; otherwise (a server in`,
49
+ ` another language, or one this package cannot load) it sends HTTP to the`,
50
+ ` running server at the base URL named in the env file, with Node's fetch or`,
51
+ ` an HTTP client the project already has. Never require '@playwright/test':`,
52
+ ` it is not installed in backend packages, and a top-level require breaks`,
53
+ ` every suite.`,
54
+ `Playwright areas only`,
55
+ `- login(page, role?): Promise<void>, a session the app really accepts (H2).`,
56
+ ` It calls suppressPopups first.`,
57
+ `- apiContext(request, role?): an authenticated API context, created from the`,
58
+ ` imported request module:`,
59
+ ` const { request: apiRequest } = require('@playwright/test');`,
60
+ ` apiRequest.newContext({ baseURL, extraHTTPHeaders })`,
61
+ ` The request fixture has no newContext().`,
62
+ `- suppressPopups(page), and bypassVerification(page, { url, role, sendUrl? })`,
63
+ ` (H7)`,
64
+ `- openStep(page, step, input?): Promise<true | string>, and STEPS: object (H9)`,
65
+ `- find, settledClick, fillFrameInputs, watchDialogs: re-exported unchanged`,
66
+ ` from require('@activate-core/test-forge/reporters/ui-helpers.cjs')`,
62
67
  ``,
63
- `THE METHOD — EVERY PART OF THE TESTKIT IS PROVEN FROM THIS PROJECT. Each`,
64
- `function is a claim about how this product stores data, signs people in and`,
65
- `behaves at run time. Read the evidence first — the data layer and its`,
66
- `schema, the auth flow, the client's storage keys, the server's response`,
67
- `serializers — and decide everything from what you read in THIS project.`,
68
- `Nothing here names a stack. A testkit built on a guess fails every test that`,
69
- `uses it, and every one of those failures reads as the product being broken.`,
70
- ``,
71
- `H1. WHERE THE DATA LIVES. Read the data configuration: how many stores,`,
72
- ` which env var names each one's connection, which records live in which.`,
73
- ` Seed each record into the store it belongs to. When this repository`,
74
- ` does not own its data (the map or a related repository shows the data`,
75
- ` layer belongs to another codebase, e.g. a client talking to a separate`,
76
- ` API), connect to no store at all: seed and read state through the app's`,
77
- ` own API, and a state only a direct write could create is not seeded —`,
78
- ` the tests needing it skip with that reason.`,
79
- `H2. HOW PEOPLE SIGN IN. Read the real login, refresh and verification flow`,
80
- ` (handlers and middleware) and the client's guard: which storage keys or`,
81
- ` cookies it reads, and what it checks a token for. login(page) injects a`,
82
- ` real token the app accepts into exactly those keys.`,
83
- `H3. EVERY RECORD THE PRODUCT'S WAY.`,
84
- ` - Credentials come from the env file: every role the suite signs in as`,
85
- ` is an account whose identifier and password are read from it`,
86
- ` (TEST_<ROLE>_EMAIL / TEST_<ROLE>_PASSWORD, or the identifier the app`,
87
- ` signs in with). The testkit never hashes or invents a credential. Each`,
88
- ` role resolves from its OWN credential, never a fallback to another`,
89
- ` role's; a placeholder (DUMMY, replace_me, empty, <bracketed>) is absent,`,
90
- ` and a role whose credential is missing makes its tests skip naming it.`,
91
- ` - Other records and states are created the way the product creates them:`,
92
- ` through its API where one exists; otherwise through the data layer by`,
93
- ` following the transition that produces the state and copying what it`,
94
- ` writes — every NOT NULL column supplied, every id of the column's real`,
95
- ` type, upserts that never clobber creation metadata. Extra account`,
96
- ` states (unverified, banned, deleted, another role) copy the stored`,
97
- ` credential of an env-provided account, then set the state.`,
98
- ` - Owner-scoped data is seeded under the id the PRODUCT reports for the`,
99
- ` signed-in identity (ask its profile/whoami endpoint) — an identity`,
100
- ` provider's subject is a different value. Browser storage for a role`,
101
- ` carries that same id.`,
102
- ` - Every id and handle is env-overridable (process.env.TEST_<NAME>_ID ||`,
103
- ` '<stable default>'), so the suite can run against hand-made data.`,
104
- ` - One resolver per seeded entity the tests read back through the API: it`,
105
- ` fetches the entity and returns it, reading field names from the`,
106
- ` server's response serializer for that endpoint. Tests call it instead of`,
107
- ` re-reading the response.`,
108
- ` - When a dynamic page is served only for a listed set of parameters,`,
109
- ` expose a seeded id from that list; if no listed id has the data,`,
110
- ` setup() says which route and which ids.`,
111
- `H4. EVERY ACCESSOR STAYS TRUE FOR THE WHOLE RUN.`,
112
- ` - setup() is idempotent and safe from every beforeAll and a global setup:`,
113
- ` connect; seed (unless SEED_MODE=existing); mint each role's token once`,
114
- ` through the real login; cache it to a temp file for other workers; set`,
115
- ` process.env for every token/id variable the sibling tests read (grep`,
116
- ` them), so no token is ever pasted by hand.`,
117
- ` - A cached token can be revoked mid-run (some backends keep only the`,
118
- ` newest session per account): ensureToken() confirms it with one cheap`,
119
- ` authenticated call — once per process and again after any refusal —`,
120
- ` and mints a fresh one when needed; apiContext() and login() go through`,
121
- ` it. Never claim in a comment that something is re-checked unless the`,
122
- ` accessor the tests call does it.`,
123
- ` - Connections are checked by their live state before use and reopened if`,
124
- ` closed — never a memoised "already set up" flag. Tests never disconnect`,
125
- ` the shared connection; teardown runs once, globally.`,
126
- `H5. NOTHING DESTRUCTIVE OUTSIDE TEST DATA.`,
127
- ` - reset() and any delete refuse unless BOTH hold: TESTFORGE_ALLOW_RESET`,
128
- ` === 'true', and the target is clearly not production (a local host, or a`,
129
- ` name containing "test"). Otherwise throw naming the target.`,
130
- ` - With SEED_MODE=existing, reset() and every seed write are skipped.`,
131
- ` - Deletes never reach beyond the test-data prefix and TEST_IDS, and`,
132
- ` children go by their owning key, never a bare drop.`,
133
- `H6. RUNS ON A FRESH MACHINE given only the env file.`,
134
- ` - Use Node built-ins, the test runner and what the project already has.`,
135
- ` A data-store client the project lacks is required lazily, throws`,
136
- ` "testkit: missing package <name> — run: npm i -D <name>" when absent,`,
137
- ` and is listed in REQUIRED_PACKAGES (an array literal).`,
138
- ` - Never shell out to another language runtime, a framework CLI or a`,
139
- ` container tool.`,
140
- ` - Only env var NAMES for secrets and connections, never literal values.`,
141
- ` A developer provides the connection, any paid third party's test key`,
142
- ` and the accounts' credentials; setup() derives the rest.`,
143
- `H7. THE APP'S OWN INTERRUPTIONS.`,
144
- ` - Pop-ups the app opens by itself for a signed-in or new user`,
145
- ` (promotions, consent, onboarding, permission prompts): suppressPopups(page)`,
146
- ` pre-sets the flags that suppress them with an init script, so they hold`,
147
- ` before the app's code runs and across navigations. login(page) calls`,
148
- ` it; a test that signs in through the UI calls it first.`,
149
- ` - Rate limits: if the limiter's state lives in a store the testkit can`,
150
- ` reach in local mode, resetRateLimits() clears ONLY the limiter entries,`,
151
- ` gated like reset(), and setup() calls it.`,
152
- ` - Codes the test cannot read (message or email code, magic link,`,
153
- ` third-party sign-in): bypassVerification(page, { url, role, sendUrl? })`,
154
- ` answers the final verify/register request at \`url\` with the app's REAL`,
155
- ` success shape (read the handler and the client that consumes it)`,
156
- ` carrying a REAL token for an env-provided account of that role, from the`,
157
- ` real login. When the environment cannot SEND the code, \`sendUrl\` is`,
158
- ` answered with the real success shape too. Scope each to exactly that`,
159
- ` request; everything after it runs against the real backend.`,
160
- `H8. WHAT A RUN CREATES. Runs share accounts and environments, so data one`,
161
- ` run leaves behind must never change another run's result.`,
162
- ` - Load the run-data runtime once:`,
163
- ` const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
68
+ `H1. Where the data lives. Read the data config: how many stores, which env`,
69
+ ` var names each one, and which records live where. Seed each record into`,
70
+ ` its own store. If this repo does not own its data (the map or a related`,
71
+ ` repo shows it lives elsewhere), connect to no store. Seed and read through`,
72
+ ` the app's API instead. A state only a direct write could create is not`,
73
+ ` seeded, and the tests that need it skip with that reason.`,
74
+ `H2. How people sign in. Read the real login, refresh and verification`,
75
+ ` handlers, and the client's guard: which storage keys or cookies it reads,`,
76
+ ` and what it checks a token for. login() puts a real, accepted token into`,
77
+ ` exactly those keys.`,
78
+ `H3. Records made the product's way.`,
79
+ ` - Credentials come from the env file: TEST_<ROLE>_EMAIL and`,
80
+ ` TEST_<ROLE>_PASSWORD, or whatever identifier the app signs in with. Never`,
81
+ ` hash or invent a credential. Each role uses its own. DUMMY, replace_me,`,
82
+ ` an empty value or a <bracketed> value counts as absent, and that role's`,
83
+ ` tests skip, naming the role.`,
84
+ ` - Other records are created through the product's API where one exists.`,
85
+ ` Otherwise, create them through the data layer by copying what the`,
86
+ ` producing transition writes: every NOT NULL column, every id in its real`,
87
+ ` type, and upserts that never overwrite creation metadata. Extra account`,
88
+ ` states (unverified, banned, deleted, another role) copy an env account's`,
89
+ ` stored credential, then set the state.`,
90
+ ` - Owner-scoped data uses the id the product reports for the signed-in user`,
91
+ ` (from its profile or whoami endpoint), not an identity provider's`,
92
+ ` subject.`,
93
+ ` - Every id is env-overridable:`,
94
+ ` process.env.TEST_<NAME>_ID || '<stable default>'.`,
95
+ ` - Each resolver fetches its entity through the API, using field names from`,
96
+ ` that endpoint's serializer.`,
97
+ ` - If a dynamic page is served only for a listed set of parameters, expose a`,
98
+ ` seeded id from that list, or have setup() name the route and the ids.`,
99
+ `H4. Accessors stay true for the whole run.`,
100
+ ` - setup() is idempotent and safe to call from any beforeAll or global`,
101
+ ` setup. It connects; seeds, unless SEED_MODE=existing; mints each role's`,
102
+ ` token once through the real login; caches the tokens in a temp file for`,
103
+ ` other workers; and sets process.env for every token and id variable the`,
104
+ ` tests read (grep for them).`,
105
+ ` - ensureToken() confirms a cached token with one cheap authenticated call,`,
106
+ ` once per process and again after any refusal, and mints a new one if`,
107
+ ` needed. login(), apiContext() and call() all go through it.`,
108
+ ` - That check uses the lightest authenticated endpoint the role can call,`,
109
+ ` one that only says who is signed in, never a screen's data endpoint that`,
110
+ ` a journey tests: a bug there would stop every test before it starts.`,
111
+ ` - Only 401 or 403 means the token is refused: mint a new one once. Any other`,
112
+ ` answer (a 5xx, a 429, no answer) is the server's, not the token's: do not`,
113
+ ` mint again and do not retry per test. Remember it for the process and`,
114
+ ` throw "testkit: token check <METHOD> <path> answered <status>" so every`,
115
+ ` test that needs that role fails at once with the same reason.`,
116
+ ` - Check connections by their live state and reopen them if closed; do not`,
117
+ ` use "already set up" flags. Tests never disconnect the shared connection.`,
118
+ `H5. Nothing destructive outside test data.`,
119
+ ` - reset() and every delete throw, naming the target, unless`,
120
+ ` TESTFORGE_ALLOW_RESET === 'true' AND the target is clearly not`,
121
+ ` production (a local host, or a name containing "test").`,
122
+ ` - With SEED_MODE=existing, reset() and all seed writes are skipped.`,
123
+ ` - Deletes stay within the test-data prefix and TEST_IDS. Child records are`,
124
+ ` deleted by their owning key, never by a bare drop.`,
125
+ `H6. Runs on a fresh machine with only the env file.`,
126
+ ` - Use Node built-ins, the test runner and the project's existing packages.`,
127
+ ` A missing data-store client is required lazily, throws`,
128
+ ` "testkit: missing package <n> — run: npm i -D <n>", and is listed in`,
129
+ ` REQUIRED_PACKAGES (an array literal). TestForge checks that list before a`,
130
+ ` run and stops with the install command.`,
131
+ ` - Never shell out to another runtime, a framework CLI or a container tool.`,
132
+ ` Use env var names only, never secret values.`,
133
+ `H7. The app's own interruptions.`,
134
+ ` - suppressPopups(page) uses an init script to pre-set the flags that stop`,
135
+ ` promotions, consent, onboarding and permission prompts, before the app's`,
136
+ ` code runs and across navigations.`,
137
+ ` - resetRateLimits() clears only the limiter's entries, and only when they`,
138
+ ` live in a store the testkit can reach locally. It is gated like reset(),`,
139
+ ` and setup() calls it.`,
140
+ ` - bypassVerification answers the final verify request at url with the`,
141
+ ` app's real success shape, carrying a real token for an env account of`,
142
+ ` that role. It answers sendUrl the same way when the environment cannot`,
143
+ ` send the code. Each answer is scoped to exactly that request.`,
144
+ `H8. What a run creates.`,
145
+ ` - const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
164
146
  ` unique(label) returns runData.unique(label). created(kind, id, cleanup,`,
165
- ` role) calls runData.record({ kind, id, role, cleanup }), where cleanup is`,
166
- ` { method, url } of the product's own request that removes or cancels that`,
167
- ` record (absolute url; read the routes), or undefined when there is none.`,
168
- ` - Every testkit helper that creates a record through the API calls`,
169
- ` created() with it.`,
170
- ` - The "run" role. TEST_RUN_ACCOUNT_MODE picks the account:`,
171
- ` fixture (default) — the "run" role IS the customer role;`,
172
- ` pool — TEST_RUN_ACCOUNTS is a comma list of identifiers sharing`,
173
- ` TEST_RUN_ACCOUNTS_PASSWORD; pick one by a stable hash of runData.runId();`,
174
- ` signup — create a fresh account through the product's own sign-up, only`,
175
- ` when this environment can complete it (no code it cannot read and`,
176
- ` cannot bypass under H7); otherwise fall back to fixture and say so once.`,
177
- ` Mint and cache its token like any role. distinctRunAccount() returns true`,
178
- ` only in pool or signup mode, when the run account really differs from`,
179
- ` the fixture one.`,
180
- `H9. STEPS OF A FLOW. A later step of a multi-step flow (onboarding, a`,
181
- ` booking or checkout wizard, a sign-up in stages) depends on what the`,
182
- ` earlier steps stored — a choice, an address, a cart, a plan — in URL`,
183
- ` parameters, browser storage or the server. Opened by its URL alone it`,
184
- ` often sends the visitor back to an earlier step without a word, and a`,
185
- ` test then "fails" on the product's correct guard.`,
186
- ` - Read the flow's step components, its router guard and its store, and`,
187
- ` give openStep(page, step, input?) the way the product reaches that step:`,
188
- ` the parameters a real entry link carries, or the stored state the`,
189
- ` earlier steps write (pre-set with an init script before the app runs).`,
190
- ` \`input\` overrides the defaults (which choice, which address).`,
191
- ` - It waits until the step is really shown and resolves true; when the`,
192
- ` product still sends the visitor elsewhere it resolves a string naming`,
193
- ` the step it landed on, so the test can skip with that reason.`,
194
- ` - Offer EVERY step the flow has, up to and including its last page before`,
195
- ` the result — the summary, review or checkout page where the visitor`,
196
- ` confirms and pays — reaching a page that follows a choice (an operator,`,
197
- ` a plan, a slot) by making that choice through the UI, as a visitor would.`,
198
- ` Keep the names in one exported object STEPS (name → what identifies the`,
199
- ` page), and look the step up in it: an unknown name resolves`,
200
- ` 'unknown step "<name>" (steps: …)'. When the project has no such flow,`,
201
- ` export STEPS = {} and openStep resolving 'this product has no multi-step flow'.`,
147
+ ` role) calls runData.record({ kind, id, role, cleanup }). cleanup is the`,
148
+ ` { method, url } (absolute url) of the product's own request that removes`,
149
+ ` the record, or undefined if there is none. Every testkit helper that`,
150
+ ` creates a record calls created().`,
151
+ ` - TEST_RUN_ACCOUNT_MODE picks the 'run' account:`,
152
+ ` fixture (default): the same account as the customer role;`,
153
+ ` pool: one of TEST_RUN_ACCOUNTS (all sharing`,
154
+ ` TEST_RUN_ACCOUNTS_PASSWORD), chosen by a stable hash of`,
155
+ ` runData.runId();`,
156
+ ` signup: a fresh account through the product's own sign-up, only if this`,
157
+ ` environment can complete it (see H7); otherwise fall back to fixture`,
158
+ ` and say so once.`,
159
+ ` distinctRunAccount() is true only when the run account really differs`,
160
+ ` from the fixture account.`,
161
+ `H9. Steps of a flow (Playwright only).`,
162
+ ` - A later step (onboarding, a booking or checkout wizard, a staged sign-up)`,
163
+ ` depends on what the earlier steps stored. Read the step components, the`,
164
+ ` router guard and the store. openStep reaches the step the way the product`,
165
+ ` does: with the parameters a real entry link carries, or with the stored`,
166
+ ` state pre-set by an init script. input overrides the defaults.`,
167
+ ` - openStep resolves true once the step is really shown; otherwise it`,
168
+ ` resolves a string naming where it landed.`,
169
+ ` - Offer every step up to the last page before the result (the summary,`,
170
+ ` review or checkout page). Reach a page that follows a choice by making`,
171
+ ` that choice through the UI. STEPS maps each name to what identifies its`,
172
+ ` page; an unknown name resolves`,
173
+ ` 'unknown step "<name>" (steps: …)'. If there is no such flow, export`,
174
+ ` STEPS = {} and have openStep resolve`,
175
+ ` 'this product has no multi-step flow'.`,
202
176
  ].join('\n');
203
177
  /**
204
178
  * Stamp written into every generated testkit: the contract it was built
@@ -226,6 +200,19 @@ export function harnessInfo(areaDir, framework, ext) {
226
200
  const kitExt = ext.replace(/\.test|\.spec/, '') || '.js';
227
201
  return { dir: `${base}/helpers`, path: `${base}/helpers/testkit${kitExt}` };
228
202
  }
203
+ /** The map's data layer, which is all a testkit or a seed needs of it; the whole map when it has no such section. */
204
+ export function dataLayer(map) {
205
+ const at = map.search(/^##\s+Data Layer\b/im);
206
+ if (at < 0)
207
+ return map;
208
+ const rest = map.slice(at);
209
+ const next = rest.slice(1).search(/^##\s/m);
210
+ return (next < 0 ? rest : rest.slice(0, next + 1)).trim();
211
+ }
212
+ /** The folder a testkit sits in. */
213
+ function harnessDir(harnessPath) {
214
+ return dirnameOf(harnessPath);
215
+ }
229
216
  /** Tiny dirname without importing path just for one call. */
230
217
  function dirnameOf(p) {
231
218
  const i = p.replace(/\/+$/, '').lastIndexOf('/');
@@ -233,7 +220,7 @@ function dirnameOf(p) {
233
220
  }
234
221
  /** Build the prompt that generates one area's testkit. */
235
222
  function buildHarnessPrompt(harnessPath, framework) {
236
- const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
223
+ const map = dataLayer(existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '');
237
224
  // A rebuild must not break the suite already written against the old kit.
238
225
  let previous = '';
239
226
  try {
@@ -243,51 +230,38 @@ function buildHarnessPrompt(harnessPath, framework) {
243
230
  // first build
244
231
  }
245
232
  return [
246
- `You are a senior SDET writing the SHARED test harness ("testkit")`,
247
- `that every test in this area will import. It must be correct against`,
248
- `the project's REAL data layer and auth.`,
233
+ `You are a senior SDET writing the shared test harness ("testkit") that every`,
234
+ `test in this area imports. You are inside the repository with read-only tools`,
235
+ `(Read, Grep, Glob, LS). Read before you write. Every function is a claim about`,
236
+ `how this product stores data, signs people in and behaves at run time, and a`,
237
+ `testkit built on a guess makes every test that uses it fail as if the product`,
238
+ `were broken. Nothing here assumes a stack: decide everything from what you`,
239
+ `read.`,
249
240
  ``,
250
- `You are inside the repository with read-only tools (Read, Grep, Glob,`,
251
- `LS). Follow THE METHOD below: read before you write.`,
241
+ `This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area. The file will be saved at`,
242
+ `exactly ${harnessPath}; compute require() paths relative to it.`,
243
+ ...(framework === 'jest' ? [``, `How this project's server can be reached, from its dependencies:`, transportHint(harnessDir(harnessPath), framework)] : []),
252
244
  ``,
253
- `This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area.`,
254
- ...(framework === 'jest'
255
- ? [
256
- `Because this is a JEST/backend area, do NOT require or import`,
257
- `'@playwright/test' anywhere (it is not installed in a backend`,
258
- `package — a top-level require would break every suite that loads`,
259
- `this testkit). OMIT the Playwright-only apiContext/login exports.`,
260
- `Backend suites drive the server in-process via the framework's own`,
261
- `inject (or an already-installed HTTP client) — never Playwright,`,
262
- `and never a library this project has not installed.`,
263
- ``,
264
- ]
265
- : []),
266
245
  HARNESS_CONTRACT,
267
246
  ``,
268
- `The file will be saved at EXACTLY: ${harnessPath}`,
269
- `Compute any require() paths to app source RELATIVE to that location.`,
270
247
  ...(previous
271
248
  ? [
272
- ``,
273
- `A TESTKIT ALREADY EXISTS at that path (below) and a suite of tests`,
274
- `imports it. Rebuild it against the contract above WITHOUT breaking`,
275
- `them: keep EVERY existing export, with the same name, parameters and`,
276
- `return shape (GREP the sibling test files for how each one is used);`,
277
- `change an implementation only where the contract requires it, and add`,
278
- `the exports it requires. A removed or reshaped export fails every`,
279
- `test that uses it.`,
280
- `--- EXISTING TESTKIT ---`,
249
+ `<existing_testkit>`,
281
250
  previous,
282
- `--- END EXISTING TESTKIT ---`,
251
+ `</existing_testkit>`,
252
+ `Tests already import this testkit. Keep every existing export with the same`,
253
+ `name, parameters and return shape (grep the test files for how each one is`,
254
+ `used). Change an implementation only where the contract above requires it,`,
255
+ `and add any missing exports.`,
256
+ ``,
283
257
  ]
284
258
  : []),
285
- ``,
286
- `Output ONLY the testkit code (CommonJS). No prose, no markdown fences.`,
287
- `The very first characters must be code. Do NOT write any file.`,
288
- ``,
289
- `--- PROJECT MAP (data layer) ---`,
259
+ `<project_map_data_layer>`,
290
260
  map,
261
+ `</project_map_data_layer>`,
262
+ ``,
263
+ `Output only the CommonJS code: no prose and no fences, and the first`,
264
+ `characters must be code. Do not write any file.`,
291
265
  ].join('\n');
292
266
  }
293
267
  /** Generate one area's testkit content (sync). */
@@ -16,6 +16,8 @@ import { journeyDir } from '../paths.js';
16
16
  const NOT_RULES = /^(controls|coverage|test setup|questions for product|not requirements|pending decisions|suggested)\b/i;
17
17
  const TAG = /\[(spec|baseline|observed)\]/;
18
18
  const CONTROL_REF = /\bC\d+[a-z]?\b/g;
19
+ /** A criterion that names its own id: "R12 [spec] …". */
20
+ const OWN_ID = /^R(\d+)\s*[:.]?\s+(?=\[(?:spec|baseline|observed)\])/i;
19
21
  /** Split a journey doc into header, controls, rules and setup. Pure: no AI, no I/O. */
20
22
  export function readRules(markdown) {
21
23
  const lines = markdown.split('\n');
@@ -88,7 +90,7 @@ export function readRules(markdown) {
88
90
  if (numbered) {
89
91
  close();
90
92
  const tag = (TAG.exec(numbered[1])?.[1] ?? '');
91
- current = { rule: { id: `R${rules.length + 1}`, section, group, tag, text: numbered[1], controls: [] }, indent: Number.MAX_SAFE_INTEGER, owns: true };
93
+ current = { rule: { id: '', section, group, tag, text: numbered[1], controls: [] }, indent: Number.MAX_SAFE_INTEGER, owns: true };
92
94
  rules.push(current.rule);
93
95
  continue;
94
96
  }
@@ -106,7 +108,13 @@ export function readRules(markdown) {
106
108
  }
107
109
  close();
108
110
  const tag = (TAG.exec(bullet[2])?.[1] ?? '');
109
- current = { rule: { id: `R${rules.length + 1}`, section, group, tag, text: bullet[2], controls: [] }, indent };
111
+ const own = OWN_ID.exec(bullet[2]);
112
+ current = {
113
+ rule: own
114
+ ? { id: `R${Number(own[1])}`, section, group, tag, text: bullet[2].slice(own[0].length), controls: [], explicit: true }
115
+ : { id: '', section, group, tag, text: bullet[2], controls: [] },
116
+ indent,
117
+ };
110
118
  rules.push(current.rule);
111
119
  continue;
112
120
  }
@@ -118,11 +126,39 @@ export function readRules(markdown) {
118
126
  close();
119
127
  // Free text between rules (a section's lead-in) becomes context for the next rule's section.
120
128
  if (line.trim())
121
- rules.push({ id: `R${rules.length + 1}`, section, group, tag: '', text: line.trim(), controls: [...new Set(line.match(CONTROL_REF) ?? [])] });
129
+ rules.push({ id: '', section, group, tag: '', text: line.trim(), controls: [...new Set(line.match(CONTROL_REF) ?? [])] });
122
130
  }
123
131
  close();
132
+ numberRules(rules);
124
133
  return { header: header.join('\n').trim(), rules, controls, setup: setup.join('\n').trim() };
125
134
  }
135
+ /**
136
+ * Ids for the rules that do not name their own: R1, R2 … in reading order,
137
+ * as before, skipping any id the doc gives a rule itself. An id the doc uses
138
+ * twice keeps its first rule; the second is numbered like an unnamed one.
139
+ */
140
+ function numberRules(rules) {
141
+ const taken = new Set();
142
+ for (const r of rules) {
143
+ if (!r.explicit)
144
+ continue;
145
+ if (taken.has(r.id)) {
146
+ r.explicit = false;
147
+ r.id = '';
148
+ continue;
149
+ }
150
+ taken.add(r.id);
151
+ }
152
+ let n = 0;
153
+ for (const r of rules) {
154
+ if (r.id)
155
+ continue;
156
+ do
157
+ n += 1;
158
+ while (taken.has(`R${n}`));
159
+ r.id = `R${n}`;
160
+ }
161
+ }
126
162
  /** One line per rule — what the planner and the gate see instead of the doc. */
127
163
  export function ruleIndex(memory, tags = ['spec', 'baseline', 'observed']) {
128
164
  return memory.rules
@@ -34,7 +34,7 @@ import { normalisePending } from './pending.js';
34
34
  import { packageRoot } from '../run/run-tests.js';
35
35
  import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
36
36
  import { groundJourney } from '../docs/grounding.js';
37
- import { generateSeed } from './seed-generator.js';
37
+ import { generateSeed, seedProblem } from './seed-generator.js';
38
38
  import { dataAccess, writeRequiredData } from './required-data.js';
39
39
  import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
40
40
  import { testedTags } from './suite-planner.js';
@@ -91,7 +91,7 @@ export async function pool(items, limit, fn) {
91
91
  }
92
92
  /** Async version of generateValidated (for concurrent per-file gen). */
93
93
  async function generateValidatedAsync(generate, ext, gate,
94
- /** Why an earlier draft of the same file (from a batch) was refused. */
94
+ /** Why an earlier draft of the same file (from a batch) was refused, with that draft. */
95
95
  firstRefusal) {
96
96
  let content = '';
97
97
  let refusal = firstRefusal;
@@ -99,7 +99,7 @@ firstRefusal) {
99
99
  if (aiUnavailableReason())
100
100
  return null; // retrying a dead CLI only burns time
101
101
  try {
102
- // A second try is told why the first was refused; asked blind, it made the same file again.
102
+ // A second try sees what it wrote and why that was refused; asked blind, it made the same file again.
103
103
  content = await generate(refusal);
104
104
  }
105
105
  catch (e) {
@@ -110,8 +110,9 @@ firstRefusal) {
110
110
  const gateMsg = gate ? gate(content) : null;
111
111
  if (isSyntaxValid(content, ext) && !gateMsg)
112
112
  return content;
113
- refusal = gateMsg || syntaxReason(content, ext);
114
- noteAiRejected(refusal);
113
+ const reason = gateMsg || syntaxReason(content, ext);
114
+ refusal = { reason, previous: content };
115
+ noteAiRejected(reason);
115
116
  }
116
117
  return null;
117
118
  }
@@ -719,7 +720,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
719
720
  const draft = drafts.get(filePath);
720
721
  const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
721
722
  const { mem, facts } = await fileMemory(pf);
722
- const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
723
+ const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
723
724
  if (ai !== null) {
724
725
  content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
725
726
  status = 'ai';
@@ -799,10 +800,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
799
800
  }
800
801
  else if (useAI) {
801
802
  const seedFacts = await grounding();
802
- let seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt);
803
+ let seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt, seedProblem);
803
804
  // A usage limit is not a failed seed: wait for the AI like every other step, then try again.
804
805
  while (seed === null && aiUnavailableReason() && waitForLimit && (await waitForAi(config))) {
805
- seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt);
806
+ seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt, seedProblem);
806
807
  }
807
808
  if (seed !== null) {
808
809
  mkdirSync(dirname(seedPath), { recursive: true });