@activate-core/test-forge 2.3.1 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -17
- package/dist/cli/index.js +12 -2
- package/dist/core/docs/discover.js +62 -40
- package/dist/core/docs/doc-check.js +257 -0
- package/dist/core/docs/doc-guidance.js +3 -176
- package/dist/core/docs/grounding.js +37 -58
- package/dist/core/docs/reference-text.js +24 -6
- package/dist/core/docs/reference.js +23 -41
- package/dist/core/docs/review.js +17 -40
- package/dist/core/gates/validate.js +78 -0
- package/dist/core/generate/ai-generator.js +138 -87
- package/dist/core/generate/ai-usage.js +9 -1
- package/dist/core/generate/defect-review.js +69 -66
- package/dist/core/generate/doc-diff.js +25 -1
- package/dist/core/generate/harness.js +184 -210
- package/dist/core/generate/journey-memory.js +39 -3
- package/dist/core/generate/run-generate.js +9 -8
- package/dist/core/generate/seed-generator.js +56 -65
- package/dist/core/generate/update-journey.js +3 -1
- package/dist/core/project/init.js +98 -0
- package/dist/core/project/setup.js +15 -20
- package/dist/core/prompts/blocks.js +340 -0
- package/dist/core/run/name-scan.js +14 -6
- package/dist/core/run/run-tests.js +21 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +32 -1
- package/reporters/jest-reporter.cjs +7 -6
- package/reporters/playwright-reporter.cjs +5 -5
|
@@ -16,6 +16,7 @@ import { createHash } from 'node:crypto';
|
|
|
16
16
|
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
17
17
|
import { cleanOutput, runWithStdin } from './ai-generator.js';
|
|
18
18
|
import { runAiSync } from './ai-usage.js';
|
|
19
|
+
import { transportHint } from '../project/detect.js';
|
|
19
20
|
const MAP_PATH = 'testforge.md';
|
|
20
21
|
/**
|
|
21
22
|
* The fixed contract every testkit exposes and every generated test
|
|
@@ -23,182 +24,155 @@ const MAP_PATH = 'testforge.md';
|
|
|
23
24
|
* are given this exact shape, so imports and usage line up.
|
|
24
25
|
*/
|
|
25
26
|
export const HARNESS_CONTRACT = [
|
|
26
|
-
`
|
|
27
|
-
`
|
|
28
|
-
`
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
`
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
`
|
|
43
|
-
|
|
44
|
-
`
|
|
45
|
-
`
|
|
46
|
-
`
|
|
47
|
-
`
|
|
48
|
-
`
|
|
49
|
-
`
|
|
50
|
-
`
|
|
51
|
-
`
|
|
52
|
-
`
|
|
53
|
-
`
|
|
54
|
-
|
|
55
|
-
`
|
|
56
|
-
|
|
57
|
-
`
|
|
58
|
-
` require('@playwright/test')
|
|
59
|
-
` extraHTTPHeaders })
|
|
60
|
-
`
|
|
61
|
-
|
|
27
|
+
`EXPORTS (CommonJS). Export at least these; generated tests are written`,
|
|
28
|
+
`against them.`,
|
|
29
|
+
`All areas`,
|
|
30
|
+
`- setup(): Promise<void>, the one entry point (H4)`,
|
|
31
|
+
`- connect(), disconnect(): Promise<void>, for every data store (H1)`,
|
|
32
|
+
`- reset(): Promise<void>, which removes only test data and is gated (H5)`,
|
|
33
|
+
`- seed(spec?): Promise<object>, idempotent, returns ids and handles (H3)`,
|
|
34
|
+
`- TEST_IDS: object, stable and env-overridable ids (H3)`,
|
|
35
|
+
`- token(role?): string, and ensureToken(role?): Promise<string>, which`,
|
|
36
|
+
` confirms the token is still accepted (H4)`,
|
|
37
|
+
`- one resolver per seeded entity that tests read back (H3)`,
|
|
38
|
+
`- unique(label?): string, and created(kind, id, cleanup?, role?): void (H8)`,
|
|
39
|
+
`- distinctRunAccount(): boolean (H8)`,
|
|
40
|
+
`- resetRateLimits(): Promise<void> (H7)`,
|
|
41
|
+
`- REQUIRED_PACKAGES: string[] (H6)`,
|
|
42
|
+
`- a 'run' role alongside the fixture roles (H8)`,
|
|
43
|
+
`Jest areas only`,
|
|
44
|
+
`- call(method, path, { body?, headers?, role? }):`,
|
|
45
|
+
` Promise<{ status, headers, body }>`,
|
|
46
|
+
` Every API test sends its requests through it, with the role's token. It`,
|
|
47
|
+
` drives the server in-process with the framework's own inject when the`,
|
|
48
|
+
` server is a Node app this package can require; otherwise (a server in`,
|
|
49
|
+
` another language, or one this package cannot load) it sends HTTP to the`,
|
|
50
|
+
` running server at the base URL named in the env file, with Node's fetch or`,
|
|
51
|
+
` an HTTP client the project already has. Never require '@playwright/test':`,
|
|
52
|
+
` it is not installed in backend packages, and a top-level require breaks`,
|
|
53
|
+
` every suite.`,
|
|
54
|
+
`Playwright areas only`,
|
|
55
|
+
`- login(page, role?): Promise<void>, a session the app really accepts (H2).`,
|
|
56
|
+
` It calls suppressPopups first.`,
|
|
57
|
+
`- apiContext(request, role?): an authenticated API context, created from the`,
|
|
58
|
+
` imported request module:`,
|
|
59
|
+
` const { request: apiRequest } = require('@playwright/test');`,
|
|
60
|
+
` apiRequest.newContext({ baseURL, extraHTTPHeaders })`,
|
|
61
|
+
` The request fixture has no newContext().`,
|
|
62
|
+
`- suppressPopups(page), and bypassVerification(page, { url, role, sendUrl? })`,
|
|
63
|
+
` (H7)`,
|
|
64
|
+
`- openStep(page, step, input?): Promise<true | string>, and STEPS: object (H9)`,
|
|
65
|
+
`- find, settledClick, fillFrameInputs, watchDialogs: re-exported unchanged`,
|
|
66
|
+
` from require('@activate-core/test-forge/reporters/ui-helpers.cjs')`,
|
|
62
67
|
``,
|
|
63
|
-
`
|
|
64
|
-
`
|
|
65
|
-
`
|
|
66
|
-
`
|
|
67
|
-
`
|
|
68
|
-
`
|
|
69
|
-
`
|
|
70
|
-
|
|
71
|
-
`
|
|
72
|
-
`
|
|
73
|
-
`
|
|
74
|
-
`
|
|
75
|
-
`
|
|
76
|
-
`
|
|
77
|
-
`
|
|
78
|
-
`
|
|
79
|
-
`
|
|
80
|
-
`
|
|
81
|
-
`
|
|
82
|
-
`
|
|
83
|
-
`
|
|
84
|
-
`
|
|
85
|
-
`
|
|
86
|
-
` (
|
|
87
|
-
`
|
|
88
|
-
`
|
|
89
|
-
`
|
|
90
|
-
`
|
|
91
|
-
`
|
|
92
|
-
`
|
|
93
|
-
`
|
|
94
|
-
`
|
|
95
|
-
`
|
|
96
|
-
`
|
|
97
|
-
`
|
|
98
|
-
`
|
|
99
|
-
`
|
|
100
|
-
`
|
|
101
|
-
`
|
|
102
|
-
`
|
|
103
|
-
`
|
|
104
|
-
`
|
|
105
|
-
`
|
|
106
|
-
`
|
|
107
|
-
`
|
|
108
|
-
`
|
|
109
|
-
`
|
|
110
|
-
`
|
|
111
|
-
`
|
|
112
|
-
`
|
|
113
|
-
`
|
|
114
|
-
`
|
|
115
|
-
`
|
|
116
|
-
`
|
|
117
|
-
` -
|
|
118
|
-
`
|
|
119
|
-
`
|
|
120
|
-
`
|
|
121
|
-
`
|
|
122
|
-
`
|
|
123
|
-
`
|
|
124
|
-
`
|
|
125
|
-
`
|
|
126
|
-
`
|
|
127
|
-
`
|
|
128
|
-
`
|
|
129
|
-
`
|
|
130
|
-
`
|
|
131
|
-
`
|
|
132
|
-
`
|
|
133
|
-
`
|
|
134
|
-
`
|
|
135
|
-
`
|
|
136
|
-
`
|
|
137
|
-
`
|
|
138
|
-
`
|
|
139
|
-
`
|
|
140
|
-
` -
|
|
141
|
-
` A developer provides the connection, any paid third party's test key`,
|
|
142
|
-
` and the accounts' credentials; setup() derives the rest.`,
|
|
143
|
-
`H7. THE APP'S OWN INTERRUPTIONS.`,
|
|
144
|
-
` - Pop-ups the app opens by itself for a signed-in or new user`,
|
|
145
|
-
` (promotions, consent, onboarding, permission prompts): suppressPopups(page)`,
|
|
146
|
-
` pre-sets the flags that suppress them with an init script, so they hold`,
|
|
147
|
-
` before the app's code runs and across navigations. login(page) calls`,
|
|
148
|
-
` it; a test that signs in through the UI calls it first.`,
|
|
149
|
-
` - Rate limits: if the limiter's state lives in a store the testkit can`,
|
|
150
|
-
` reach in local mode, resetRateLimits() clears ONLY the limiter entries,`,
|
|
151
|
-
` gated like reset(), and setup() calls it.`,
|
|
152
|
-
` - Codes the test cannot read (message or email code, magic link,`,
|
|
153
|
-
` third-party sign-in): bypassVerification(page, { url, role, sendUrl? })`,
|
|
154
|
-
` answers the final verify/register request at \`url\` with the app's REAL`,
|
|
155
|
-
` success shape (read the handler and the client that consumes it)`,
|
|
156
|
-
` carrying a REAL token for an env-provided account of that role, from the`,
|
|
157
|
-
` real login. When the environment cannot SEND the code, \`sendUrl\` is`,
|
|
158
|
-
` answered with the real success shape too. Scope each to exactly that`,
|
|
159
|
-
` request; everything after it runs against the real backend.`,
|
|
160
|
-
`H8. WHAT A RUN CREATES. Runs share accounts and environments, so data one`,
|
|
161
|
-
` run leaves behind must never change another run's result.`,
|
|
162
|
-
` - Load the run-data runtime once:`,
|
|
163
|
-
` const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
|
|
68
|
+
`H1. Where the data lives. Read the data config: how many stores, which env`,
|
|
69
|
+
` var names each one, and which records live where. Seed each record into`,
|
|
70
|
+
` its own store. If this repo does not own its data (the map or a related`,
|
|
71
|
+
` repo shows it lives elsewhere), connect to no store. Seed and read through`,
|
|
72
|
+
` the app's API instead. A state only a direct write could create is not`,
|
|
73
|
+
` seeded, and the tests that need it skip with that reason.`,
|
|
74
|
+
`H2. How people sign in. Read the real login, refresh and verification`,
|
|
75
|
+
` handlers, and the client's guard: which storage keys or cookies it reads,`,
|
|
76
|
+
` and what it checks a token for. login() puts a real, accepted token into`,
|
|
77
|
+
` exactly those keys.`,
|
|
78
|
+
`H3. Records made the product's way.`,
|
|
79
|
+
` - Credentials come from the env file: TEST_<ROLE>_EMAIL and`,
|
|
80
|
+
` TEST_<ROLE>_PASSWORD, or whatever identifier the app signs in with. Never`,
|
|
81
|
+
` hash or invent a credential. Each role uses its own. DUMMY, replace_me,`,
|
|
82
|
+
` an empty value or a <bracketed> value counts as absent, and that role's`,
|
|
83
|
+
` tests skip, naming the role.`,
|
|
84
|
+
` - Other records are created through the product's API where one exists.`,
|
|
85
|
+
` Otherwise, create them through the data layer by copying what the`,
|
|
86
|
+
` producing transition writes: every NOT NULL column, every id in its real`,
|
|
87
|
+
` type, and upserts that never overwrite creation metadata. Extra account`,
|
|
88
|
+
` states (unverified, banned, deleted, another role) copy an env account's`,
|
|
89
|
+
` stored credential, then set the state.`,
|
|
90
|
+
` - Owner-scoped data uses the id the product reports for the signed-in user`,
|
|
91
|
+
` (from its profile or whoami endpoint), not an identity provider's`,
|
|
92
|
+
` subject.`,
|
|
93
|
+
` - Every id is env-overridable:`,
|
|
94
|
+
` process.env.TEST_<NAME>_ID || '<stable default>'.`,
|
|
95
|
+
` - Each resolver fetches its entity through the API, using field names from`,
|
|
96
|
+
` that endpoint's serializer.`,
|
|
97
|
+
` - If a dynamic page is served only for a listed set of parameters, expose a`,
|
|
98
|
+
` seeded id from that list, or have setup() name the route and the ids.`,
|
|
99
|
+
`H4. Accessors stay true for the whole run.`,
|
|
100
|
+
` - setup() is idempotent and safe to call from any beforeAll or global`,
|
|
101
|
+
` setup. It connects; seeds, unless SEED_MODE=existing; mints each role's`,
|
|
102
|
+
` token once through the real login; caches the tokens in a temp file for`,
|
|
103
|
+
` other workers; and sets process.env for every token and id variable the`,
|
|
104
|
+
` tests read (grep for them).`,
|
|
105
|
+
` - ensureToken() confirms a cached token with one cheap authenticated call,`,
|
|
106
|
+
` once per process and again after any refusal, and mints a new one if`,
|
|
107
|
+
` needed. login(), apiContext() and call() all go through it.`,
|
|
108
|
+
` - That check uses the lightest authenticated endpoint the role can call,`,
|
|
109
|
+
` one that only says who is signed in, never a screen's data endpoint that`,
|
|
110
|
+
` a journey tests: a bug there would stop every test before it starts.`,
|
|
111
|
+
` - Only 401 or 403 means the token is refused: mint a new one once. Any other`,
|
|
112
|
+
` answer (a 5xx, a 429, no answer) is the server's, not the token's: do not`,
|
|
113
|
+
` mint again and do not retry per test. Remember it for the process and`,
|
|
114
|
+
` throw "testkit: token check <METHOD> <path> answered <status>" so every`,
|
|
115
|
+
` test that needs that role fails at once with the same reason.`,
|
|
116
|
+
` - Check connections by their live state and reopen them if closed; do not`,
|
|
117
|
+
` use "already set up" flags. Tests never disconnect the shared connection.`,
|
|
118
|
+
`H5. Nothing destructive outside test data.`,
|
|
119
|
+
` - reset() and every delete throw, naming the target, unless`,
|
|
120
|
+
` TESTFORGE_ALLOW_RESET === 'true' AND the target is clearly not`,
|
|
121
|
+
` production (a local host, or a name containing "test").`,
|
|
122
|
+
` - With SEED_MODE=existing, reset() and all seed writes are skipped.`,
|
|
123
|
+
` - Deletes stay within the test-data prefix and TEST_IDS. Child records are`,
|
|
124
|
+
` deleted by their owning key, never by a bare drop.`,
|
|
125
|
+
`H6. Runs on a fresh machine with only the env file.`,
|
|
126
|
+
` - Use Node built-ins, the test runner and the project's existing packages.`,
|
|
127
|
+
` A missing data-store client is required lazily, throws`,
|
|
128
|
+
` "testkit: missing package <n> — run: npm i -D <n>", and is listed in`,
|
|
129
|
+
` REQUIRED_PACKAGES (an array literal). TestForge checks that list before a`,
|
|
130
|
+
` run and stops with the install command.`,
|
|
131
|
+
` - Never shell out to another runtime, a framework CLI or a container tool.`,
|
|
132
|
+
` Use env var names only, never secret values.`,
|
|
133
|
+
`H7. The app's own interruptions.`,
|
|
134
|
+
` - suppressPopups(page) uses an init script to pre-set the flags that stop`,
|
|
135
|
+
` promotions, consent, onboarding and permission prompts, before the app's`,
|
|
136
|
+
` code runs and across navigations.`,
|
|
137
|
+
` - resetRateLimits() clears only the limiter's entries, and only when they`,
|
|
138
|
+
` live in a store the testkit can reach locally. It is gated like reset(),`,
|
|
139
|
+
` and setup() calls it.`,
|
|
140
|
+
` - bypassVerification answers the final verify request at url with the`,
|
|
141
|
+
` app's real success shape, carrying a real token for an env account of`,
|
|
142
|
+
` that role. It answers sendUrl the same way when the environment cannot`,
|
|
143
|
+
` send the code. Each answer is scoped to exactly that request.`,
|
|
144
|
+
`H8. What a run creates.`,
|
|
145
|
+
` - const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
|
|
164
146
|
` unique(label) returns runData.unique(label). created(kind, id, cleanup,`,
|
|
165
|
-
` role) calls runData.record({ kind, id, role, cleanup })
|
|
166
|
-
` { method, url } of the product's own request that removes
|
|
167
|
-
`
|
|
168
|
-
`
|
|
169
|
-
`
|
|
170
|
-
`
|
|
171
|
-
`
|
|
172
|
-
`
|
|
173
|
-
`
|
|
174
|
-
` signup
|
|
175
|
-
`
|
|
176
|
-
`
|
|
177
|
-
`
|
|
178
|
-
`
|
|
179
|
-
`
|
|
180
|
-
`
|
|
181
|
-
`
|
|
182
|
-
`
|
|
183
|
-
`
|
|
184
|
-
`
|
|
185
|
-
`
|
|
186
|
-
`
|
|
187
|
-
`
|
|
188
|
-
`
|
|
189
|
-
`
|
|
190
|
-
`
|
|
191
|
-
`
|
|
192
|
-
`
|
|
193
|
-
`
|
|
194
|
-
` - Offer EVERY step the flow has, up to and including its last page before`,
|
|
195
|
-
` the result — the summary, review or checkout page where the visitor`,
|
|
196
|
-
` confirms and pays — reaching a page that follows a choice (an operator,`,
|
|
197
|
-
` a plan, a slot) by making that choice through the UI, as a visitor would.`,
|
|
198
|
-
` Keep the names in one exported object STEPS (name → what identifies the`,
|
|
199
|
-
` page), and look the step up in it: an unknown name resolves`,
|
|
200
|
-
` 'unknown step "<name>" (steps: …)'. When the project has no such flow,`,
|
|
201
|
-
` export STEPS = {} and openStep resolving 'this product has no multi-step flow'.`,
|
|
147
|
+
` role) calls runData.record({ kind, id, role, cleanup }). cleanup is the`,
|
|
148
|
+
` { method, url } (absolute url) of the product's own request that removes`,
|
|
149
|
+
` the record, or undefined if there is none. Every testkit helper that`,
|
|
150
|
+
` creates a record calls created().`,
|
|
151
|
+
` - TEST_RUN_ACCOUNT_MODE picks the 'run' account:`,
|
|
152
|
+
` fixture (default): the same account as the customer role;`,
|
|
153
|
+
` pool: one of TEST_RUN_ACCOUNTS (all sharing`,
|
|
154
|
+
` TEST_RUN_ACCOUNTS_PASSWORD), chosen by a stable hash of`,
|
|
155
|
+
` runData.runId();`,
|
|
156
|
+
` signup: a fresh account through the product's own sign-up, only if this`,
|
|
157
|
+
` environment can complete it (see H7); otherwise fall back to fixture`,
|
|
158
|
+
` and say so once.`,
|
|
159
|
+
` distinctRunAccount() is true only when the run account really differs`,
|
|
160
|
+
` from the fixture account.`,
|
|
161
|
+
`H9. Steps of a flow (Playwright only).`,
|
|
162
|
+
` - A later step (onboarding, a booking or checkout wizard, a staged sign-up)`,
|
|
163
|
+
` depends on what the earlier steps stored. Read the step components, the`,
|
|
164
|
+
` router guard and the store. openStep reaches the step the way the product`,
|
|
165
|
+
` does: with the parameters a real entry link carries, or with the stored`,
|
|
166
|
+
` state pre-set by an init script. input overrides the defaults.`,
|
|
167
|
+
` - openStep resolves true once the step is really shown; otherwise it`,
|
|
168
|
+
` resolves a string naming where it landed.`,
|
|
169
|
+
` - Offer every step up to the last page before the result (the summary,`,
|
|
170
|
+
` review or checkout page). Reach a page that follows a choice by making`,
|
|
171
|
+
` that choice through the UI. STEPS maps each name to what identifies its`,
|
|
172
|
+
` page; an unknown name resolves`,
|
|
173
|
+
` 'unknown step "<name>" (steps: …)'. If there is no such flow, export`,
|
|
174
|
+
` STEPS = {} and have openStep resolve`,
|
|
175
|
+
` 'this product has no multi-step flow'.`,
|
|
202
176
|
].join('\n');
|
|
203
177
|
/**
|
|
204
178
|
* Stamp written into every generated testkit: the contract it was built
|
|
@@ -226,6 +200,19 @@ export function harnessInfo(areaDir, framework, ext) {
|
|
|
226
200
|
const kitExt = ext.replace(/\.test|\.spec/, '') || '.js';
|
|
227
201
|
return { dir: `${base}/helpers`, path: `${base}/helpers/testkit${kitExt}` };
|
|
228
202
|
}
|
|
203
|
+
/** The map's data layer, which is all a testkit or a seed needs of it; the whole map when it has no such section. */
|
|
204
|
+
export function dataLayer(map) {
|
|
205
|
+
const at = map.search(/^##\s+Data Layer\b/im);
|
|
206
|
+
if (at < 0)
|
|
207
|
+
return map;
|
|
208
|
+
const rest = map.slice(at);
|
|
209
|
+
const next = rest.slice(1).search(/^##\s/m);
|
|
210
|
+
return (next < 0 ? rest : rest.slice(0, next + 1)).trim();
|
|
211
|
+
}
|
|
212
|
+
/** The folder a testkit sits in. */
|
|
213
|
+
function harnessDir(harnessPath) {
|
|
214
|
+
return dirnameOf(harnessPath);
|
|
215
|
+
}
|
|
229
216
|
/** Tiny dirname without importing path just for one call. */
|
|
230
217
|
function dirnameOf(p) {
|
|
231
218
|
const i = p.replace(/\/+$/, '').lastIndexOf('/');
|
|
@@ -233,7 +220,7 @@ function dirnameOf(p) {
|
|
|
233
220
|
}
|
|
234
221
|
/** Build the prompt that generates one area's testkit. */
|
|
235
222
|
function buildHarnessPrompt(harnessPath, framework) {
|
|
236
|
-
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
223
|
+
const map = dataLayer(existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '');
|
|
237
224
|
// A rebuild must not break the suite already written against the old kit.
|
|
238
225
|
let previous = '';
|
|
239
226
|
try {
|
|
@@ -243,51 +230,38 @@ function buildHarnessPrompt(harnessPath, framework) {
|
|
|
243
230
|
// first build
|
|
244
231
|
}
|
|
245
232
|
return [
|
|
246
|
-
`You are a senior SDET writing the
|
|
247
|
-
`
|
|
248
|
-
`
|
|
233
|
+
`You are a senior SDET writing the shared test harness ("testkit") that every`,
|
|
234
|
+
`test in this area imports. You are inside the repository with read-only tools`,
|
|
235
|
+
`(Read, Grep, Glob, LS). Read before you write. Every function is a claim about`,
|
|
236
|
+
`how this product stores data, signs people in and behaves at run time, and a`,
|
|
237
|
+
`testkit built on a guess makes every test that uses it fail as if the product`,
|
|
238
|
+
`were broken. Nothing here assumes a stack: decide everything from what you`,
|
|
239
|
+
`read.`,
|
|
249
240
|
``,
|
|
250
|
-
`
|
|
251
|
-
`
|
|
241
|
+
`This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area. The file will be saved at`,
|
|
242
|
+
`exactly ${harnessPath}; compute require() paths relative to it.`,
|
|
243
|
+
...(framework === 'jest' ? [``, `How this project's server can be reached, from its dependencies:`, transportHint(harnessDir(harnessPath), framework)] : []),
|
|
252
244
|
``,
|
|
253
|
-
`This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area.`,
|
|
254
|
-
...(framework === 'jest'
|
|
255
|
-
? [
|
|
256
|
-
`Because this is a JEST/backend area, do NOT require or import`,
|
|
257
|
-
`'@playwright/test' anywhere (it is not installed in a backend`,
|
|
258
|
-
`package — a top-level require would break every suite that loads`,
|
|
259
|
-
`this testkit). OMIT the Playwright-only apiContext/login exports.`,
|
|
260
|
-
`Backend suites drive the server in-process via the framework's own`,
|
|
261
|
-
`inject (or an already-installed HTTP client) — never Playwright,`,
|
|
262
|
-
`and never a library this project has not installed.`,
|
|
263
|
-
``,
|
|
264
|
-
]
|
|
265
|
-
: []),
|
|
266
245
|
HARNESS_CONTRACT,
|
|
267
246
|
``,
|
|
268
|
-
`The file will be saved at EXACTLY: ${harnessPath}`,
|
|
269
|
-
`Compute any require() paths to app source RELATIVE to that location.`,
|
|
270
247
|
...(previous
|
|
271
248
|
? [
|
|
272
|
-
|
|
273
|
-
`A TESTKIT ALREADY EXISTS at that path (below) and a suite of tests`,
|
|
274
|
-
`imports it. Rebuild it against the contract above WITHOUT breaking`,
|
|
275
|
-
`them: keep EVERY existing export, with the same name, parameters and`,
|
|
276
|
-
`return shape (GREP the sibling test files for how each one is used);`,
|
|
277
|
-
`change an implementation only where the contract requires it, and add`,
|
|
278
|
-
`the exports it requires. A removed or reshaped export fails every`,
|
|
279
|
-
`test that uses it.`,
|
|
280
|
-
`--- EXISTING TESTKIT ---`,
|
|
249
|
+
`<existing_testkit>`,
|
|
281
250
|
previous,
|
|
282
|
-
|
|
251
|
+
`</existing_testkit>`,
|
|
252
|
+
`Tests already import this testkit. Keep every existing export with the same`,
|
|
253
|
+
`name, parameters and return shape (grep the test files for how each one is`,
|
|
254
|
+
`used). Change an implementation only where the contract above requires it,`,
|
|
255
|
+
`and add any missing exports.`,
|
|
256
|
+
``,
|
|
283
257
|
]
|
|
284
258
|
: []),
|
|
285
|
-
|
|
286
|
-
`Output ONLY the testkit code (CommonJS). No prose, no markdown fences.`,
|
|
287
|
-
`The very first characters must be code. Do NOT write any file.`,
|
|
288
|
-
``,
|
|
289
|
-
`--- PROJECT MAP (data layer) ---`,
|
|
259
|
+
`<project_map_data_layer>`,
|
|
290
260
|
map,
|
|
261
|
+
`</project_map_data_layer>`,
|
|
262
|
+
``,
|
|
263
|
+
`Output only the CommonJS code: no prose and no fences, and the first`,
|
|
264
|
+
`characters must be code. Do not write any file.`,
|
|
291
265
|
].join('\n');
|
|
292
266
|
}
|
|
293
267
|
/** Generate one area's testkit content (sync). */
|
|
@@ -16,6 +16,8 @@ import { journeyDir } from '../paths.js';
|
|
|
16
16
|
const NOT_RULES = /^(controls|coverage|test setup|questions for product|not requirements|pending decisions|suggested)\b/i;
|
|
17
17
|
const TAG = /\[(spec|baseline|observed)\]/;
|
|
18
18
|
const CONTROL_REF = /\bC\d+[a-z]?\b/g;
|
|
19
|
+
/** A criterion that names its own id: "R12 [spec] …". */
|
|
20
|
+
const OWN_ID = /^R(\d+)\s*[:.]?\s+(?=\[(?:spec|baseline|observed)\])/i;
|
|
19
21
|
/** Split a journey doc into header, controls, rules and setup. Pure: no AI, no I/O. */
|
|
20
22
|
export function readRules(markdown) {
|
|
21
23
|
const lines = markdown.split('\n');
|
|
@@ -88,7 +90,7 @@ export function readRules(markdown) {
|
|
|
88
90
|
if (numbered) {
|
|
89
91
|
close();
|
|
90
92
|
const tag = (TAG.exec(numbered[1])?.[1] ?? '');
|
|
91
|
-
current = { rule: { id:
|
|
93
|
+
current = { rule: { id: '', section, group, tag, text: numbered[1], controls: [] }, indent: Number.MAX_SAFE_INTEGER, owns: true };
|
|
92
94
|
rules.push(current.rule);
|
|
93
95
|
continue;
|
|
94
96
|
}
|
|
@@ -106,7 +108,13 @@ export function readRules(markdown) {
|
|
|
106
108
|
}
|
|
107
109
|
close();
|
|
108
110
|
const tag = (TAG.exec(bullet[2])?.[1] ?? '');
|
|
109
|
-
|
|
111
|
+
const own = OWN_ID.exec(bullet[2]);
|
|
112
|
+
current = {
|
|
113
|
+
rule: own
|
|
114
|
+
? { id: `R${Number(own[1])}`, section, group, tag, text: bullet[2].slice(own[0].length), controls: [], explicit: true }
|
|
115
|
+
: { id: '', section, group, tag, text: bullet[2], controls: [] },
|
|
116
|
+
indent,
|
|
117
|
+
};
|
|
110
118
|
rules.push(current.rule);
|
|
111
119
|
continue;
|
|
112
120
|
}
|
|
@@ -118,11 +126,39 @@ export function readRules(markdown) {
|
|
|
118
126
|
close();
|
|
119
127
|
// Free text between rules (a section's lead-in) becomes context for the next rule's section.
|
|
120
128
|
if (line.trim())
|
|
121
|
-
rules.push({ id:
|
|
129
|
+
rules.push({ id: '', section, group, tag: '', text: line.trim(), controls: [...new Set(line.match(CONTROL_REF) ?? [])] });
|
|
122
130
|
}
|
|
123
131
|
close();
|
|
132
|
+
numberRules(rules);
|
|
124
133
|
return { header: header.join('\n').trim(), rules, controls, setup: setup.join('\n').trim() };
|
|
125
134
|
}
|
|
135
|
+
/**
|
|
136
|
+
* Ids for the rules that do not name their own: R1, R2 … in reading order,
|
|
137
|
+
* as before, skipping any id the doc gives a rule itself. An id the doc uses
|
|
138
|
+
* twice keeps its first rule; the second is numbered like an unnamed one.
|
|
139
|
+
*/
|
|
140
|
+
function numberRules(rules) {
|
|
141
|
+
const taken = new Set();
|
|
142
|
+
for (const r of rules) {
|
|
143
|
+
if (!r.explicit)
|
|
144
|
+
continue;
|
|
145
|
+
if (taken.has(r.id)) {
|
|
146
|
+
r.explicit = false;
|
|
147
|
+
r.id = '';
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
taken.add(r.id);
|
|
151
|
+
}
|
|
152
|
+
let n = 0;
|
|
153
|
+
for (const r of rules) {
|
|
154
|
+
if (r.id)
|
|
155
|
+
continue;
|
|
156
|
+
do
|
|
157
|
+
n += 1;
|
|
158
|
+
while (taken.has(`R${n}`));
|
|
159
|
+
r.id = `R${n}`;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
126
162
|
/** One line per rule — what the planner and the gate see instead of the doc. */
|
|
127
163
|
export function ruleIndex(memory, tags = ['spec', 'baseline', 'observed']) {
|
|
128
164
|
return memory.rules
|
|
@@ -34,7 +34,7 @@ import { normalisePending } from './pending.js';
|
|
|
34
34
|
import { packageRoot } from '../run/run-tests.js';
|
|
35
35
|
import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
|
|
36
36
|
import { groundJourney } from '../docs/grounding.js';
|
|
37
|
-
import { generateSeed } from './seed-generator.js';
|
|
37
|
+
import { generateSeed, seedProblem } from './seed-generator.js';
|
|
38
38
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
39
39
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
40
40
|
import { testedTags } from './suite-planner.js';
|
|
@@ -91,7 +91,7 @@ export async function pool(items, limit, fn) {
|
|
|
91
91
|
}
|
|
92
92
|
/** Async version of generateValidated (for concurrent per-file gen). */
|
|
93
93
|
async function generateValidatedAsync(generate, ext, gate,
|
|
94
|
-
/** Why an earlier draft of the same file (from a batch) was refused. */
|
|
94
|
+
/** Why an earlier draft of the same file (from a batch) was refused, with that draft. */
|
|
95
95
|
firstRefusal) {
|
|
96
96
|
let content = '';
|
|
97
97
|
let refusal = firstRefusal;
|
|
@@ -99,7 +99,7 @@ firstRefusal) {
|
|
|
99
99
|
if (aiUnavailableReason())
|
|
100
100
|
return null; // retrying a dead CLI only burns time
|
|
101
101
|
try {
|
|
102
|
-
// A second try
|
|
102
|
+
// A second try sees what it wrote and why that was refused; asked blind, it made the same file again.
|
|
103
103
|
content = await generate(refusal);
|
|
104
104
|
}
|
|
105
105
|
catch (e) {
|
|
@@ -110,8 +110,9 @@ firstRefusal) {
|
|
|
110
110
|
const gateMsg = gate ? gate(content) : null;
|
|
111
111
|
if (isSyntaxValid(content, ext) && !gateMsg)
|
|
112
112
|
return content;
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
const reason = gateMsg || syntaxReason(content, ext);
|
|
114
|
+
refusal = { reason, previous: content };
|
|
115
|
+
noteAiRejected(reason);
|
|
115
116
|
}
|
|
116
117
|
return null;
|
|
117
118
|
}
|
|
@@ -719,7 +720,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
719
720
|
const draft = drafts.get(filePath);
|
|
720
721
|
const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
|
|
721
722
|
const { mem, facts } = await fileMemory(pf);
|
|
722
|
-
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal
|
|
723
|
+
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
|
|
723
724
|
if (ai !== null) {
|
|
724
725
|
content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
|
|
725
726
|
status = 'ai';
|
|
@@ -799,10 +800,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
799
800
|
}
|
|
800
801
|
else if (useAI) {
|
|
801
802
|
const seedFacts = await grounding();
|
|
802
|
-
let seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt);
|
|
803
|
+
let seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt, seedProblem);
|
|
803
804
|
// A usage limit is not a failed seed: wait for the AI like every other step, then try again.
|
|
804
805
|
while (seed === null && aiUnavailableReason() && waitForLimit && (await waitForAi(config))) {
|
|
805
|
-
seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt);
|
|
806
|
+
seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt, seedProblem);
|
|
806
807
|
}
|
|
807
808
|
if (seed !== null) {
|
|
808
809
|
mkdirSync(dirname(seedPath), { recursive: true });
|