@activate-core/test-forge 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -17
- package/dist/cli/index.js +12 -2
- package/dist/core/docs/discover.js +62 -40
- package/dist/core/docs/doc-check.js +257 -0
- package/dist/core/docs/doc-guidance.js +3 -176
- package/dist/core/docs/grounding.js +37 -58
- package/dist/core/docs/reference-text.js +24 -6
- package/dist/core/docs/reference.js +23 -41
- package/dist/core/docs/review.js +17 -40
- package/dist/core/gates/validate.js +78 -0
- package/dist/core/generate/ai-generator.js +138 -87
- package/dist/core/generate/ai-usage.js +9 -1
- package/dist/core/generate/defect-review.js +69 -66
- package/dist/core/generate/doc-diff.js +25 -1
- package/dist/core/generate/harness.js +176 -210
- package/dist/core/generate/journey-memory.js +39 -3
- package/dist/core/generate/run-generate.js +9 -8
- package/dist/core/generate/seed-generator.js +56 -65
- package/dist/core/generate/update-journey.js +3 -1
- package/dist/core/project/init.js +98 -0
- package/dist/core/project/setup.js +15 -20
- package/dist/core/prompts/blocks.js +340 -0
- package/dist/core/run/name-scan.js +14 -6
- package/dist/core/run/run-tests.js +21 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +25 -1
- package/reporters/interface.cjs +28 -4
- package/reporters/jest-reporter.cjs +7 -6
- package/reporters/playwright-reporter.cjs +5 -5
|
@@ -16,6 +16,7 @@ import { createHash } from 'node:crypto';
|
|
|
16
16
|
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
17
17
|
import { cleanOutput, runWithStdin } from './ai-generator.js';
|
|
18
18
|
import { runAiSync } from './ai-usage.js';
|
|
19
|
+
import { transportHint } from '../project/detect.js';
|
|
19
20
|
const MAP_PATH = 'testforge.md';
|
|
20
21
|
/**
|
|
21
22
|
* The fixed contract every testkit exposes and every generated test
|
|
@@ -23,182 +24,147 @@ const MAP_PATH = 'testforge.md';
|
|
|
23
24
|
* are given this exact shape, so imports and usage line up.
|
|
24
25
|
*/
|
|
25
26
|
export const HARNESS_CONTRACT = [
|
|
26
|
-
`
|
|
27
|
-
`
|
|
28
|
-
`
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
`
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
`
|
|
43
|
-
|
|
44
|
-
`
|
|
45
|
-
`
|
|
46
|
-
`
|
|
47
|
-
`
|
|
48
|
-
`
|
|
49
|
-
`
|
|
50
|
-
`
|
|
51
|
-
`
|
|
52
|
-
`
|
|
53
|
-
`
|
|
54
|
-
|
|
55
|
-
`
|
|
56
|
-
|
|
57
|
-
`
|
|
58
|
-
` require('@playwright/test')
|
|
59
|
-
` extraHTTPHeaders })
|
|
60
|
-
`
|
|
61
|
-
|
|
27
|
+
`EXPORTS (CommonJS). Export at least these; generated tests are written`,
|
|
28
|
+
`against them.`,
|
|
29
|
+
`All areas`,
|
|
30
|
+
`- setup(): Promise<void>, the one entry point (H4)`,
|
|
31
|
+
`- connect(), disconnect(): Promise<void>, for every data store (H1)`,
|
|
32
|
+
`- reset(): Promise<void>, which removes only test data and is gated (H5)`,
|
|
33
|
+
`- seed(spec?): Promise<object>, idempotent, returns ids and handles (H3)`,
|
|
34
|
+
`- TEST_IDS: object, stable and env-overridable ids (H3)`,
|
|
35
|
+
`- token(role?): string, and ensureToken(role?): Promise<string>, which`,
|
|
36
|
+
` confirms the token is still accepted (H4)`,
|
|
37
|
+
`- one resolver per seeded entity that tests read back (H3)`,
|
|
38
|
+
`- unique(label?): string, and created(kind, id, cleanup?, role?): void (H8)`,
|
|
39
|
+
`- distinctRunAccount(): boolean (H8)`,
|
|
40
|
+
`- resetRateLimits(): Promise<void> (H7)`,
|
|
41
|
+
`- REQUIRED_PACKAGES: string[] (H6)`,
|
|
42
|
+
`- a 'run' role alongside the fixture roles (H8)`,
|
|
43
|
+
`Jest areas only`,
|
|
44
|
+
`- call(method, path, { body?, headers?, role? }):`,
|
|
45
|
+
` Promise<{ status, headers, body }>`,
|
|
46
|
+
` Every API test sends its requests through it, with the role's token. It`,
|
|
47
|
+
` drives the server in-process with the framework's own inject when the`,
|
|
48
|
+
` server is a Node app this package can require; otherwise (a server in`,
|
|
49
|
+
` another language, or one this package cannot load) it sends HTTP to the`,
|
|
50
|
+
` running server at the base URL named in the env file, with Node's fetch or`,
|
|
51
|
+
` an HTTP client the project already has. Never require '@playwright/test':`,
|
|
52
|
+
` it is not installed in backend packages, and a top-level require breaks`,
|
|
53
|
+
` every suite.`,
|
|
54
|
+
`Playwright areas only`,
|
|
55
|
+
`- login(page, role?): Promise<void>, a session the app really accepts (H2).`,
|
|
56
|
+
` It calls suppressPopups first.`,
|
|
57
|
+
`- apiContext(request, role?): an authenticated API context, created from the`,
|
|
58
|
+
` imported request module:`,
|
|
59
|
+
` const { request: apiRequest } = require('@playwright/test');`,
|
|
60
|
+
` apiRequest.newContext({ baseURL, extraHTTPHeaders })`,
|
|
61
|
+
` The request fixture has no newContext().`,
|
|
62
|
+
`- suppressPopups(page), and bypassVerification(page, { url, role, sendUrl? })`,
|
|
63
|
+
` (H7)`,
|
|
64
|
+
`- openStep(page, step, input?): Promise<true | string>, and STEPS: object (H9)`,
|
|
65
|
+
`- find, settledClick, fillFrameInputs, watchDialogs: re-exported unchanged`,
|
|
66
|
+
` from require('@activate-core/test-forge/reporters/ui-helpers.cjs')`,
|
|
62
67
|
``,
|
|
63
|
-
`
|
|
64
|
-
`
|
|
65
|
-
`
|
|
66
|
-
`
|
|
67
|
-
`
|
|
68
|
-
`
|
|
69
|
-
`
|
|
70
|
-
|
|
71
|
-
`
|
|
72
|
-
`
|
|
73
|
-
`
|
|
74
|
-
`
|
|
75
|
-
`
|
|
76
|
-
`
|
|
77
|
-
`
|
|
78
|
-
`
|
|
79
|
-
`
|
|
80
|
-
`
|
|
81
|
-
`
|
|
82
|
-
`
|
|
83
|
-
`
|
|
84
|
-
`
|
|
85
|
-
`
|
|
86
|
-
` (
|
|
87
|
-
`
|
|
88
|
-
`
|
|
89
|
-
`
|
|
90
|
-
`
|
|
91
|
-
`
|
|
92
|
-
`
|
|
93
|
-
`
|
|
94
|
-
`
|
|
95
|
-
`
|
|
96
|
-
`
|
|
97
|
-
`
|
|
98
|
-
`
|
|
99
|
-
`
|
|
100
|
-
`
|
|
101
|
-
`
|
|
102
|
-
`
|
|
103
|
-
`
|
|
104
|
-
`
|
|
105
|
-
`
|
|
106
|
-
`
|
|
107
|
-
`
|
|
108
|
-
`
|
|
109
|
-
`
|
|
110
|
-
`
|
|
111
|
-
`
|
|
112
|
-
`
|
|
113
|
-
`
|
|
114
|
-
`
|
|
115
|
-
`
|
|
116
|
-
`
|
|
117
|
-
`
|
|
118
|
-
`
|
|
119
|
-
`
|
|
120
|
-
`
|
|
121
|
-
`
|
|
122
|
-
`
|
|
123
|
-
`
|
|
124
|
-
`
|
|
125
|
-
` the
|
|
126
|
-
`
|
|
127
|
-
` -
|
|
128
|
-
`
|
|
129
|
-
`
|
|
130
|
-
`
|
|
131
|
-
`
|
|
132
|
-
`
|
|
133
|
-
`H6. RUNS ON A FRESH MACHINE given only the env file.`,
|
|
134
|
-
` - Use Node built-ins, the test runner and what the project already has.`,
|
|
135
|
-
` A data-store client the project lacks is required lazily, throws`,
|
|
136
|
-
` "testkit: missing package <name> — run: npm i -D <name>" when absent,`,
|
|
137
|
-
` and is listed in REQUIRED_PACKAGES (an array literal).`,
|
|
138
|
-
` - Never shell out to another language runtime, a framework CLI or a`,
|
|
139
|
-
` container tool.`,
|
|
140
|
-
` - Only env var NAMES for secrets and connections, never literal values.`,
|
|
141
|
-
` A developer provides the connection, any paid third party's test key`,
|
|
142
|
-
` and the accounts' credentials; setup() derives the rest.`,
|
|
143
|
-
`H7. THE APP'S OWN INTERRUPTIONS.`,
|
|
144
|
-
` - Pop-ups the app opens by itself for a signed-in or new user`,
|
|
145
|
-
` (promotions, consent, onboarding, permission prompts): suppressPopups(page)`,
|
|
146
|
-
` pre-sets the flags that suppress them with an init script, so they hold`,
|
|
147
|
-
` before the app's code runs and across navigations. login(page) calls`,
|
|
148
|
-
` it; a test that signs in through the UI calls it first.`,
|
|
149
|
-
` - Rate limits: if the limiter's state lives in a store the testkit can`,
|
|
150
|
-
` reach in local mode, resetRateLimits() clears ONLY the limiter entries,`,
|
|
151
|
-
` gated like reset(), and setup() calls it.`,
|
|
152
|
-
` - Codes the test cannot read (message or email code, magic link,`,
|
|
153
|
-
` third-party sign-in): bypassVerification(page, { url, role, sendUrl? })`,
|
|
154
|
-
` answers the final verify/register request at \`url\` with the app's REAL`,
|
|
155
|
-
` success shape (read the handler and the client that consumes it)`,
|
|
156
|
-
` carrying a REAL token for an env-provided account of that role, from the`,
|
|
157
|
-
` real login. When the environment cannot SEND the code, \`sendUrl\` is`,
|
|
158
|
-
` answered with the real success shape too. Scope each to exactly that`,
|
|
159
|
-
` request; everything after it runs against the real backend.`,
|
|
160
|
-
`H8. WHAT A RUN CREATES. Runs share accounts and environments, so data one`,
|
|
161
|
-
` run leaves behind must never change another run's result.`,
|
|
162
|
-
` - Load the run-data runtime once:`,
|
|
163
|
-
` const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
|
|
68
|
+
`H1. Where the data lives. Read the data config: how many stores, which env`,
|
|
69
|
+
` var names each one, and which records live where. Seed each record into`,
|
|
70
|
+
` its own store. If this repo does not own its data (the map or a related`,
|
|
71
|
+
` repo shows it lives elsewhere), connect to no store. Seed and read through`,
|
|
72
|
+
` the app's API instead. A state only a direct write could create is not`,
|
|
73
|
+
` seeded, and the tests that need it skip with that reason.`,
|
|
74
|
+
`H2. How people sign in. Read the real login, refresh and verification`,
|
|
75
|
+
` handlers, and the client's guard: which storage keys or cookies it reads,`,
|
|
76
|
+
` and what it checks a token for. login() puts a real, accepted token into`,
|
|
77
|
+
` exactly those keys.`,
|
|
78
|
+
`H3. Records made the product's way.`,
|
|
79
|
+
` - Credentials come from the env file: TEST_<ROLE>_EMAIL and`,
|
|
80
|
+
` TEST_<ROLE>_PASSWORD, or whatever identifier the app signs in with. Never`,
|
|
81
|
+
` hash or invent a credential. Each role uses its own. DUMMY, replace_me,`,
|
|
82
|
+
` an empty value or a <bracketed> value counts as absent, and that role's`,
|
|
83
|
+
` tests skip, naming the role.`,
|
|
84
|
+
` - Other records are created through the product's API where one exists.`,
|
|
85
|
+
` Otherwise, create them through the data layer by copying what the`,
|
|
86
|
+
` producing transition writes: every NOT NULL column, every id in its real`,
|
|
87
|
+
` type, and upserts that never overwrite creation metadata. Extra account`,
|
|
88
|
+
` states (unverified, banned, deleted, another role) copy an env account's`,
|
|
89
|
+
` stored credential, then set the state.`,
|
|
90
|
+
` - Owner-scoped data uses the id the product reports for the signed-in user`,
|
|
91
|
+
` (from its profile or whoami endpoint), not an identity provider's`,
|
|
92
|
+
` subject.`,
|
|
93
|
+
` - Every id is env-overridable:`,
|
|
94
|
+
` process.env.TEST_<NAME>_ID || '<stable default>'.`,
|
|
95
|
+
` - Each resolver fetches its entity through the API, using field names from`,
|
|
96
|
+
` that endpoint's serializer.`,
|
|
97
|
+
` - If a dynamic page is served only for a listed set of parameters, expose a`,
|
|
98
|
+
` seeded id from that list, or have setup() name the route and the ids.`,
|
|
99
|
+
`H4. Accessors stay true for the whole run.`,
|
|
100
|
+
` - setup() is idempotent and safe to call from any beforeAll or global`,
|
|
101
|
+
` setup. It connects; seeds, unless SEED_MODE=existing; mints each role's`,
|
|
102
|
+
` token once through the real login; caches the tokens in a temp file for`,
|
|
103
|
+
` other workers; and sets process.env for every token and id variable the`,
|
|
104
|
+
` tests read (grep for them).`,
|
|
105
|
+
` - ensureToken() confirms a cached token with one cheap authenticated call,`,
|
|
106
|
+
` once per process and again after any refusal, and mints a new one if`,
|
|
107
|
+
` needed. login(), apiContext() and call() all go through it.`,
|
|
108
|
+
` - Check connections by their live state and reopen them if closed; do not`,
|
|
109
|
+
` use "already set up" flags. Tests never disconnect the shared connection.`,
|
|
110
|
+
`H5. Nothing destructive outside test data.`,
|
|
111
|
+
` - reset() and every delete throw, naming the target, unless`,
|
|
112
|
+
` TESTFORGE_ALLOW_RESET === 'true' AND the target is clearly not`,
|
|
113
|
+
` production (a local host, or a name containing "test").`,
|
|
114
|
+
` - With SEED_MODE=existing, reset() and all seed writes are skipped.`,
|
|
115
|
+
` - Deletes stay within the test-data prefix and TEST_IDS. Child records are`,
|
|
116
|
+
` deleted by their owning key, never by a bare drop.`,
|
|
117
|
+
`H6. Runs on a fresh machine with only the env file.`,
|
|
118
|
+
` - Use Node built-ins, the test runner and the project's existing packages.`,
|
|
119
|
+
` A missing data-store client is required lazily, throws`,
|
|
120
|
+
` "testkit: missing package <n> — run: npm i -D <n>", and is listed in`,
|
|
121
|
+
` REQUIRED_PACKAGES (an array literal). TestForge checks that list before a`,
|
|
122
|
+
` run and stops with the install command.`,
|
|
123
|
+
` - Never shell out to another runtime, a framework CLI or a container tool.`,
|
|
124
|
+
` Use env var names only, never secret values.`,
|
|
125
|
+
`H7. The app's own interruptions.`,
|
|
126
|
+
` - suppressPopups(page) uses an init script to pre-set the flags that stop`,
|
|
127
|
+
` promotions, consent, onboarding and permission prompts, before the app's`,
|
|
128
|
+
` code runs and across navigations.`,
|
|
129
|
+
` - resetRateLimits() clears only the limiter's entries, and only when they`,
|
|
130
|
+
` live in a store the testkit can reach locally. It is gated like reset(),`,
|
|
131
|
+
` and setup() calls it.`,
|
|
132
|
+
` - bypassVerification answers the final verify request at url with the`,
|
|
133
|
+
` app's real success shape, carrying a real token for an env account of`,
|
|
134
|
+
` that role. It answers sendUrl the same way when the environment cannot`,
|
|
135
|
+
` send the code. Each answer is scoped to exactly that request.`,
|
|
136
|
+
`H8. What a run creates.`,
|
|
137
|
+
` - const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
|
|
164
138
|
` unique(label) returns runData.unique(label). created(kind, id, cleanup,`,
|
|
165
|
-
` role) calls runData.record({ kind, id, role, cleanup })
|
|
166
|
-
` { method, url } of the product's own request that removes
|
|
167
|
-
`
|
|
168
|
-
`
|
|
169
|
-
`
|
|
170
|
-
`
|
|
171
|
-
`
|
|
172
|
-
`
|
|
173
|
-
`
|
|
174
|
-
` signup
|
|
175
|
-
`
|
|
176
|
-
`
|
|
177
|
-
`
|
|
178
|
-
`
|
|
179
|
-
`
|
|
180
|
-
`
|
|
181
|
-
`
|
|
182
|
-
`
|
|
183
|
-
`
|
|
184
|
-
`
|
|
185
|
-
`
|
|
186
|
-
`
|
|
187
|
-
`
|
|
188
|
-
`
|
|
189
|
-
`
|
|
190
|
-
`
|
|
191
|
-
`
|
|
192
|
-
`
|
|
193
|
-
`
|
|
194
|
-
` - Offer EVERY step the flow has, up to and including its last page before`,
|
|
195
|
-
` the result — the summary, review or checkout page where the visitor`,
|
|
196
|
-
` confirms and pays — reaching a page that follows a choice (an operator,`,
|
|
197
|
-
` a plan, a slot) by making that choice through the UI, as a visitor would.`,
|
|
198
|
-
` Keep the names in one exported object STEPS (name → what identifies the`,
|
|
199
|
-
` page), and look the step up in it: an unknown name resolves`,
|
|
200
|
-
` 'unknown step "<name>" (steps: …)'. When the project has no such flow,`,
|
|
201
|
-
` export STEPS = {} and openStep resolving 'this product has no multi-step flow'.`,
|
|
139
|
+
` role) calls runData.record({ kind, id, role, cleanup }). cleanup is the`,
|
|
140
|
+
` { method, url } (absolute url) of the product's own request that removes`,
|
|
141
|
+
` the record, or undefined if there is none. Every testkit helper that`,
|
|
142
|
+
` creates a record calls created().`,
|
|
143
|
+
` - TEST_RUN_ACCOUNT_MODE picks the 'run' account:`,
|
|
144
|
+
` fixture (default): the same account as the customer role;`,
|
|
145
|
+
` pool: one of TEST_RUN_ACCOUNTS (all sharing`,
|
|
146
|
+
` TEST_RUN_ACCOUNTS_PASSWORD), chosen by a stable hash of`,
|
|
147
|
+
` runData.runId();`,
|
|
148
|
+
` signup: a fresh account through the product's own sign-up, only if this`,
|
|
149
|
+
` environment can complete it (see H7); otherwise fall back to fixture`,
|
|
150
|
+
` and say so once.`,
|
|
151
|
+
` distinctRunAccount() is true only when the run account really differs`,
|
|
152
|
+
` from the fixture account.`,
|
|
153
|
+
`H9. Steps of a flow (Playwright only).`,
|
|
154
|
+
` - A later step (onboarding, a booking or checkout wizard, a staged sign-up)`,
|
|
155
|
+
` depends on what the earlier steps stored. Read the step components, the`,
|
|
156
|
+
` router guard and the store. openStep reaches the step the way the product`,
|
|
157
|
+
` does: with the parameters a real entry link carries, or with the stored`,
|
|
158
|
+
` state pre-set by an init script. input overrides the defaults.`,
|
|
159
|
+
` - openStep resolves true once the step is really shown; otherwise it`,
|
|
160
|
+
` resolves a string naming where it landed.`,
|
|
161
|
+
` - Offer every step up to the last page before the result (the summary,`,
|
|
162
|
+
` review or checkout page). Reach a page that follows a choice by making`,
|
|
163
|
+
` that choice through the UI. STEPS maps each name to what identifies its`,
|
|
164
|
+
` page; an unknown name resolves`,
|
|
165
|
+
` 'unknown step "<name>" (steps: …)'. If there is no such flow, export`,
|
|
166
|
+
` STEPS = {} and have openStep resolve`,
|
|
167
|
+
` 'this product has no multi-step flow'.`,
|
|
202
168
|
].join('\n');
|
|
203
169
|
/**
|
|
204
170
|
* Stamp written into every generated testkit: the contract it was built
|
|
@@ -226,6 +192,19 @@ export function harnessInfo(areaDir, framework, ext) {
|
|
|
226
192
|
const kitExt = ext.replace(/\.test|\.spec/, '') || '.js';
|
|
227
193
|
return { dir: `${base}/helpers`, path: `${base}/helpers/testkit${kitExt}` };
|
|
228
194
|
}
|
|
195
|
+
/** The map's data layer, which is all a testkit or a seed needs of it; the whole map when it has no such section. */
|
|
196
|
+
export function dataLayer(map) {
|
|
197
|
+
const at = map.search(/^##\s+Data Layer\b/im);
|
|
198
|
+
if (at < 0)
|
|
199
|
+
return map;
|
|
200
|
+
const rest = map.slice(at);
|
|
201
|
+
const next = rest.slice(1).search(/^##\s/m);
|
|
202
|
+
return (next < 0 ? rest : rest.slice(0, next + 1)).trim();
|
|
203
|
+
}
|
|
204
|
+
/** The folder a testkit sits in. */
|
|
205
|
+
function harnessDir(harnessPath) {
|
|
206
|
+
return dirnameOf(harnessPath);
|
|
207
|
+
}
|
|
229
208
|
/** Tiny dirname without importing path just for one call. */
|
|
230
209
|
function dirnameOf(p) {
|
|
231
210
|
const i = p.replace(/\/+$/, '').lastIndexOf('/');
|
|
@@ -233,7 +212,7 @@ function dirnameOf(p) {
|
|
|
233
212
|
}
|
|
234
213
|
/** Build the prompt that generates one area's testkit. */
|
|
235
214
|
function buildHarnessPrompt(harnessPath, framework) {
|
|
236
|
-
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
215
|
+
const map = dataLayer(existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '');
|
|
237
216
|
// A rebuild must not break the suite already written against the old kit.
|
|
238
217
|
let previous = '';
|
|
239
218
|
try {
|
|
@@ -243,51 +222,38 @@ function buildHarnessPrompt(harnessPath, framework) {
|
|
|
243
222
|
// first build
|
|
244
223
|
}
|
|
245
224
|
return [
|
|
246
|
-
`You are a senior SDET writing the
|
|
247
|
-
`
|
|
248
|
-
`
|
|
225
|
+
`You are a senior SDET writing the shared test harness ("testkit") that every`,
|
|
226
|
+
`test in this area imports. You are inside the repository with read-only tools`,
|
|
227
|
+
`(Read, Grep, Glob, LS). Read before you write. Every function is a claim about`,
|
|
228
|
+
`how this product stores data, signs people in and behaves at run time, and a`,
|
|
229
|
+
`testkit built on a guess makes every test that uses it fail as if the product`,
|
|
230
|
+
`were broken. Nothing here assumes a stack: decide everything from what you`,
|
|
231
|
+
`read.`,
|
|
249
232
|
``,
|
|
250
|
-
`
|
|
251
|
-
`
|
|
233
|
+
`This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area. The file will be saved at`,
|
|
234
|
+
`exactly ${harnessPath}; compute require() paths relative to it.`,
|
|
235
|
+
...(framework === 'jest' ? [``, `How this project's server can be reached, from its dependencies:`, transportHint(harnessDir(harnessPath), framework)] : []),
|
|
252
236
|
``,
|
|
253
|
-
`This is a ${framework === 'jest' ? 'Jest/backend' : 'Playwright/UI'} area.`,
|
|
254
|
-
...(framework === 'jest'
|
|
255
|
-
? [
|
|
256
|
-
`Because this is a JEST/backend area, do NOT require or import`,
|
|
257
|
-
`'@playwright/test' anywhere (it is not installed in a backend`,
|
|
258
|
-
`package — a top-level require would break every suite that loads`,
|
|
259
|
-
`this testkit). OMIT the Playwright-only apiContext/login exports.`,
|
|
260
|
-
`Backend suites drive the server in-process via the framework's own`,
|
|
261
|
-
`inject (or an already-installed HTTP client) — never Playwright,`,
|
|
262
|
-
`and never a library this project has not installed.`,
|
|
263
|
-
``,
|
|
264
|
-
]
|
|
265
|
-
: []),
|
|
266
237
|
HARNESS_CONTRACT,
|
|
267
238
|
``,
|
|
268
|
-
`The file will be saved at EXACTLY: ${harnessPath}`,
|
|
269
|
-
`Compute any require() paths to app source RELATIVE to that location.`,
|
|
270
239
|
...(previous
|
|
271
240
|
? [
|
|
272
|
-
|
|
273
|
-
`A TESTKIT ALREADY EXISTS at that path (below) and a suite of tests`,
|
|
274
|
-
`imports it. Rebuild it against the contract above WITHOUT breaking`,
|
|
275
|
-
`them: keep EVERY existing export, with the same name, parameters and`,
|
|
276
|
-
`return shape (GREP the sibling test files for how each one is used);`,
|
|
277
|
-
`change an implementation only where the contract requires it, and add`,
|
|
278
|
-
`the exports it requires. A removed or reshaped export fails every`,
|
|
279
|
-
`test that uses it.`,
|
|
280
|
-
`--- EXISTING TESTKIT ---`,
|
|
241
|
+
`<existing_testkit>`,
|
|
281
242
|
previous,
|
|
282
|
-
|
|
243
|
+
`</existing_testkit>`,
|
|
244
|
+
`Tests already import this testkit. Keep every existing export with the same`,
|
|
245
|
+
`name, parameters and return shape (grep the test files for how each one is`,
|
|
246
|
+
`used). Change an implementation only where the contract above requires it,`,
|
|
247
|
+
`and add any missing exports.`,
|
|
248
|
+
``,
|
|
283
249
|
]
|
|
284
250
|
: []),
|
|
285
|
-
|
|
286
|
-
`Output ONLY the testkit code (CommonJS). No prose, no markdown fences.`,
|
|
287
|
-
`The very first characters must be code. Do NOT write any file.`,
|
|
288
|
-
``,
|
|
289
|
-
`--- PROJECT MAP (data layer) ---`,
|
|
251
|
+
`<project_map_data_layer>`,
|
|
290
252
|
map,
|
|
253
|
+
`</project_map_data_layer>`,
|
|
254
|
+
``,
|
|
255
|
+
`Output only the CommonJS code: no prose and no fences, and the first`,
|
|
256
|
+
`characters must be code. Do not write any file.`,
|
|
291
257
|
].join('\n');
|
|
292
258
|
}
|
|
293
259
|
/** Generate one area's testkit content (sync). */
|
|
@@ -16,6 +16,8 @@ import { journeyDir } from '../paths.js';
|
|
|
16
16
|
const NOT_RULES = /^(controls|coverage|test setup|questions for product|not requirements|pending decisions|suggested)\b/i;
|
|
17
17
|
const TAG = /\[(spec|baseline|observed)\]/;
|
|
18
18
|
const CONTROL_REF = /\bC\d+[a-z]?\b/g;
|
|
19
|
+
/** A criterion that names its own id: "R12 [spec] …". */
|
|
20
|
+
const OWN_ID = /^R(\d+)\s*[:.]?\s+(?=\[(?:spec|baseline|observed)\])/i;
|
|
19
21
|
/** Split a journey doc into header, controls, rules and setup. Pure: no AI, no I/O. */
|
|
20
22
|
export function readRules(markdown) {
|
|
21
23
|
const lines = markdown.split('\n');
|
|
@@ -88,7 +90,7 @@ export function readRules(markdown) {
|
|
|
88
90
|
if (numbered) {
|
|
89
91
|
close();
|
|
90
92
|
const tag = (TAG.exec(numbered[1])?.[1] ?? '');
|
|
91
|
-
current = { rule: { id:
|
|
93
|
+
current = { rule: { id: '', section, group, tag, text: numbered[1], controls: [] }, indent: Number.MAX_SAFE_INTEGER, owns: true };
|
|
92
94
|
rules.push(current.rule);
|
|
93
95
|
continue;
|
|
94
96
|
}
|
|
@@ -106,7 +108,13 @@ export function readRules(markdown) {
|
|
|
106
108
|
}
|
|
107
109
|
close();
|
|
108
110
|
const tag = (TAG.exec(bullet[2])?.[1] ?? '');
|
|
109
|
-
|
|
111
|
+
const own = OWN_ID.exec(bullet[2]);
|
|
112
|
+
current = {
|
|
113
|
+
rule: own
|
|
114
|
+
? { id: `R${Number(own[1])}`, section, group, tag, text: bullet[2].slice(own[0].length), controls: [], explicit: true }
|
|
115
|
+
: { id: '', section, group, tag, text: bullet[2], controls: [] },
|
|
116
|
+
indent,
|
|
117
|
+
};
|
|
110
118
|
rules.push(current.rule);
|
|
111
119
|
continue;
|
|
112
120
|
}
|
|
@@ -118,11 +126,39 @@ export function readRules(markdown) {
|
|
|
118
126
|
close();
|
|
119
127
|
// Free text between rules (a section's lead-in) becomes context for the next rule's section.
|
|
120
128
|
if (line.trim())
|
|
121
|
-
rules.push({ id:
|
|
129
|
+
rules.push({ id: '', section, group, tag: '', text: line.trim(), controls: [...new Set(line.match(CONTROL_REF) ?? [])] });
|
|
122
130
|
}
|
|
123
131
|
close();
|
|
132
|
+
numberRules(rules);
|
|
124
133
|
return { header: header.join('\n').trim(), rules, controls, setup: setup.join('\n').trim() };
|
|
125
134
|
}
|
|
135
|
+
/**
|
|
136
|
+
* Ids for the rules that do not name their own: R1, R2 … in reading order,
|
|
137
|
+
* as before, skipping any id the doc gives a rule itself. An id the doc uses
|
|
138
|
+
* twice keeps its first rule; the second is numbered like an unnamed one.
|
|
139
|
+
*/
|
|
140
|
+
function numberRules(rules) {
|
|
141
|
+
const taken = new Set();
|
|
142
|
+
for (const r of rules) {
|
|
143
|
+
if (!r.explicit)
|
|
144
|
+
continue;
|
|
145
|
+
if (taken.has(r.id)) {
|
|
146
|
+
r.explicit = false;
|
|
147
|
+
r.id = '';
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
taken.add(r.id);
|
|
151
|
+
}
|
|
152
|
+
let n = 0;
|
|
153
|
+
for (const r of rules) {
|
|
154
|
+
if (r.id)
|
|
155
|
+
continue;
|
|
156
|
+
do
|
|
157
|
+
n += 1;
|
|
158
|
+
while (taken.has(`R${n}`));
|
|
159
|
+
r.id = `R${n}`;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
126
162
|
/** One line per rule — what the planner and the gate see instead of the doc. */
|
|
127
163
|
export function ruleIndex(memory, tags = ['spec', 'baseline', 'observed']) {
|
|
128
164
|
return memory.rules
|
|
@@ -34,7 +34,7 @@ import { normalisePending } from './pending.js';
|
|
|
34
34
|
import { packageRoot } from '../run/run-tests.js';
|
|
35
35
|
import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
|
|
36
36
|
import { groundJourney } from '../docs/grounding.js';
|
|
37
|
-
import { generateSeed } from './seed-generator.js';
|
|
37
|
+
import { generateSeed, seedProblem } from './seed-generator.js';
|
|
38
38
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
39
39
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
40
40
|
import { testedTags } from './suite-planner.js';
|
|
@@ -91,7 +91,7 @@ export async function pool(items, limit, fn) {
|
|
|
91
91
|
}
|
|
92
92
|
/** Async version of generateValidated (for concurrent per-file gen). */
|
|
93
93
|
async function generateValidatedAsync(generate, ext, gate,
|
|
94
|
-
/** Why an earlier draft of the same file (from a batch) was refused. */
|
|
94
|
+
/** Why an earlier draft of the same file (from a batch) was refused, with that draft. */
|
|
95
95
|
firstRefusal) {
|
|
96
96
|
let content = '';
|
|
97
97
|
let refusal = firstRefusal;
|
|
@@ -99,7 +99,7 @@ firstRefusal) {
|
|
|
99
99
|
if (aiUnavailableReason())
|
|
100
100
|
return null; // retrying a dead CLI only burns time
|
|
101
101
|
try {
|
|
102
|
-
// A second try
|
|
102
|
+
// A second try sees what it wrote and why that was refused; asked blind, it made the same file again.
|
|
103
103
|
content = await generate(refusal);
|
|
104
104
|
}
|
|
105
105
|
catch (e) {
|
|
@@ -110,8 +110,9 @@ firstRefusal) {
|
|
|
110
110
|
const gateMsg = gate ? gate(content) : null;
|
|
111
111
|
if (isSyntaxValid(content, ext) && !gateMsg)
|
|
112
112
|
return content;
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
const reason = gateMsg || syntaxReason(content, ext);
|
|
114
|
+
refusal = { reason, previous: content };
|
|
115
|
+
noteAiRejected(reason);
|
|
115
116
|
}
|
|
116
117
|
return null;
|
|
117
118
|
}
|
|
@@ -719,7 +720,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
719
720
|
const draft = drafts.get(filePath);
|
|
720
721
|
const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
|
|
721
722
|
const { mem, facts } = await fileMemory(pf);
|
|
722
|
-
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal
|
|
723
|
+
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
|
|
723
724
|
if (ai !== null) {
|
|
724
725
|
content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
|
|
725
726
|
status = 'ai';
|
|
@@ -799,10 +800,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
799
800
|
}
|
|
800
801
|
else if (useAI) {
|
|
801
802
|
const seedFacts = await grounding();
|
|
802
|
-
let seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt);
|
|
803
|
+
let seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt, seedProblem);
|
|
803
804
|
// A usage limit is not a failed seed: wait for the AI like every other step, then try again.
|
|
804
805
|
while (seed === null && aiUnavailableReason() && waitForLimit && (await waitForAi(config))) {
|
|
805
|
-
seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt);
|
|
806
|
+
seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt, seedProblem);
|
|
806
807
|
}
|
|
807
808
|
if (seed !== null) {
|
|
808
809
|
mkdirSync(dirname(seedPath), { recursive: true });
|