@activate-core/test-forge 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/core/docs/excerpts.js +49 -3
- package/dist/core/docs/grounding.js +4 -0
- package/dist/core/gates/validate.js +49 -23
- package/dist/core/generate/complete-file.js +28 -6
- package/dist/core/generate/run-generate.js +86 -9
- package/dist/core/project/env-diff.js +3 -1
- package/dist/core/project/setup.js +10 -2
- package/dist/core/prompts/blocks.js +13 -5
- package/dist/core/run/name-scan.js +58 -3
- package/dist/core/run/report.js +2 -0
- package/dist/core/run/run-tests.js +5 -0
- package/package.json +3 -2
- package/reporters/interface.cjs +26 -3
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 2.7.0
|
|
4
|
+
|
|
5
|
+
- The names new tests still wait on are looked up once, at generate time, against the journey's code facts; a value is kept only if the code has it.
|
|
6
|
+
- The writer never sees a constant or a "names already asked" entry that is still blank, so rewritten tests do not ask for the same blank names again.
|
|
7
|
+
- A kept file whose tests were taken out to be written again stays as it was when the new tests do not come back (a call that fails or is refused).
|
|
8
|
+
- The facts follow the methods a route handler calls: the class it is given (what `create()` returns) and its response helpers, through the helpers they call (the `{ error: { message } }` shape).
|
|
9
|
+
- `fillPath(URL, …values)` from `reporters/interface.cjs` fills a path's parameters whatever the product calls them.
|
|
10
|
+
- A waiting test says why its name is blank: not a kind TestForge looks up, nothing like it in the code yet, looked for and not found, or the lookup could not run.
|
|
11
|
+
- A rewritten test keeps its title (dashboard history continues).
|
|
12
|
+
- A testkit rebuild that fails, or drops an export the tests use, is not taken: the existing testkit stays.
|
|
13
|
+
- Rules asked for and refused are not asked again until the doc or the journey's code changes.
|
|
14
|
+
- "route", "controller" and "endpoint" no longer match every route line.
|
|
15
|
+
- `env --diff` and `report` read an unfilled `KEY= # hint` as unset; `run --journey` accepts only a journey id.
|
|
16
|
+
- `setup` stops when the config it wrote has problems, before any doc is drafted.
|
|
17
|
+
|
|
18
|
+
## 2.6.0
|
|
19
|
+
|
|
20
|
+
- The reporters record passed tests in `testforge/passed.json`; generate never removes or rewrites them (only `--force` does).
|
|
21
|
+
- A kept file's tests are reported by newer checks, never removed; when code arrives only the tests using its blank names are written again; a failed rewrite keeps the earlier file.
|
|
22
|
+
- A journey's facts move only when its own code changes; tests written with no code are written again once it is there, with the seed.
|
|
23
|
+
- Route handlers and their rule classes come first in the facts; the doc's Test setup reaches the facts; the testkit's `insert` contract always reaches the writer.
|
|
24
|
+
- Name scan: rule, logic, request and enum files; whole-word matching; the enclosing route prefix; methods, roles, nested fields; values the code no longer has are blanked; blanked names are asked again.
|
|
25
|
+
- `run`: files of a feature that is not live are held back; prints how many tests ran of how many; exits 0 when nothing failed for a product reason.
|
|
26
|
+
- A saved review verdict applies to the same failure only; `report --results` files waiting tests as waiting.
|
|
27
|
+
- Globals-style Jest files parse; an unfilled env line is empty; `coverage.json` keeps other journeys; other AI CLIs get the writer rules; a ticket key counts as a source; a missing `--env-file` stops the run.
|
|
28
|
+
|
|
29
|
+
### Upgrading
|
|
30
|
+
- New state files under `testforge/`: `passed.json` (commit it with the tests), `last-run.json`, `interface-filled.json`, `interface-status.json`, and `journeys/<area>/<id>/refused.json`.
|
|
31
|
+
- Facts are read once again after upgrading (one grounding call per journey).
|
|
@@ -17,7 +17,7 @@ export function sourceRoot(base = '.') {
|
|
|
17
17
|
const FILE_REF = /((?:[\w.-]+\/)+[\w.-]+\.(?:tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go))(?::(\d+)(?:[-–](\d+))?)?/g;
|
|
18
18
|
const QUOTED = /"([^"\n]{4,60})"/g;
|
|
19
19
|
/** Words in a control's name that name no component in particular. */
|
|
20
|
-
const GENERIC = new Set(['button', 'buttons', 'field', 'input', 'inputs', 'header', 'footer', 'items', 'entry', 'entries', 'shown', 'guest', 'logged', 'customer', 'computer', 'phone', 'sized', 'screen', 'page', 'pages', 'where', 'opens', 'close', 'link', 'links', 'text', 'label', 'click', 'state', 'result', 'neighbour', 'neighbouring', 'other', 'while', 'their', 'there', 'which', 'every', 'itself', 'changes', 'layout']);
|
|
20
|
+
const GENERIC = new Set(['route', 'routes', 'controller', 'endpoint', 'button', 'buttons', 'field', 'input', 'inputs', 'header', 'footer', 'items', 'entry', 'entries', 'shown', 'guest', 'logged', 'customer', 'computer', 'phone', 'sized', 'screen', 'page', 'pages', 'where', 'opens', 'close', 'link', 'links', 'text', 'label', 'click', 'state', 'result', 'neighbour', 'neighbouring', 'other', 'while', 'their', 'there', 'which', 'every', 'itself', 'changes', 'layout']);
|
|
21
21
|
function walk(dir, out) {
|
|
22
22
|
let entries;
|
|
23
23
|
try {
|
|
@@ -302,6 +302,7 @@ export function sourceExcerpts(markdown, roots, budget = 70_000) {
|
|
|
302
302
|
const handlers = new Set();
|
|
303
303
|
const handlerFiles = [];
|
|
304
304
|
const seenRules = new Set();
|
|
305
|
+
const seenMethods = new Set();
|
|
305
306
|
// Only the routes the doc names by their path: theirs are the requests its tests send.
|
|
306
307
|
for (const c of candidates.filter((x) => x.score >= 10).slice(0, 12)) {
|
|
307
308
|
let line = '';
|
|
@@ -335,8 +336,53 @@ export function sourceExcerpts(markdown, roots, budget = 70_000) {
|
|
|
335
336
|
// A handler that hands the request to a class it is given
|
|
336
337
|
// (bookingOrderV2(Request $r, OrderLogicV2 $logic)) keeps its rules there.
|
|
337
338
|
const sig = text[at].match(/\(([^)]*)\)/)?.[1] ?? '';
|
|
338
|
-
|
|
339
|
-
|
|
339
|
+
const deps = [...sig.matchAll(/\b([A-Z]\w+)\s+\$(\w+)/g)].filter((d) => d[1] !== 'Request');
|
|
340
|
+
for (const d of deps)
|
|
341
|
+
rulesOf(d[1], 2);
|
|
342
|
+
// What the response looks like is in the methods the handler calls: the
|
|
343
|
+
// class it is given ($order_logic->create() → what create() returns) and
|
|
344
|
+
// its own response helpers ($this->respondWithSuccess, often in a trait).
|
|
345
|
+
const body = text.slice(at, at + 45).join('\n');
|
|
346
|
+
for (const d of deps) {
|
|
347
|
+
for (const call of new Set([...body.matchAll(new RegExp(`\\$${d[2]}->(\\w+)\\(`, 'g'))].map((x) => x[1]))) {
|
|
348
|
+
if (/^(?:setInput|getErrors|fails|validate)$/.test(call))
|
|
349
|
+
continue;
|
|
350
|
+
methodWindow(d[1], call, 60);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
for (const call of new Set([...body.matchAll(/\$this->(respond\w*|\w*[Rr]esponse\w*|json\w*)\(/g)].map((x) => x[1])))
|
|
354
|
+
methodWindow('', call, 15, 3);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
// The body of `method` in class `cls` (any class when cls is ''), where it is defined.
|
|
358
|
+
function methodWindow(cls, method, lines, helperDepth = 0) {
|
|
359
|
+
if (seenMethods.has(`${cls}::${method}`))
|
|
360
|
+
return;
|
|
361
|
+
seenMethods.add(`${cls}::${method}`);
|
|
362
|
+
const re = new RegExp(`function\\s+${method}\\s*\\(`);
|
|
363
|
+
for (const f of files.filter((x) => x.endsWith('.php') && (!cls || x.endsWith(`/${cls}.php`))).slice(0, 4000)) {
|
|
364
|
+
let text;
|
|
365
|
+
try {
|
|
366
|
+
text = readFileSync(f, 'utf-8').split('\n');
|
|
367
|
+
}
|
|
368
|
+
catch {
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
const i = text.findIndex((l) => re.test(l));
|
|
372
|
+
if (i === -1)
|
|
373
|
+
continue;
|
|
374
|
+
add(f, i + 1, i + lines);
|
|
375
|
+
if (!handlerFiles.includes(f))
|
|
376
|
+
handlerFiles.push(f);
|
|
377
|
+
// A helper that builds its answer through another (respondInvalidParameters → respondWithError).
|
|
378
|
+
if (helperDepth > 1) {
|
|
379
|
+
for (const next of new Set([...text.slice(i, i + lines).join('\n').matchAll(/(?:->|::)(respond\w*|\w*[Rr]esponse\w*|json\w*)\(/g)].map((x) => x[1]))) {
|
|
380
|
+
if (next !== method)
|
|
381
|
+
methodWindow('', next, lines, helperDepth - 1);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
if (!cls)
|
|
385
|
+
return; // a helper is defined once (a trait or base controller)
|
|
340
386
|
}
|
|
341
387
|
}
|
|
342
388
|
// The rule methods of a class (validate, rules, getXRules), and of the
|
|
@@ -124,6 +124,10 @@ function journeyCode(journey, srcDir) {
|
|
|
124
124
|
catch { /* unreadable tree: no code */ }
|
|
125
125
|
return { excerpts, hash: createHash('sha1').update(excerpts).digest('hex').slice(0, 16) };
|
|
126
126
|
}
|
|
127
|
+
/** A short fingerprint of the code this journey reads (no AI). */
|
|
128
|
+
export function journeyCodeHash(journey, srcDir = 'src') {
|
|
129
|
+
return journeyCode(journey, srcDir).hash;
|
|
130
|
+
}
|
|
127
131
|
/** True when the journey's facts were read from code that has since changed. */
|
|
128
132
|
export function factsCodeMoved(journey, srcDir = 'src') {
|
|
129
133
|
const path = groundingPath(journey);
|
|
@@ -51,10 +51,11 @@ export const DEFAULT_EXTERNAL_MODULES = [
|
|
|
51
51
|
* clean. Runs AFTER syntax validation.
|
|
52
52
|
*/
|
|
53
53
|
export function checkTestGates(content, _ext, ctx) {
|
|
54
|
+
const reasons = [];
|
|
54
55
|
// F3 — runner correctness: a Jest/backend area must never pull in
|
|
55
56
|
// Playwright (not installed there; a top-level import breaks the suite).
|
|
56
57
|
if (ctx.framework === 'jest' && /@playwright\/test/.test(content)) {
|
|
57
|
-
|
|
58
|
+
reasons.push("jest area must not import '@playwright/test'");
|
|
58
59
|
}
|
|
59
60
|
// F4 — goto guard: `page.goto(...)` must take a real route/URL, not a
|
|
60
61
|
// pasted description and not an unfilled route-file placeholder. Literal
|
|
@@ -67,10 +68,10 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
67
68
|
while ((m = gotoRe.exec(content)) !== null) {
|
|
68
69
|
const arg = m[2];
|
|
69
70
|
if (/\s/.test(arg)) {
|
|
70
|
-
|
|
71
|
+
reasons.push(`goto() argument looks like prose, not a URL/path: "${arg.slice(0, 40)}…"`);
|
|
71
72
|
}
|
|
72
73
|
if (/\[[A-Za-z_]|\/:[A-Za-z_]|\{[A-Za-z_]|<[A-Za-z_]/.test(arg)) {
|
|
73
|
-
|
|
74
|
+
reasons.push(`goto() has an unresolved route placeholder — fill it with a real value: "${arg.slice(0, 40)}"`);
|
|
74
75
|
}
|
|
75
76
|
}
|
|
76
77
|
// F13 — a swallowed action must be bounded. `.click({ force: true })`
|
|
@@ -85,7 +86,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
85
86
|
let sw;
|
|
86
87
|
while ((sw = swallowed.exec(content)) !== null) {
|
|
87
88
|
if (!/timeout\s*:/.test(sw[0])) {
|
|
88
|
-
|
|
89
|
+
reasons.push('an action whose failure is swallowed with .catch() has no timeout — ' +
|
|
89
90
|
'it will wait until the whole test times out and blame a later line');
|
|
90
91
|
}
|
|
91
92
|
}
|
|
@@ -96,7 +97,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
96
97
|
// that never comes. Hold the promise first, then act, then await it.
|
|
97
98
|
const lateWait = /\b(?:await\s+)?page\.goto\([^\n]*\);?\s*\n(?:\s*\/\/[^\n]*\n)*\s*await\s+page\.waitFor(?:Response|Request|Event)\s*\(/;
|
|
98
99
|
if (lateWait.test(content)) {
|
|
99
|
-
|
|
100
|
+
reasons.push('page.waitForResponse/Request is armed AFTER the navigation that triggers ' +
|
|
100
101
|
'it — hold the promise first, then navigate, then await it');
|
|
101
102
|
}
|
|
102
103
|
// F2 — no silent false-green: a file with RUNNABLE tests must assert
|
|
@@ -108,7 +109,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
108
109
|
const assertions = (content.match(/\bexpect\s*\(/g) || []).length +
|
|
109
110
|
(content.match(/\bassert\b/g) || []).length;
|
|
110
111
|
if (activeTests > 0 && assertions === 0) {
|
|
111
|
-
|
|
112
|
+
reasons.push('has runnable test(s) but no assertions — would pass without testing anything');
|
|
112
113
|
}
|
|
113
114
|
// F1 — dependency awareness: reject imports/requires of a module the
|
|
114
115
|
// project can't resolve (bare specifier not in allowedModules).
|
|
@@ -116,7 +117,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
116
117
|
for (const mod of importedModules(content)) {
|
|
117
118
|
const top = topLevelModule(mod);
|
|
118
119
|
if (!ctx.allowedModules.has(top)) {
|
|
119
|
-
|
|
120
|
+
reasons.push(`imports '${mod}' which the project does not have installed`);
|
|
120
121
|
}
|
|
121
122
|
}
|
|
122
123
|
}
|
|
@@ -139,7 +140,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
139
140
|
content.includes(`jest.mock('${top}'`) ||
|
|
140
141
|
content.includes(`jest.mock("${top}"`);
|
|
141
142
|
if (!mocked) {
|
|
142
|
-
|
|
143
|
+
reasons.push(`imports external service '${top}' without a jest.mock — the test would hit the live service; mock it at the seam`);
|
|
143
144
|
}
|
|
144
145
|
}
|
|
145
146
|
}
|
|
@@ -160,7 +161,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
160
161
|
const showedWork = /\b(?:port|origin|path\s*prefix|prefix|resource\s*type|header|predicate)\b/i;
|
|
161
162
|
const m = content.match(sameHost);
|
|
162
163
|
if (m && !showedWork.test(m[0])) {
|
|
163
|
-
|
|
164
|
+
reasons.push('a test is skipped because the dependency "shares a host" with the app, ' +
|
|
164
165
|
'without saying what else was checked — a different port is a different ' +
|
|
165
166
|
'origin, and a path prefix or a request predicate usually separates them ' +
|
|
166
167
|
'too; scope the stub to one of those, or name the ones you ruled out');
|
|
@@ -169,20 +170,20 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
169
170
|
// F16 — no placeholders in a freshly written file: a placeholder never runs,
|
|
170
171
|
// and a suite full of them reads as coverage it does not have.
|
|
171
172
|
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
172
|
-
|
|
173
|
+
reasons.push('placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file');
|
|
173
174
|
}
|
|
174
175
|
// F23 — need() asks for the name of a thing, anywhere in the file (a constant at
|
|
175
176
|
// the top too), never for data the test should make itself.
|
|
176
177
|
if (ctx.forbidPlaceholders) {
|
|
177
178
|
const data = [...content.matchAll(/\bneed\(\s*(['"`])([^'"`\n]+)\1/g)].map((m) => m[2]).filter((n) => !isThingName(n));
|
|
178
179
|
if (data.length) {
|
|
179
|
-
|
|
180
|
+
reasons.push(`asks need() for ${[...new Set(data)].slice(0, 3).map((n) => `"${n}"`).join(', ')}, which is data, not the name of a thing: the test makes its own ids, payloads and records; need() is only for "… URL", "… command", "… table", "… field", "… label"`);
|
|
180
181
|
}
|
|
181
182
|
}
|
|
182
183
|
// F22 — a fresh file exists to test its rules: one with no test at all says
|
|
183
184
|
// nothing, and passes. It goes back once, told to stub what it cannot create.
|
|
184
185
|
if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
|
|
185
|
-
|
|
186
|
+
reasons.push('the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out');
|
|
186
187
|
}
|
|
187
188
|
// F24 — one comment policy: the header, an oracle line above each test,
|
|
188
189
|
// cannot-test lines, a timeout reason, and a short note on a parametrised
|
|
@@ -190,7 +191,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
190
191
|
if (ctx.forbidPlaceholders) {
|
|
191
192
|
const stray = strayComment(content);
|
|
192
193
|
if (stray) {
|
|
193
|
-
|
|
194
|
+
reasons.push(`comment not allowed (line ${stray.line}): "${stray.text.slice(0, 60)}" — the only comments are the // @generated header, one // oracle: line above each test, // cannot-test: lines, a // timeout: reason, and a one-line note on a parametrised row`);
|
|
194
195
|
}
|
|
195
196
|
}
|
|
196
197
|
// F21 — a run-time skip is for what the ENVIRONMENT lacks (an account, a
|
|
@@ -200,27 +201,27 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
200
201
|
if (ctx.forbidPlaceholders) {
|
|
201
202
|
const why = elementSkips(content);
|
|
202
203
|
if (why.length) {
|
|
203
|
-
|
|
204
|
+
reasons.push(`a test skips itself because an element is not on the page (${why[0]}): a missing element the doc names is a failure — assert it instead. Skip only for an account, a variable or data the environment lacks`);
|
|
204
205
|
}
|
|
205
206
|
}
|
|
206
207
|
// F17 — relative imports resolve from where the file is written.
|
|
207
208
|
if (ctx.filePath) {
|
|
208
209
|
const missing = unresolvedImports(content, ctx.filePath);
|
|
209
210
|
if (missing.length) {
|
|
210
|
-
|
|
211
|
+
reasons.push(`relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`);
|
|
211
212
|
}
|
|
212
213
|
}
|
|
213
214
|
// F18 — no fixed sleeps. A wait for a set time either wastes it or is too
|
|
214
215
|
// short on a slower machine; wait for the event or the state instead.
|
|
215
216
|
if (ctx.framework === 'playwright' && ctx.forbidPlaceholders && /\bwaitForTimeout\s*\(/.test(content)) {
|
|
216
|
-
|
|
217
|
+
reasons.push('page.waitForTimeout() is not used: wait for the element, response or URL the test needs (expect(...).toBeVisible(), page.waitForResponse(), expect(page).toHaveURL())');
|
|
217
218
|
}
|
|
218
219
|
// F19 — a long test timeout needs a reason. A 240 s budget hides a test that
|
|
219
220
|
// is stuck; the cap makes it fail early with its real error.
|
|
220
221
|
if (ctx.maxTestTimeoutMs) {
|
|
221
222
|
const long = longTimeoutsWithoutReason(content, ctx.maxTestTimeoutMs);
|
|
222
223
|
if (long.length) {
|
|
223
|
-
|
|
224
|
+
reasons.push(`test.setTimeout(${long[0]}) is above ${ctx.maxTestTimeoutMs} ms with no "// timeout: <why>" comment: wait for what the test needs instead, or state why it takes that long`);
|
|
224
225
|
}
|
|
225
226
|
}
|
|
226
227
|
// F20 — one criterion, a few tests. Thirty inputs against a rule that only
|
|
@@ -229,7 +230,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
229
230
|
if (ctx.maxTestsPerRule && ctx.forbidPlaceholders) {
|
|
230
231
|
const over = testsPerCriterion(content).find((c) => c.tests > ctx.maxTestsPerRule);
|
|
231
232
|
if (over) {
|
|
232
|
-
|
|
233
|
+
reasons.push(`${over.tests} tests check the same criterion (${over.criterion.slice(0, 80)}…): keep at most ${ctx.maxTestsPerRule}, one per branch, and put further inputs in one parametrised test`);
|
|
233
234
|
}
|
|
234
235
|
}
|
|
235
236
|
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
@@ -238,7 +239,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
238
239
|
if (ctx.framework === 'playwright') {
|
|
239
240
|
const bare = reasonlessPending(content);
|
|
240
241
|
if (bare.length) {
|
|
241
|
-
|
|
242
|
+
reasons.push(`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
|
|
242
243
|
`test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
|
|
243
244
|
}
|
|
244
245
|
}
|
|
@@ -257,14 +258,14 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
257
258
|
const sel = String.raw `(?:getByTestId\s*\(|\[\s*data-(?:testid|cy|test)\s*=)`;
|
|
258
259
|
const selNearMarker = new RegExp(`(?:${sel}[^\\n]*?\\b${marker}\\b)|(?:\\b${marker}\\b[^\\n]*?${sel})`, 'i');
|
|
259
260
|
if (selNearMarker.test(active)) {
|
|
260
|
-
|
|
261
|
+
reasons.push('a runnable test uses a guessed/placeholder selector — ground it in a real component attribute, or defer the test (test.fixme/skip)');
|
|
261
262
|
}
|
|
262
263
|
// (b) When the project's real test-hooks are known, any test-id a
|
|
263
264
|
// runnable test uses that isn't one of them is an invented selector.
|
|
264
265
|
if (ctx.knownSelectors && ctx.knownSelectors.size > 0) {
|
|
265
266
|
for (const id of usedTestIds(active)) {
|
|
266
267
|
if (!ctx.knownSelectors.has(id)) {
|
|
267
|
-
|
|
268
|
+
reasons.push(`references selector '${id}' not found in the project's components — ground it or defer the test`);
|
|
268
269
|
}
|
|
269
270
|
}
|
|
270
271
|
}
|
|
@@ -272,11 +273,13 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
272
273
|
if (ctx.knownTexts) {
|
|
273
274
|
const missing = usedLabels(active).filter((l) => !ctx.knownTexts.includes(l.toLowerCase()));
|
|
274
275
|
if (missing.length) {
|
|
275
|
-
|
|
276
|
+
reasons.push(`uses ${missing.map((m) => `"${m}"`).slice(0, 3).join(', ')} as an accessible name, label or placeholder, but that text appears nowhere in the app's source or the journey doc — take every name from the code facts or the rules`);
|
|
276
277
|
}
|
|
277
278
|
}
|
|
278
279
|
}
|
|
279
|
-
|
|
280
|
+
// Every reason at once: a retry told one problem fixes it and trips on the next.
|
|
281
|
+
const unique = [...new Set(reasons)];
|
|
282
|
+
return unique.length ? (unique.length === 1 ? unique[0] : `${unique[0]}\nAlso:\n${unique.slice(1, 5).map((r) => `- ${r}`).join('\n')}`) : null;
|
|
280
283
|
}
|
|
281
284
|
/**
|
|
282
285
|
* The run-time skips whose condition looks at the page: test.skip(true, …),
|
|
@@ -676,6 +679,29 @@ function testCriteria(content) {
|
|
|
676
679
|
return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
|
|
677
680
|
});
|
|
678
681
|
}
|
|
682
|
+
/**
|
|
683
|
+
* The file with at most `max` tests per criterion: the first ones are kept,
|
|
684
|
+
* the extras removed. A mechanical fix — no AI retry for "too many tests".
|
|
685
|
+
*/
|
|
686
|
+
export function trimExtraTests(content, max) {
|
|
687
|
+
const blocks = findTestBlocks(content);
|
|
688
|
+
const crit = testCriteria(content);
|
|
689
|
+
const seen = new Map();
|
|
690
|
+
const extra = blocks.filter((_, i) => {
|
|
691
|
+
const c = crit[i]?.criterion;
|
|
692
|
+
if (!c)
|
|
693
|
+
return false;
|
|
694
|
+
const n = (seen.get(c) ?? 0) + 1;
|
|
695
|
+
seen.set(c, n);
|
|
696
|
+
return n > max;
|
|
697
|
+
});
|
|
698
|
+
if (!extra.length)
|
|
699
|
+
return content;
|
|
700
|
+
let out = content;
|
|
701
|
+
for (const b of [...extra].sort((x, y) => y.start - x.start))
|
|
702
|
+
out = out.slice(0, b.start) + out.slice(b.end).replace(/^;?[ \t]*\n?/, '');
|
|
703
|
+
return out;
|
|
704
|
+
}
|
|
679
705
|
/** How many tests of a file check each criterion (tests with no oracle are not counted). */
|
|
680
706
|
export function testsPerCriterion(content) {
|
|
681
707
|
const counts = new Map();
|
|
@@ -27,6 +27,12 @@ export function shortReason(why) {
|
|
|
27
27
|
export function withoutNames(content) {
|
|
28
28
|
return content.replace(/^[ \t]*(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])[^'"`\n]*\2\s*\);?[ \t]*\n/gm, (line, name) => new RegExp(`\\b${name}\\b`).test(content.replace(line, '')) ? line : '');
|
|
29
29
|
}
|
|
30
|
+
/** A file top without the `const X = need('…')` lines whose name is still blank. */
|
|
31
|
+
export function withoutBlankNames(head, isBlank) {
|
|
32
|
+
if (!isBlank)
|
|
33
|
+
return head;
|
|
34
|
+
return head.replace(/^[ \t]*(?:const|let)\s+[A-Za-z_$][\w$]*\s*=\s*need\(\s*(['"`])([^'"`\n]*)\1\s*\);?[ \t]*\n/gm, (line, _q, name) => (isBlank(name) ? '' : line));
|
|
35
|
+
}
|
|
30
36
|
/** The file without the tests that have a problem. */
|
|
31
37
|
function withoutBlocks(content, problems) {
|
|
32
38
|
const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
|
|
@@ -71,8 +77,10 @@ export async function completeFile(content, ctx) {
|
|
|
71
77
|
: [];
|
|
72
78
|
const redo = redoBlocks
|
|
73
79
|
.filter((b) => !first.problems.some((p) => p.block.start === b.start))
|
|
74
|
-
.map((b) => ({ title: b.title, rules: b.rules, reason:
|
|
80
|
+
.map((b) => ({ title: b.title, rules: b.rules, reason: `it waits on a name the code now has: write it again from the code with the same title, "${b.title}"`, block: b }));
|
|
75
81
|
const firstDrop = [...first.problems.filter(removable), ...redo];
|
|
82
|
+
let failedRedo = false;
|
|
83
|
+
const refusedRules = [];
|
|
76
84
|
const reported = first.problems.filter((p) => !removable(p));
|
|
77
85
|
if (reported.length) {
|
|
78
86
|
notes.push(`left ${reported.length} earlier test(s) in place that a check flags (${[...new Set(reported.map((p) => shortReason(p.reason)))].slice(0, 3).join('; ')}) — not removed, they were already in the file`);
|
|
@@ -82,24 +90,31 @@ export async function completeFile(content, ctx) {
|
|
|
82
90
|
drop(firstDrop);
|
|
83
91
|
// Only rules the doc has are written again: an id with no text makes the writer invent one.
|
|
84
92
|
const known = new Set(ctx.memory.rules.filter((r) => r.tag).map((r) => r.id));
|
|
85
|
-
const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])]
|
|
93
|
+
const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])]
|
|
94
|
+
.filter((id) => known.has(id) && ctx.rules.includes(id) && !(ctx.skip?.has(id) && !redo.some((p) => p.rules.includes(id))));
|
|
86
95
|
if (left.length) {
|
|
87
96
|
const blocks = findTestBlocks(out);
|
|
88
|
-
|
|
97
|
+
// The file's top as the writer sees it: a constant that still waits on a
|
|
98
|
+
// blank name is left out, or the new tests reuse it and wait again.
|
|
99
|
+
const head = withoutBlankNames(out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000), ctx.isBlank);
|
|
89
100
|
const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(out)?.[2] ?? ctx.journey.name;
|
|
90
101
|
const slice = journeySlice(ctx.memory, left);
|
|
91
102
|
calls = 1;
|
|
92
103
|
try {
|
|
93
104
|
const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(firstDrop, first.cov));
|
|
94
|
-
const candidate = appendTests(out, onlyNewTests(tests, out, left));
|
|
105
|
+
const candidate = (ctx.fix ?? ((c) => c))(appendTests(out, onlyNewTests(tests, out, left)));
|
|
95
106
|
const refused = ctx.check(candidate);
|
|
96
|
-
if (refused)
|
|
107
|
+
if (refused) {
|
|
97
108
|
notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
|
|
109
|
+
failedRedo = true;
|
|
110
|
+
refusedRules.push(...left);
|
|
111
|
+
}
|
|
98
112
|
else
|
|
99
113
|
out = candidate;
|
|
100
114
|
}
|
|
101
115
|
catch (e) {
|
|
102
116
|
notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
|
|
117
|
+
failedRedo = true;
|
|
103
118
|
}
|
|
104
119
|
// What the second answer still gets wrong is removed, not sent again.
|
|
105
120
|
const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts }).filter(removable);
|
|
@@ -109,6 +124,13 @@ export async function completeFile(content, ctx) {
|
|
|
109
124
|
}
|
|
110
125
|
}
|
|
111
126
|
}
|
|
127
|
+
// A kept file whose tests were taken out to be written again, and the new
|
|
128
|
+
// ones did not come: it stays as it was. A waiting test beats no test.
|
|
129
|
+
if (ctx.kept && failedRedo && redo.length + firstDrop.length > 0) {
|
|
130
|
+
notes.push('kept the file as it was: the tests to write again did not come back');
|
|
131
|
+
out = content;
|
|
132
|
+
dropped.clear();
|
|
133
|
+
}
|
|
112
134
|
// The last word, checked mechanically: every test that passed is still here.
|
|
113
135
|
const lost = [...locked].filter((t) => before.has(t) && !findTestBlocks(out).some((b) => b.title === t));
|
|
114
136
|
if (lost.length) {
|
|
@@ -133,5 +155,5 @@ export async function completeFile(content, ctx) {
|
|
|
133
155
|
}
|
|
134
156
|
if (cov.missing.length)
|
|
135
157
|
notes.push(`no test for ${cov.missing.join(', ')}`);
|
|
136
|
-
return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes };
|
|
158
|
+
return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes, refusedRules: [...new Set([...refusedRules, ...(calls ? ruleCoverage(out, ctx.rules).missing.filter((id) => !refusedRules.includes(id) && !failedRedo) : [])])] };
|
|
137
159
|
}
|
|
@@ -30,6 +30,7 @@ import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, wri
|
|
|
30
30
|
import { completeFile } from './complete-file.js';
|
|
31
31
|
import { tuckStrayTests } from './test-blocks.js';
|
|
32
32
|
import { passedTitles } from './passed.js';
|
|
33
|
+
import { scanNames } from '../run/name-scan.js';
|
|
33
34
|
import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
|
|
34
35
|
/** Names the file asks for that testforge/interface.json still has blank. */
|
|
35
36
|
/** Does this test use a name the product has not given yet (directly, or through the constant it was put in)? */
|
|
@@ -43,6 +44,20 @@ function usesBlankName(content, block) {
|
|
|
43
44
|
return !!v && new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(text);
|
|
44
45
|
});
|
|
45
46
|
}
|
|
47
|
+
/** The names a testkit exports (module.exports = { … } or export …). */
|
|
48
|
+
export function exportNames(src) {
|
|
49
|
+
const names = new Set();
|
|
50
|
+
const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(src);
|
|
51
|
+
if (cjs)
|
|
52
|
+
for (const n of cjs[1].split(',')) {
|
|
53
|
+
const m = /^\s*(\w+)/.exec(n);
|
|
54
|
+
if (m)
|
|
55
|
+
names.add(m[1]);
|
|
56
|
+
}
|
|
57
|
+
for (const m of src.matchAll(/^export\s+(?:async\s+)?(?:function\*?|const|let|class)\s+(\w+)/gm))
|
|
58
|
+
names.add(m[1]);
|
|
59
|
+
return [...names];
|
|
60
|
+
}
|
|
46
61
|
function blankNames(content) {
|
|
47
62
|
const iface = readInterface();
|
|
48
63
|
return [...content.matchAll(/\bneed\(\s*(['"`])([^'"`]+)\1\s*\)/g)].map((m) => m[2]).filter((n) => {
|
|
@@ -54,7 +69,7 @@ import { harnessSteps, ruleCoverage } from '../gates/test-quality.js';
|
|
|
54
69
|
import { normalisePending } from './pending.js';
|
|
55
70
|
import { packageRoot } from '../run/run-tests.js';
|
|
56
71
|
import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
|
|
57
|
-
import { factsCodeMoved, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
|
|
72
|
+
import { factsCodeMoved, journeyCodeHash, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
|
|
58
73
|
import { generateSeed, seedProblem } from './seed-generator.js';
|
|
59
74
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
60
75
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
@@ -64,7 +79,7 @@ import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
|
64
79
|
import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
|
|
65
80
|
import { updateJourney } from './update-journey.js';
|
|
66
81
|
import { sourceRoot, sourceText } from '../docs/excerpts.js';
|
|
67
|
-
import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
82
|
+
import { isSyntaxValid, checkTestGates, trimExtraTests, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
68
83
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
69
84
|
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
70
85
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
@@ -113,7 +128,9 @@ export async function pool(items, limit, fn) {
|
|
|
113
128
|
/** Async version of generateValidated (for concurrent per-file gen). */
|
|
114
129
|
async function generateValidatedAsync(generate, ext, gate,
|
|
115
130
|
/** Why an earlier draft of the same file (from a batch) was refused, with that draft. */
|
|
116
|
-
firstRefusal
|
|
131
|
+
firstRefusal,
|
|
132
|
+
/** Mechanical fixes applied before the gate: what code can fix costs no retry. */
|
|
133
|
+
fix = (c) => c) {
|
|
117
134
|
let content = '';
|
|
118
135
|
let refusal = firstRefusal;
|
|
119
136
|
for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
|
|
@@ -121,7 +138,7 @@ firstRefusal) {
|
|
|
121
138
|
return null; // retrying a dead CLI only burns time
|
|
122
139
|
try {
|
|
123
140
|
// A second try sees what it wrote and why that was refused; asked blind, it made the same file again.
|
|
124
|
-
content = await generate(refusal);
|
|
141
|
+
content = fix(await generate(refusal));
|
|
125
142
|
}
|
|
126
143
|
catch (e) {
|
|
127
144
|
noteAiFailure(e);
|
|
@@ -294,7 +311,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
294
311
|
// command. Test files are untouched.
|
|
295
312
|
console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
|
|
296
313
|
}
|
|
314
|
+
const old = existsSync(info.path) ? readFileSync(info.path, 'utf-8') : '';
|
|
297
315
|
const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
|
|
316
|
+
// The tests already written call the old testkit: a rebuild that fails, or
|
|
317
|
+
// drops something they use, is not taken — the old one stays.
|
|
318
|
+
const lost = old && kit !== null ? exportNames(old).filter((n) => !exportNames(kit).includes(n)) : [];
|
|
319
|
+
if (old && (kit === null || lost.length)) {
|
|
320
|
+
console.log(` kept the existing ${area} test harness: the rebuild ${kit === null ? 'failed' : `dropped ${lost.join(', ')}`}`);
|
|
321
|
+
exportFind(info.path);
|
|
322
|
+
harnessByArea.set(area, info.path);
|
|
323
|
+
return info.path;
|
|
324
|
+
}
|
|
298
325
|
if (kit === null) {
|
|
299
326
|
harnessByArea.set(area, null);
|
|
300
327
|
return undefined;
|
|
@@ -453,6 +480,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
453
480
|
const appTexts = () => framework === 'playwright'
|
|
454
481
|
? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
455
482
|
: undefined;
|
|
483
|
+
const fixTests = (content) => trimExtraTests(content, config.maxTestsPerRule ?? 2);
|
|
456
484
|
const genGate = (content, filePath) => testGate(content) ??
|
|
457
485
|
checkTestGates(content, ext, {
|
|
458
486
|
framework, forbidPlaceholders: true, filePath,
|
|
@@ -463,7 +491,21 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
463
491
|
// to retry (a resume pass fills those in without touching real files).
|
|
464
492
|
// A suite whose tests leave rules out or cannot fail is not done either:
|
|
465
493
|
// it goes on, and only those tests are written again.
|
|
494
|
+
// Rules asked for and refused before, with this doc and this code: not asked again until one moves.
|
|
495
|
+
const refusedPath = join(journeyDir(journey), 'refused.json');
|
|
496
|
+
let refusedKeyMemo;
|
|
497
|
+
const refusedKey = () => (refusedKeyMemo ??= `${hash}:${journeyCodeHash(journey, sourceRoot())}`);
|
|
498
|
+
const readRefused = () => {
|
|
499
|
+
try {
|
|
500
|
+
const r = JSON.parse(readFileSync(refusedPath, 'utf-8'));
|
|
501
|
+
return r.key === refusedKey() ? (r.rules ?? {}) : {};
|
|
502
|
+
}
|
|
503
|
+
catch {
|
|
504
|
+
return {};
|
|
505
|
+
}
|
|
506
|
+
};
|
|
466
507
|
const suiteComplete = () => {
|
|
508
|
+
const refused = readRefused();
|
|
467
509
|
const plan = readPlan(journey);
|
|
468
510
|
if (!plan?.some((f) => f.rules?.length))
|
|
469
511
|
return true;
|
|
@@ -475,7 +517,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
475
517
|
return true;
|
|
476
518
|
// Rules left without a test make a suite unfinished. A check that flags an
|
|
477
519
|
// existing test does not: kept tests are reported, never rewritten.
|
|
478
|
-
return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.length;
|
|
520
|
+
return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.filter((id) => !(refused[f.slug] ?? []).includes(id)).length;
|
|
479
521
|
});
|
|
480
522
|
};
|
|
481
523
|
// Tests an earlier version appended after the outer describe go back inside
|
|
@@ -574,7 +616,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
574
616
|
// What each file really tests once written: the memory and the report come from this, not the plan.
|
|
575
617
|
const written = [];
|
|
576
618
|
// need() names asked so far (the project's and this run's): handed to every later file.
|
|
577
|
-
|
|
619
|
+
// Only names the product has given are handed on for reuse: a blank one in
|
|
620
|
+
// that list reads as "use this name", and the new test waits on it again.
|
|
621
|
+
const filledNow = readInterface();
|
|
622
|
+
const askedNames = new Set(Object.keys(filledNow).filter((n) => isThingName(n) && String(filledNow[n] ?? '').trim()));
|
|
578
623
|
const finish = async (content, pf, filePath, kept = false) => {
|
|
579
624
|
if (!memory || !pf.rules?.length)
|
|
580
625
|
return content;
|
|
@@ -582,13 +627,27 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
582
627
|
const done = await completeFile(content, {
|
|
583
628
|
kept, locked: passedTitles(filePath),
|
|
584
629
|
redo: kept && stale.has(filePath) ? (redoAll ? () => true : (b) => usesBlankName(content, b)) : undefined,
|
|
630
|
+
isBlank: (n) => blankNames(`need('${n}')`).length > 0,
|
|
631
|
+
skip: new Set(readRefused()[pf.slug] ?? []),
|
|
632
|
+
fix: fixTests,
|
|
585
633
|
journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
|
|
586
634
|
harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
|
|
587
635
|
check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
|
|
588
636
|
});
|
|
589
637
|
written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
|
|
638
|
+
// What was asked and still has no test is not asked again for this doc and code.
|
|
639
|
+
if (done.refusedRules.length) {
|
|
640
|
+
try {
|
|
641
|
+
const all = readRefused();
|
|
642
|
+
all[pf.slug] = [...new Set([...(all[pf.slug] ?? []), ...done.refusedRules])];
|
|
643
|
+
mkdirSync(dirname(refusedPath), { recursive: true });
|
|
644
|
+
writeFileSync(refusedPath, JSON.stringify({ key: refusedKey(), rules: all }, null, 2), 'utf-8');
|
|
645
|
+
}
|
|
646
|
+
catch { /* best effort */ }
|
|
647
|
+
}
|
|
590
648
|
for (const n of neededNames(done.content))
|
|
591
|
-
|
|
649
|
+
if (!blankNames(`need('${n}')`).length)
|
|
650
|
+
askedNames.add(n);
|
|
592
651
|
for (const n of done.notes)
|
|
593
652
|
warnings.push(`${pf.slug}: ${n}`);
|
|
594
653
|
return done.content;
|
|
@@ -774,7 +833,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
774
833
|
const got = await writeFilesWithAI(files);
|
|
775
834
|
noteAiSuccess();
|
|
776
835
|
for (const [p, c] of got)
|
|
777
|
-
drafts.set(p, c);
|
|
836
|
+
drafts.set(p, fixTests(c));
|
|
778
837
|
}
|
|
779
838
|
catch (e) {
|
|
780
839
|
noteAiFailure(e);
|
|
@@ -835,7 +894,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
835
894
|
}
|
|
836
895
|
}
|
|
837
896
|
const { mem, facts } = await fileMemory(pf);
|
|
838
|
-
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
|
|
897
|
+
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined, fixTests);
|
|
839
898
|
if (ai !== null) {
|
|
840
899
|
content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
|
|
841
900
|
status = 'ai';
|
|
@@ -876,6 +935,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
876
935
|
}
|
|
877
936
|
return { slug: pf.slug, path: filePath, status };
|
|
878
937
|
});
|
|
938
|
+
// The names the new tests still wait on, looked up once against this
|
|
939
|
+
// journey's code facts: a field the facts show is filled now, not left
|
|
940
|
+
// for the run. Kept only if the code itself has the value.
|
|
941
|
+
if (useAI && !notBuilt && memory) {
|
|
942
|
+
const facts = await grounding();
|
|
943
|
+
const files = planned.map(pathOf).filter((p) => existsSync(p));
|
|
944
|
+
const blank = [...new Set(files.flatMap((p) => { try {
|
|
945
|
+
return blankNames(readFileSync(p, 'utf-8'));
|
|
946
|
+
}
|
|
947
|
+
catch {
|
|
948
|
+
return [];
|
|
949
|
+
} }))];
|
|
950
|
+
if (facts && blank.length) {
|
|
951
|
+
const res = scanNames(blank, sourceRoot(), facts);
|
|
952
|
+
if (res.filled.length)
|
|
953
|
+
warnings.push(`filled ${res.filled.length} name(s) from the code facts: ${res.filled.slice(0, 6).map((f) => `${f.name} = ${f.value}`).join(', ')}${res.filled.length > 6 ? ', …' : ''}`);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
879
956
|
if (memory && written.length) {
|
|
880
957
|
const sum = recordWritten(journey, written);
|
|
881
958
|
console.log(` ${journey.id}: ${sum}`);
|
|
@@ -17,7 +17,9 @@ export function envNames(text) {
|
|
|
17
17
|
const out = new Set();
|
|
18
18
|
for (const line of text.split('\n')) {
|
|
19
19
|
const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
|
|
20
|
-
|
|
20
|
+
// An unquoted value ends at a comment: `KEY= # hint` (an unfilled template line) is unset.
|
|
21
|
+
const v = /^\s*(['"]).*\1\s*$/.test(m?.[2] ?? '') ? (m?.[2] ?? '').trim() : (m?.[2] ?? '').replace(/(^|\s)#.*$/, '').trim();
|
|
22
|
+
if (m && v.replace(/^(['"])\1$/, '') !== '')
|
|
21
23
|
out.add(m[1]);
|
|
22
24
|
}
|
|
23
25
|
return out;
|
|
@@ -36,10 +36,18 @@ export async function runSetup(srcDir, opts = {}) {
|
|
|
36
36
|
runSetupCall(loadConfig().relatedDirs ?? []);
|
|
37
37
|
if (existsSync('testforge.config.json')) {
|
|
38
38
|
const problems = checkSetupConfig();
|
|
39
|
-
|
|
39
|
+
if (problems.length) {
|
|
40
|
+
// Drafting docs on a config with problems spends AI calls on a project
|
|
41
|
+
// that cannot run yet: stop, and say what to fix.
|
|
42
|
+
console.log(` testforge.config.json written, with problems to fix first:\n ${problems.join('\n ')}\n`);
|
|
43
|
+
console.log('Fix these, then run `testforge setup` again (or `testforge discover`).');
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
console.log(' done — testforge.config.json');
|
|
40
47
|
}
|
|
41
48
|
else {
|
|
42
|
-
console.log('
|
|
49
|
+
console.log(' testforge.config.json was not created — nothing else was done. Run `testforge setup` again.');
|
|
50
|
+
return;
|
|
43
51
|
}
|
|
44
52
|
console.log(existsSync('testforge.md') ? ' done — testforge.md\n' : ' warning: testforge.md was not created\n');
|
|
45
53
|
// Step 3 — discover. This is where the source-of-truth barrier lives:
|
|
@@ -183,6 +183,9 @@ E2. What you touch comes from <code_facts> and <testkit>.
|
|
|
183
183
|
- Look in ROUTES, SHAPES and DATA before need(): a name any of them shows is
|
|
184
184
|
written as it is there. need() is only for a name none of them has (see
|
|
185
185
|
NAMES). Never guess one.
|
|
186
|
+
- A path from need() may hold parameters ({booking}, :id): fill them with
|
|
187
|
+
fillPath(URL, value, …) from the same module as need(), never with
|
|
188
|
+
.replace() on a guessed placeholder name.
|
|
186
189
|
- Never need() what the testkit already gives: kit.insert returns the new
|
|
187
190
|
row's id; a second person of a role (another operator, another customer)
|
|
188
191
|
is a row made with kit.insert; the roles are the ones <testkit> lists.
|
|
@@ -199,9 +202,11 @@ of making it.
|
|
|
199
202
|
- Owner-scoped data ("my orders") follows the token owner, not a query
|
|
200
203
|
parameter. A non-owner check needs a second token; without one, skip and
|
|
201
204
|
name the missing identity.
|
|
202
|
-
- If a precondition really cannot be met
|
|
203
|
-
|
|
204
|
-
|
|
205
|
+
- If a precondition really cannot be met (an account, an env variable or data
|
|
206
|
+
the environment lacks — never an element missing from the page), skip at
|
|
207
|
+
run time inside the test and state what was observed ("TEST_X is not set",
|
|
208
|
+
"GET /x answered 404"), never a guessed cause. That is the only skip: no
|
|
209
|
+
skipped test declarations, describe.skip or test.todo.
|
|
205
210
|
|
|
206
211
|
E4. What you wait for is the event that actually happens. Wait on a
|
|
207
212
|
condition, never for a fixed time.
|
|
@@ -290,8 +295,11 @@ export const WEB = `WEB (Playwright)
|
|
|
290
295
|
title. A stubbed item carries every field the screen reads to show it,
|
|
291
296
|
including the value the list filters on (its status), and the test finds it
|
|
292
297
|
on screen by a field the screen renders as text, never by one it does not
|
|
293
|
-
show.
|
|
294
|
-
|
|
298
|
+
show. Finding it is not the check: assert what the product does with that
|
|
299
|
+
state (the badge, the message, the button it disables or hides), never only
|
|
300
|
+
that the text you stubbed appears. Match the stub by exact host, port, path
|
|
301
|
+
prefix or a request predicate, never by a substring that also matches the
|
|
302
|
+
app's own pages.
|
|
295
303
|
- Waiting: prefer retrying assertions. Register waitForResponse or
|
|
296
304
|
waitForRequest before the action that triggers it. After a login, sign-out
|
|
297
305
|
or navigation, wait for the control you will use, then assert the URL the
|
|
@@ -18,6 +18,8 @@ import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
|
18
18
|
import { runAiSync } from '../generate/ai-usage.js';
|
|
19
19
|
import { clearInterface, readInterface, recordInterface } from '../generate/interface-file.js';
|
|
20
20
|
const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
|
|
21
|
+
/** Why each still-blank name is blank, for the waiting message (name → reason). */
|
|
22
|
+
const STATUS_PATH = join(STATE_DIR, 'interface-status.json');
|
|
21
23
|
/** Values the scan filled (name → value), so each run can check the code still has them. */
|
|
22
24
|
const FILLED_PATH = join(STATE_DIR, 'interface-filled.json');
|
|
23
25
|
const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
|
|
@@ -215,7 +217,27 @@ export function namesToAsk(blank, root, memo) {
|
|
|
215
217
|
* Look for the blank names in the code and fill what is found. `blank` are
|
|
216
218
|
* the names the tests to run still wait for. Returns what was filled.
|
|
217
219
|
*/
|
|
218
|
-
|
|
220
|
+
/** Is `value` in the project's code, as a key, string, route path or verb (the facts are read from it, so this checks them)? */
|
|
221
|
+
function inProjectCode(value, root) {
|
|
222
|
+
const files = codeFiles(root);
|
|
223
|
+
const route = files.filter((f) => fileMatches('route', relative(root, f)));
|
|
224
|
+
const read = (fs_) => fs_.flatMap((f) => { try {
|
|
225
|
+
return readFileSync(f, 'utf-8').split('\n');
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
return [];
|
|
229
|
+
} });
|
|
230
|
+
if (/^\/|^[\w-]+\//.test(value.trim()))
|
|
231
|
+
return foundInCode(value, read(route));
|
|
232
|
+
return foundInCode(value, read(files));
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Look for the blank names in the code and fill what is found. `blank` are
|
|
236
|
+
* the names the tests to run still wait for. With `facts` (at generate time)
|
|
237
|
+
* every blank name is asked once against the journey's code facts too; a
|
|
238
|
+
* value is kept only if the code itself has it. Returns what was filled.
|
|
239
|
+
*/
|
|
240
|
+
export function scanNames(blank, root = '.', facts) {
|
|
219
241
|
const memo = (() => {
|
|
220
242
|
try {
|
|
221
243
|
return JSON.parse(readFileSync(SCAN_PATH, 'utf-8'));
|
|
@@ -225,6 +247,10 @@ export function scanNames(blank, root = '.') {
|
|
|
225
247
|
}
|
|
226
248
|
})();
|
|
227
249
|
const all = namesToAsk(blank, root, memo);
|
|
250
|
+
if (facts)
|
|
251
|
+
for (const n of blank)
|
|
252
|
+
if (!all.some((a) => a.name === n))
|
|
253
|
+
all.push({ name: n, lines: [] });
|
|
228
254
|
// One call stays small: names past the budget are asked on the next run.
|
|
229
255
|
const ask = [];
|
|
230
256
|
let size = 0;
|
|
@@ -239,11 +265,14 @@ export function scanNames(blank, root = '.') {
|
|
|
239
265
|
}
|
|
240
266
|
const filled = [];
|
|
241
267
|
let calls = 0;
|
|
268
|
+
let callFailed = false;
|
|
242
269
|
if (ask.length) {
|
|
243
270
|
const known = Object.entries(readInterface()).filter(([, v]) => String(v).trim()).slice(0, 8);
|
|
244
271
|
const prompt = [
|
|
245
272
|
'You match names a test needs to values in code. For each name in <names>,',
|
|
246
|
-
|
|
273
|
+
facts
|
|
274
|
+
? "pick its value from that name's own lines or from <code_facts> (the routes, request fields, response shapes and tables of this feature), or null."
|
|
275
|
+
: "pick its value from that name's own lines only, or null.",
|
|
247
276
|
'',
|
|
248
277
|
'A value is copied from the lines, never invented.',
|
|
249
278
|
'- A URL is the path a client calls: it starts with "/" and includes every',
|
|
@@ -255,6 +284,7 @@ export function scanNames(blank, root = '.') {
|
|
|
255
284
|
'If no line clearly is that thing, the value is null.',
|
|
256
285
|
...(known.length ? ['', '<filled_examples>', ...known.map(([k, v]) => `- ${k}: ${v}`), '</filled_examples>', 'These values are already filled; follow their format.'] : []),
|
|
257
286
|
'',
|
|
287
|
+
...(facts ? ['<code_facts>', facts.slice(0, 20_000), '</code_facts>', ''] : []),
|
|
258
288
|
'<names>',
|
|
259
289
|
...ask.flatMap((a) => [`NAME: ${a.name}`, ...a.lines.map((l) => ` ${l}`)]),
|
|
260
290
|
'</names>',
|
|
@@ -272,7 +302,7 @@ export function scanNames(blank, root = '.') {
|
|
|
272
302
|
const keep = {};
|
|
273
303
|
for (const a of ask) {
|
|
274
304
|
const v = json[a.name];
|
|
275
|
-
if (typeof v === 'string' && foundInCode(v, a.lines))
|
|
305
|
+
if (typeof v === 'string' && v.trim() && (foundInCode(v, a.lines) || (facts && inProjectCode(v, root))))
|
|
276
306
|
keep[a.name] = v.trim();
|
|
277
307
|
}
|
|
278
308
|
recordInterface(keep);
|
|
@@ -297,6 +327,7 @@ export function scanNames(blank, root = '.') {
|
|
|
297
327
|
console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
|
|
298
328
|
for (const a of ask)
|
|
299
329
|
delete memo[a.name]; // look again next time
|
|
330
|
+
callFailed = true;
|
|
300
331
|
}
|
|
301
332
|
}
|
|
302
333
|
try {
|
|
@@ -307,6 +338,30 @@ export function scanNames(blank, root = '.') {
|
|
|
307
338
|
// best-effort memo
|
|
308
339
|
}
|
|
309
340
|
const done = new Set(filled.map((f) => f.name));
|
|
341
|
+
// Why each name is still blank, so a waiting test says whether to look at
|
|
342
|
+
// the product or at TestForge.
|
|
343
|
+
try {
|
|
344
|
+
const status = (() => { try {
|
|
345
|
+
return JSON.parse(readFileSync(STATUS_PATH, 'utf-8'));
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
return {};
|
|
349
|
+
} })();
|
|
350
|
+
for (const n of blank) {
|
|
351
|
+
if (done.has(n)) {
|
|
352
|
+
delete status[n];
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
const asked = ask.find((a) => a.name === n);
|
|
356
|
+
status[n] = !kindOf(n) && !facts ? 'TestForge does not look this kind of name up in the code; fill it by hand'
|
|
357
|
+
: asked && callFailed ? 'the lookup could not run this time; it is tried again on the next run'
|
|
358
|
+
: asked ? 'looked for in the code and not found'
|
|
359
|
+
: 'nothing in the code looks like it yet';
|
|
360
|
+
}
|
|
361
|
+
mkdirSync(dirname(STATUS_PATH), { recursive: true });
|
|
362
|
+
writeFileSync(STATUS_PATH, `${JSON.stringify(status, null, 2)}\n`, 'utf-8');
|
|
363
|
+
}
|
|
364
|
+
catch { /* the message just has no reason */ }
|
|
310
365
|
return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
|
|
311
366
|
}
|
|
312
367
|
/**
|
package/dist/core/run/report.js
CHANGED
|
@@ -43,6 +43,8 @@ export function loadDotenv(cwd = process.cwd()) {
|
|
|
43
43
|
let v = s.slice(i + 1).trim();
|
|
44
44
|
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))
|
|
45
45
|
v = v.slice(1, -1);
|
|
46
|
+
else
|
|
47
|
+
v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
|
|
46
48
|
if (k && process.env[k] === undefined)
|
|
47
49
|
process.env[k] = v;
|
|
48
50
|
}
|
|
@@ -110,6 +110,11 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
|
|
|
110
110
|
}
|
|
111
111
|
export function runTests(opts = {}) {
|
|
112
112
|
const config = loadConfig();
|
|
113
|
+
// The journey goes into a shell command: only a journey id's characters.
|
|
114
|
+
if (opts.journey && !/^[\w.-]+$/.test(opts.journey)) {
|
|
115
|
+
console.log(`"${opts.journey}" is not a journey id (letters, digits, - _ . only) — nothing was run.`);
|
|
116
|
+
return 1;
|
|
117
|
+
}
|
|
113
118
|
const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
|
|
114
119
|
// An env file named on the command line that is not there is a mistake, not a default.
|
|
115
120
|
if (opts.envFile && !existsSync(opts.envFile)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
"files": [
|
|
11
11
|
"dist",
|
|
12
12
|
"reporters",
|
|
13
|
-
"README.md"
|
|
13
|
+
"README.md",
|
|
14
|
+
"CHANGELOG.md"
|
|
14
15
|
],
|
|
15
16
|
"scripts": {
|
|
16
17
|
"build": "rm -rf dist && tsc",
|
package/reporters/interface.cjs
CHANGED
|
@@ -103,7 +103,9 @@ function usersOf(file, name) {
|
|
|
103
103
|
const end = close < 0 ? lines.length : lineOf(close) + 1;
|
|
104
104
|
return { start: i, end, text: lines.slice(i, end).join('\n'), title: (/\(\s*(['"`])((?:(?!\1).)*)\1/.exec(lines[i]) || [])[2] };
|
|
105
105
|
});
|
|
106
|
-
const
|
|
106
|
+
const word = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
|
|
107
|
+
// A name inside a string ('redeem points URL') is not a use of the constant URL.
|
|
108
|
+
const uses = { test: (t) => word.test(String(t).replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, '""')) };
|
|
107
109
|
const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
|
|
108
110
|
if (lines.some((l, i) => i !== declLine && !inTest(i) && uses.test(l))) return undefined;
|
|
109
111
|
return tests.filter((t) => uses.test(t.text));
|
|
@@ -159,14 +161,35 @@ function quietHooks() {
|
|
|
159
161
|
}
|
|
160
162
|
}
|
|
161
163
|
|
|
164
|
+
/** Why the last name scan left `name` blank, if it recorded one. */
|
|
165
|
+
function scanReason(name) {
|
|
166
|
+
try {
|
|
167
|
+
const p = path.join(path.dirname(interfacePath()), 'interface-status.json');
|
|
168
|
+
return JSON.parse(fs.readFileSync(p, 'utf-8'))[name];
|
|
169
|
+
} catch {
|
|
170
|
+
return undefined;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
162
174
|
/** The product's value for `name`, or an error naming what to fill. */
|
|
163
175
|
function need(name) {
|
|
164
176
|
const v = values()[name];
|
|
165
177
|
if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
|
|
166
178
|
// The prefix lets the reporters file this as "waiting for the product", not as a defect.
|
|
167
|
-
const
|
|
179
|
+
const why = scanReason(name);
|
|
180
|
+
const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json)${why ? ` — ${why}` : ''}: this test runs for real once it is there.`;
|
|
168
181
|
if (blockTestsThatUse(message, name)) return `<waiting: ${name}>`;
|
|
169
182
|
throw new Error(message);
|
|
170
183
|
}
|
|
171
184
|
|
|
172
|
-
|
|
185
|
+
/**
|
|
186
|
+
* A path from need() with its parameters filled in order, whatever the
|
|
187
|
+
* product calls them: /bookings/{booking}/cancel, /orders/:id or {id}.
|
|
188
|
+
* fillPath(need('booking cancel URL'), bookingId)
|
|
189
|
+
*/
|
|
190
|
+
function fillPath(template, ...values) {
|
|
191
|
+
let i = 0;
|
|
192
|
+
return String(template).replace(/\{[^}/]+\}|(?<=\/):[A-Za-z_]\w*/g, (m) => (i < values.length ? encodeURIComponent(String(values[i++])) : m));
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
module.exports = { need, fillPath, _usersOf: usersOf };
|