@activate-core/test-forge 2.6.0 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/cli/index.js +4 -1
- package/dist/core/docs/excerpts.js +49 -3
- package/dist/core/docs/grounding.js +4 -0
- package/dist/core/gates/validate.js +49 -23
- package/dist/core/generate/ai-generator.js +41 -0
- package/dist/core/generate/ai-usage.js +1 -0
- package/dist/core/generate/complete-file.js +46 -7
- package/dist/core/generate/run-generate.js +224 -18
- package/dist/core/generate/test-blocks.js +87 -0
- package/dist/core/project/env-diff.js +3 -1
- package/dist/core/project/setup.js +10 -2
- package/dist/core/prompts/blocks.js +13 -5
- package/dist/core/run/name-scan.js +58 -3
- package/dist/core/run/report.js +2 -0
- package/dist/core/run/run-tests.js +5 -0
- package/package.json +3 -2
- package/reporters/interface.cjs +26 -3
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 2.7.1
|
|
4
|
+
|
|
5
|
+
- New code never makes generate write a test again. With the doc unchanged and every rule tested, `generate --refresh-code` reads the code facts again and fills the names the tests wait on from them (two small calls); the test files are not rewritten. `--rewrite-waiting` writes the waiting tests again when asked by name.
|
|
6
|
+
- kit.insert's id is used as the id it is, with no AI: `op[ID_FIELD]` (ID_FIELD = need('inserted row id field')) becomes `op`, and the blank name goes.
|
|
7
|
+
- With `--rewrite-waiting`: a file whose blank names sit in its hooks or helpers has only that setup written again (its tests stay); a file whose blank names sit in a parametrised table or loop is written again whole, once per doc and code, never one holding a passed test.
|
|
8
|
+
- A refused answer keeps the tests in it that pass the checks on their own.
|
|
9
|
+
|
|
10
|
+
## 2.7.0
|
|
11
|
+
|
|
12
|
+
- The names new tests still wait on are looked up once, at generate time, against the journey's code facts; a value is kept only if the code has it.
|
|
13
|
+
- The writer never sees a constant or a "names already asked" entry that is still blank, so rewritten tests do not ask for the same blank names again.
|
|
14
|
+
- A kept file whose tests were taken out to be written again stays as it was when the new tests do not come back (a call that fails or is refused).
|
|
15
|
+
- The facts follow the methods a route handler calls: the class it is given (what `create()` returns) and its response helpers, through the helpers they call (the `{ error: { message } }` shape).
|
|
16
|
+
- `fillPath(URL, …values)` from `reporters/interface.cjs` fills a path's parameters whatever the product calls them.
|
|
17
|
+
- A waiting test says why its name is blank: not a kind TestForge looks up, nothing like it in the code yet, looked for and not found, or the lookup could not run.
|
|
18
|
+
- A rewritten test keeps its title (dashboard history continues).
|
|
19
|
+
- A testkit rebuild that fails, or drops an export the tests use, is not taken: the existing testkit stays.
|
|
20
|
+
- Rules asked for and refused are not asked again until the doc or the journey's code changes.
|
|
21
|
+
- "route", "controller" and "endpoint" no longer match every route line.
|
|
22
|
+
- `env --diff` and `report` read an unfilled `KEY= # hint` as unset; `run --journey` accepts only a journey id.
|
|
23
|
+
- `setup` stops when the config it wrote has problems, before any doc is drafted.
|
|
24
|
+
|
|
25
|
+
## 2.6.0
|
|
26
|
+
|
|
27
|
+
- The reporters record passed tests in `testforge/passed.json`; generate never removes or rewrites them (only `--force` does).
|
|
28
|
+
- A kept file's tests are reported by newer checks, never removed; when code arrives only the tests using its blank names are written again; a failed rewrite keeps the earlier file.
|
|
29
|
+
- A journey's facts move only when its own code changes; tests written with no code are written again once it is there, with the seed.
|
|
30
|
+
- Route handlers and their rule classes come first in the facts; the doc's Test setup reaches the facts; the testkit's `insert` contract always reaches the writer.
|
|
31
|
+
- Name scan: rule, logic, request and enum files; whole-word matching; the enclosing route prefix; methods, roles, nested fields; values the code no longer has are blanked; blanked names are asked again.
|
|
32
|
+
- `run`: files of a feature that is not live are held back; prints how many tests ran of how many; exits 0 when nothing failed for a product reason.
|
|
33
|
+
- A saved review verdict applies to the same failure only; `report --results` files waiting tests as waiting.
|
|
34
|
+
- Globals-style Jest files parse; an unfilled env line is empty; `coverage.json` keeps other journeys; other AI CLIs get the writer rules; a ticket key counts as a source; a missing `--env-file` stops the run.
|
|
35
|
+
|
|
36
|
+
### Upgrading
|
|
37
|
+
- New state files under `testforge/`: `passed.json` (commit it with the tests), `last-run.json`, `interface-filled.json`, `interface-status.json`, and `journeys/<area>/<id>/refused.json`.
|
|
38
|
+
- Facts are read once again after upgrading (one grounding call per journey).
|
package/dist/cli/index.js
CHANGED
|
@@ -93,7 +93,7 @@ function printUsage() {
|
|
|
93
93
|
console.log(' testforge generate --basic Use the basic generator only');
|
|
94
94
|
console.log(' testforge generate --force Regenerate everything');
|
|
95
95
|
console.log(' testforge generate --no-wait Stop when the AI hits its usage limit (default: wait for the reset and resume)');
|
|
96
|
-
console.log(' testforge generate --plan-only Plan and check each journey (testforge/journeys/<area>/<id>/plan-check.md), write no tests', ' testforge generate --journey <name> --refresh-code Read the code again and rewrite the tests still waiting on names (
|
|
96
|
+
console.log(' testforge generate --plan-only Plan and check each journey (testforge/journeys/<area>/<id>/plan-check.md), write no tests', ' testforge generate --journey <name> --refresh-code Read the code again and fill the names the tests wait on (tests are not written again)', ' testforge generate --journey <name> --rewrite-waiting Also write again the tests still waiting on names (costs AI calls)');
|
|
97
97
|
console.log(' testforge generate --journey <name> Generate just ONE journey');
|
|
98
98
|
console.log(' testforge generate --area <area> Generate a whole area (e.g. api)');
|
|
99
99
|
console.log(' testforge discover Draft journey docs (asks: map or reference)');
|
|
@@ -250,6 +250,7 @@ async function main() {
|
|
|
250
250
|
: undefined, // undefined -> config.waitForLimit (default on)
|
|
251
251
|
planOnly: process.argv.includes('--plan-only'),
|
|
252
252
|
refreshCode: process.argv.includes('--refresh-code'),
|
|
253
|
+
rewriteWaiting: process.argv.includes('--rewrite-waiting'),
|
|
253
254
|
});
|
|
254
255
|
if (results.length === 0) {
|
|
255
256
|
console.log('No journey files found in testforge/docs/. Add a .md file and try again.');
|
|
@@ -260,6 +261,8 @@ async function main() {
|
|
|
260
261
|
// A row with no test dir is the run-level notice (AI unavailable), not an
|
|
261
262
|
// unchanged journey — do not label it as one.
|
|
262
263
|
console.log(r.testDir ? ` [skip] ${r.journeyName} (unchanged)` : ` [skip] ${r.journeyName}`);
|
|
264
|
+
for (const w of r.warnings ?? [])
|
|
265
|
+
console.log(` ! ${w}`);
|
|
263
266
|
}
|
|
264
267
|
else {
|
|
265
268
|
const tag = r.status === 'ai' ? '[AI]' : '[basic]';
|
|
@@ -17,7 +17,7 @@ export function sourceRoot(base = '.') {
|
|
|
17
17
|
const FILE_REF = /((?:[\w.-]+\/)+[\w.-]+\.(?:tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go))(?::(\d+)(?:[-–](\d+))?)?/g;
|
|
18
18
|
const QUOTED = /"([^"\n]{4,60})"/g;
|
|
19
19
|
/** Words in a control's name that name no component in particular. */
|
|
20
|
-
const GENERIC = new Set(['button', 'buttons', 'field', 'input', 'inputs', 'header', 'footer', 'items', 'entry', 'entries', 'shown', 'guest', 'logged', 'customer', 'computer', 'phone', 'sized', 'screen', 'page', 'pages', 'where', 'opens', 'close', 'link', 'links', 'text', 'label', 'click', 'state', 'result', 'neighbour', 'neighbouring', 'other', 'while', 'their', 'there', 'which', 'every', 'itself', 'changes', 'layout']);
|
|
20
|
+
const GENERIC = new Set(['route', 'routes', 'controller', 'endpoint', 'button', 'buttons', 'field', 'input', 'inputs', 'header', 'footer', 'items', 'entry', 'entries', 'shown', 'guest', 'logged', 'customer', 'computer', 'phone', 'sized', 'screen', 'page', 'pages', 'where', 'opens', 'close', 'link', 'links', 'text', 'label', 'click', 'state', 'result', 'neighbour', 'neighbouring', 'other', 'while', 'their', 'there', 'which', 'every', 'itself', 'changes', 'layout']);
|
|
21
21
|
function walk(dir, out) {
|
|
22
22
|
let entries;
|
|
23
23
|
try {
|
|
@@ -302,6 +302,7 @@ export function sourceExcerpts(markdown, roots, budget = 70_000) {
|
|
|
302
302
|
const handlers = new Set();
|
|
303
303
|
const handlerFiles = [];
|
|
304
304
|
const seenRules = new Set();
|
|
305
|
+
const seenMethods = new Set();
|
|
305
306
|
// Only the routes the doc names by their path: theirs are the requests its tests send.
|
|
306
307
|
for (const c of candidates.filter((x) => x.score >= 10).slice(0, 12)) {
|
|
307
308
|
let line = '';
|
|
@@ -335,8 +336,53 @@ export function sourceExcerpts(markdown, roots, budget = 70_000) {
|
|
|
335
336
|
// A handler that hands the request to a class it is given
|
|
336
337
|
// (bookingOrderV2(Request $r, OrderLogicV2 $logic)) keeps its rules there.
|
|
337
338
|
const sig = text[at].match(/\(([^)]*)\)/)?.[1] ?? '';
|
|
338
|
-
|
|
339
|
-
|
|
339
|
+
const deps = [...sig.matchAll(/\b([A-Z]\w+)\s+\$(\w+)/g)].filter((d) => d[1] !== 'Request');
|
|
340
|
+
for (const d of deps)
|
|
341
|
+
rulesOf(d[1], 2);
|
|
342
|
+
// What the response looks like is in the methods the handler calls: the
|
|
343
|
+
// class it is given ($order_logic->create() → what create() returns) and
|
|
344
|
+
// its own response helpers ($this->respondWithSuccess, often in a trait).
|
|
345
|
+
const body = text.slice(at, at + 45).join('\n');
|
|
346
|
+
for (const d of deps) {
|
|
347
|
+
for (const call of new Set([...body.matchAll(new RegExp(`\\$${d[2]}->(\\w+)\\(`, 'g'))].map((x) => x[1]))) {
|
|
348
|
+
if (/^(?:setInput|getErrors|fails|validate)$/.test(call))
|
|
349
|
+
continue;
|
|
350
|
+
methodWindow(d[1], call, 60);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
for (const call of new Set([...body.matchAll(/\$this->(respond\w*|\w*[Rr]esponse\w*|json\w*)\(/g)].map((x) => x[1])))
|
|
354
|
+
methodWindow('', call, 15, 3);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
// The body of `method` in class `cls` (any class when cls is ''), where it is defined.
|
|
358
|
+
function methodWindow(cls, method, lines, helperDepth = 0) {
|
|
359
|
+
if (seenMethods.has(`${cls}::${method}`))
|
|
360
|
+
return;
|
|
361
|
+
seenMethods.add(`${cls}::${method}`);
|
|
362
|
+
const re = new RegExp(`function\\s+${method}\\s*\\(`);
|
|
363
|
+
for (const f of files.filter((x) => x.endsWith('.php') && (!cls || x.endsWith(`/${cls}.php`))).slice(0, 4000)) {
|
|
364
|
+
let text;
|
|
365
|
+
try {
|
|
366
|
+
text = readFileSync(f, 'utf-8').split('\n');
|
|
367
|
+
}
|
|
368
|
+
catch {
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
const i = text.findIndex((l) => re.test(l));
|
|
372
|
+
if (i === -1)
|
|
373
|
+
continue;
|
|
374
|
+
add(f, i + 1, i + lines);
|
|
375
|
+
if (!handlerFiles.includes(f))
|
|
376
|
+
handlerFiles.push(f);
|
|
377
|
+
// A helper that builds its answer through another (respondInvalidParameters → respondWithError).
|
|
378
|
+
if (helperDepth > 1) {
|
|
379
|
+
for (const next of new Set([...text.slice(i, i + lines).join('\n').matchAll(/(?:->|::)(respond\w*|\w*[Rr]esponse\w*|json\w*)\(/g)].map((x) => x[1]))) {
|
|
380
|
+
if (next !== method)
|
|
381
|
+
methodWindow('', next, lines, helperDepth - 1);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
if (!cls)
|
|
385
|
+
return; // a helper is defined once (a trait or base controller)
|
|
340
386
|
}
|
|
341
387
|
}
|
|
342
388
|
// The rule methods of a class (validate, rules, getXRules), and of the
|
|
@@ -124,6 +124,10 @@ function journeyCode(journey, srcDir) {
|
|
|
124
124
|
catch { /* unreadable tree: no code */ }
|
|
125
125
|
return { excerpts, hash: createHash('sha1').update(excerpts).digest('hex').slice(0, 16) };
|
|
126
126
|
}
|
|
127
|
+
/** A short fingerprint of the code this journey reads (no AI). */
|
|
128
|
+
export function journeyCodeHash(journey, srcDir = 'src') {
|
|
129
|
+
return journeyCode(journey, srcDir).hash;
|
|
130
|
+
}
|
|
127
131
|
/** True when the journey's facts were read from code that has since changed. */
|
|
128
132
|
export function factsCodeMoved(journey, srcDir = 'src') {
|
|
129
133
|
const path = groundingPath(journey);
|
|
@@ -51,10 +51,11 @@ export const DEFAULT_EXTERNAL_MODULES = [
|
|
|
51
51
|
* clean. Runs AFTER syntax validation.
|
|
52
52
|
*/
|
|
53
53
|
export function checkTestGates(content, _ext, ctx) {
|
|
54
|
+
const reasons = [];
|
|
54
55
|
// F3 — runner correctness: a Jest/backend area must never pull in
|
|
55
56
|
// Playwright (not installed there; a top-level import breaks the suite).
|
|
56
57
|
if (ctx.framework === 'jest' && /@playwright\/test/.test(content)) {
|
|
57
|
-
|
|
58
|
+
reasons.push("jest area must not import '@playwright/test'");
|
|
58
59
|
}
|
|
59
60
|
// F4 — goto guard: `page.goto(...)` must take a real route/URL, not a
|
|
60
61
|
// pasted description and not an unfilled route-file placeholder. Literal
|
|
@@ -67,10 +68,10 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
67
68
|
while ((m = gotoRe.exec(content)) !== null) {
|
|
68
69
|
const arg = m[2];
|
|
69
70
|
if (/\s/.test(arg)) {
|
|
70
|
-
|
|
71
|
+
reasons.push(`goto() argument looks like prose, not a URL/path: "${arg.slice(0, 40)}…"`);
|
|
71
72
|
}
|
|
72
73
|
if (/\[[A-Za-z_]|\/:[A-Za-z_]|\{[A-Za-z_]|<[A-Za-z_]/.test(arg)) {
|
|
73
|
-
|
|
74
|
+
reasons.push(`goto() has an unresolved route placeholder — fill it with a real value: "${arg.slice(0, 40)}"`);
|
|
74
75
|
}
|
|
75
76
|
}
|
|
76
77
|
// F13 — a swallowed action must be bounded. `.click({ force: true })`
|
|
@@ -85,7 +86,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
85
86
|
let sw;
|
|
86
87
|
while ((sw = swallowed.exec(content)) !== null) {
|
|
87
88
|
if (!/timeout\s*:/.test(sw[0])) {
|
|
88
|
-
|
|
89
|
+
reasons.push('an action whose failure is swallowed with .catch() has no timeout — ' +
|
|
89
90
|
'it will wait until the whole test times out and blame a later line');
|
|
90
91
|
}
|
|
91
92
|
}
|
|
@@ -96,7 +97,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
96
97
|
// that never comes. Hold the promise first, then act, then await it.
|
|
97
98
|
const lateWait = /\b(?:await\s+)?page\.goto\([^\n]*\);?\s*\n(?:\s*\/\/[^\n]*\n)*\s*await\s+page\.waitFor(?:Response|Request|Event)\s*\(/;
|
|
98
99
|
if (lateWait.test(content)) {
|
|
99
|
-
|
|
100
|
+
reasons.push('page.waitForResponse/Request is armed AFTER the navigation that triggers ' +
|
|
100
101
|
'it — hold the promise first, then navigate, then await it');
|
|
101
102
|
}
|
|
102
103
|
// F2 — no silent false-green: a file with RUNNABLE tests must assert
|
|
@@ -108,7 +109,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
108
109
|
const assertions = (content.match(/\bexpect\s*\(/g) || []).length +
|
|
109
110
|
(content.match(/\bassert\b/g) || []).length;
|
|
110
111
|
if (activeTests > 0 && assertions === 0) {
|
|
111
|
-
|
|
112
|
+
reasons.push('has runnable test(s) but no assertions — would pass without testing anything');
|
|
112
113
|
}
|
|
113
114
|
// F1 — dependency awareness: reject imports/requires of a module the
|
|
114
115
|
// project can't resolve (bare specifier not in allowedModules).
|
|
@@ -116,7 +117,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
116
117
|
for (const mod of importedModules(content)) {
|
|
117
118
|
const top = topLevelModule(mod);
|
|
118
119
|
if (!ctx.allowedModules.has(top)) {
|
|
119
|
-
|
|
120
|
+
reasons.push(`imports '${mod}' which the project does not have installed`);
|
|
120
121
|
}
|
|
121
122
|
}
|
|
122
123
|
}
|
|
@@ -139,7 +140,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
139
140
|
content.includes(`jest.mock('${top}'`) ||
|
|
140
141
|
content.includes(`jest.mock("${top}"`);
|
|
141
142
|
if (!mocked) {
|
|
142
|
-
|
|
143
|
+
reasons.push(`imports external service '${top}' without a jest.mock — the test would hit the live service; mock it at the seam`);
|
|
143
144
|
}
|
|
144
145
|
}
|
|
145
146
|
}
|
|
@@ -160,7 +161,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
160
161
|
const showedWork = /\b(?:port|origin|path\s*prefix|prefix|resource\s*type|header|predicate)\b/i;
|
|
161
162
|
const m = content.match(sameHost);
|
|
162
163
|
if (m && !showedWork.test(m[0])) {
|
|
163
|
-
|
|
164
|
+
reasons.push('a test is skipped because the dependency "shares a host" with the app, ' +
|
|
164
165
|
'without saying what else was checked — a different port is a different ' +
|
|
165
166
|
'origin, and a path prefix or a request predicate usually separates them ' +
|
|
166
167
|
'too; scope the stub to one of those, or name the ones you ruled out');
|
|
@@ -169,20 +170,20 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
169
170
|
// F16 — no placeholders in a freshly written file: a placeholder never runs,
|
|
170
171
|
// and a suite full of them reads as coverage it does not have.
|
|
171
172
|
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
172
|
-
|
|
173
|
+
reasons.push('placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file');
|
|
173
174
|
}
|
|
174
175
|
// F23 — need() asks for the name of a thing, anywhere in the file (a constant at
|
|
175
176
|
// the top too), never for data the test should make itself.
|
|
176
177
|
if (ctx.forbidPlaceholders) {
|
|
177
178
|
const data = [...content.matchAll(/\bneed\(\s*(['"`])([^'"`\n]+)\1/g)].map((m) => m[2]).filter((n) => !isThingName(n));
|
|
178
179
|
if (data.length) {
|
|
179
|
-
|
|
180
|
+
reasons.push(`asks need() for ${[...new Set(data)].slice(0, 3).map((n) => `"${n}"`).join(', ')}, which is data, not the name of a thing: the test makes its own ids, payloads and records; need() is only for "… URL", "… command", "… table", "… field", "… label"`);
|
|
180
181
|
}
|
|
181
182
|
}
|
|
182
183
|
// F22 — a fresh file exists to test its rules: one with no test at all says
|
|
183
184
|
// nothing, and passes. It goes back once, told to stub what it cannot create.
|
|
184
185
|
if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
|
|
185
|
-
|
|
186
|
+
reasons.push('the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out');
|
|
186
187
|
}
|
|
187
188
|
// F24 — one comment policy: the header, an oracle line above each test,
|
|
188
189
|
// cannot-test lines, a timeout reason, and a short note on a parametrised
|
|
@@ -190,7 +191,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
190
191
|
if (ctx.forbidPlaceholders) {
|
|
191
192
|
const stray = strayComment(content);
|
|
192
193
|
if (stray) {
|
|
193
|
-
|
|
194
|
+
reasons.push(`comment not allowed (line ${stray.line}): "${stray.text.slice(0, 60)}" — the only comments are the // @generated header, one // oracle: line above each test, // cannot-test: lines, a // timeout: reason, and a one-line note on a parametrised row`);
|
|
194
195
|
}
|
|
195
196
|
}
|
|
196
197
|
// F21 — a run-time skip is for what the ENVIRONMENT lacks (an account, a
|
|
@@ -200,27 +201,27 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
200
201
|
if (ctx.forbidPlaceholders) {
|
|
201
202
|
const why = elementSkips(content);
|
|
202
203
|
if (why.length) {
|
|
203
|
-
|
|
204
|
+
reasons.push(`a test skips itself because an element is not on the page (${why[0]}): a missing element the doc names is a failure — assert it instead. Skip only for an account, a variable or data the environment lacks`);
|
|
204
205
|
}
|
|
205
206
|
}
|
|
206
207
|
// F17 — relative imports resolve from where the file is written.
|
|
207
208
|
if (ctx.filePath) {
|
|
208
209
|
const missing = unresolvedImports(content, ctx.filePath);
|
|
209
210
|
if (missing.length) {
|
|
210
|
-
|
|
211
|
+
reasons.push(`relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`);
|
|
211
212
|
}
|
|
212
213
|
}
|
|
213
214
|
// F18 — no fixed sleeps. A wait for a set time either wastes it or is too
|
|
214
215
|
// short on a slower machine; wait for the event or the state instead.
|
|
215
216
|
if (ctx.framework === 'playwright' && ctx.forbidPlaceholders && /\bwaitForTimeout\s*\(/.test(content)) {
|
|
216
|
-
|
|
217
|
+
reasons.push('page.waitForTimeout() is not used: wait for the element, response or URL the test needs (expect(...).toBeVisible(), page.waitForResponse(), expect(page).toHaveURL())');
|
|
217
218
|
}
|
|
218
219
|
// F19 — a long test timeout needs a reason. A 240 s budget hides a test that
|
|
219
220
|
// is stuck; the cap makes it fail early with its real error.
|
|
220
221
|
if (ctx.maxTestTimeoutMs) {
|
|
221
222
|
const long = longTimeoutsWithoutReason(content, ctx.maxTestTimeoutMs);
|
|
222
223
|
if (long.length) {
|
|
223
|
-
|
|
224
|
+
reasons.push(`test.setTimeout(${long[0]}) is above ${ctx.maxTestTimeoutMs} ms with no "// timeout: <why>" comment: wait for what the test needs instead, or state why it takes that long`);
|
|
224
225
|
}
|
|
225
226
|
}
|
|
226
227
|
// F20 — one criterion, a few tests. Thirty inputs against a rule that only
|
|
@@ -229,7 +230,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
229
230
|
if (ctx.maxTestsPerRule && ctx.forbidPlaceholders) {
|
|
230
231
|
const over = testsPerCriterion(content).find((c) => c.tests > ctx.maxTestsPerRule);
|
|
231
232
|
if (over) {
|
|
232
|
-
|
|
233
|
+
reasons.push(`${over.tests} tests check the same criterion (${over.criterion.slice(0, 80)}…): keep at most ${ctx.maxTestsPerRule}, one per branch, and put further inputs in one parametrised test`);
|
|
233
234
|
}
|
|
234
235
|
}
|
|
235
236
|
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
@@ -238,7 +239,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
238
239
|
if (ctx.framework === 'playwright') {
|
|
239
240
|
const bare = reasonlessPending(content);
|
|
240
241
|
if (bare.length) {
|
|
241
|
-
|
|
242
|
+
reasons.push(`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
|
|
242
243
|
`test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
|
|
243
244
|
}
|
|
244
245
|
}
|
|
@@ -257,14 +258,14 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
257
258
|
const sel = String.raw `(?:getByTestId\s*\(|\[\s*data-(?:testid|cy|test)\s*=)`;
|
|
258
259
|
const selNearMarker = new RegExp(`(?:${sel}[^\\n]*?\\b${marker}\\b)|(?:\\b${marker}\\b[^\\n]*?${sel})`, 'i');
|
|
259
260
|
if (selNearMarker.test(active)) {
|
|
260
|
-
|
|
261
|
+
reasons.push('a runnable test uses a guessed/placeholder selector — ground it in a real component attribute, or defer the test (test.fixme/skip)');
|
|
261
262
|
}
|
|
262
263
|
// (b) When the project's real test-hooks are known, any test-id a
|
|
263
264
|
// runnable test uses that isn't one of them is an invented selector.
|
|
264
265
|
if (ctx.knownSelectors && ctx.knownSelectors.size > 0) {
|
|
265
266
|
for (const id of usedTestIds(active)) {
|
|
266
267
|
if (!ctx.knownSelectors.has(id)) {
|
|
267
|
-
|
|
268
|
+
reasons.push(`references selector '${id}' not found in the project's components — ground it or defer the test`);
|
|
268
269
|
}
|
|
269
270
|
}
|
|
270
271
|
}
|
|
@@ -272,11 +273,13 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
272
273
|
if (ctx.knownTexts) {
|
|
273
274
|
const missing = usedLabels(active).filter((l) => !ctx.knownTexts.includes(l.toLowerCase()));
|
|
274
275
|
if (missing.length) {
|
|
275
|
-
|
|
276
|
+
reasons.push(`uses ${missing.map((m) => `"${m}"`).slice(0, 3).join(', ')} as an accessible name, label or placeholder, but that text appears nowhere in the app's source or the journey doc — take every name from the code facts or the rules`);
|
|
276
277
|
}
|
|
277
278
|
}
|
|
278
279
|
}
|
|
279
|
-
|
|
280
|
+
// Every reason at once: a retry told one problem fixes it and trips on the next.
|
|
281
|
+
const unique = [...new Set(reasons)];
|
|
282
|
+
return unique.length ? (unique.length === 1 ? unique[0] : `${unique[0]}\nAlso:\n${unique.slice(1, 5).map((r) => `- ${r}`).join('\n')}`) : null;
|
|
280
283
|
}
|
|
281
284
|
/**
|
|
282
285
|
* The run-time skips whose condition looks at the page: test.skip(true, …),
|
|
@@ -676,6 +679,29 @@ function testCriteria(content) {
|
|
|
676
679
|
return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
|
|
677
680
|
});
|
|
678
681
|
}
|
|
682
|
+
/**
|
|
683
|
+
* The file with at most `max` tests per criterion: the first ones are kept,
|
|
684
|
+
* the extras removed. A mechanical fix — no AI retry for "too many tests".
|
|
685
|
+
*/
|
|
686
|
+
export function trimExtraTests(content, max) {
|
|
687
|
+
const blocks = findTestBlocks(content);
|
|
688
|
+
const crit = testCriteria(content);
|
|
689
|
+
const seen = new Map();
|
|
690
|
+
const extra = blocks.filter((_, i) => {
|
|
691
|
+
const c = crit[i]?.criterion;
|
|
692
|
+
if (!c)
|
|
693
|
+
return false;
|
|
694
|
+
const n = (seen.get(c) ?? 0) + 1;
|
|
695
|
+
seen.set(c, n);
|
|
696
|
+
return n > max;
|
|
697
|
+
});
|
|
698
|
+
if (!extra.length)
|
|
699
|
+
return content;
|
|
700
|
+
let out = content;
|
|
701
|
+
for (const b of [...extra].sort((x, y) => y.start - x.start))
|
|
702
|
+
out = out.slice(0, b.start) + out.slice(b.end).replace(/^;?[ \t]*\n?/, '');
|
|
703
|
+
return out;
|
|
704
|
+
}
|
|
679
705
|
/** How many tests of a file check each criterion (tests with no oracle are not counted). */
|
|
680
706
|
export function testsPerCriterion(content) {
|
|
681
707
|
const counts = new Map();
|
|
@@ -966,3 +966,44 @@ export function cleanOutput(text) {
|
|
|
966
966
|
.join('\n')
|
|
967
967
|
.trim();
|
|
968
968
|
}
|
|
969
|
+
/**
|
|
970
|
+
* Write a file's setup again (its hooks and helpers), not its tests: the
|
|
971
|
+
* setup waited on names the code facts answer, or built data through routes
|
|
972
|
+
* the facts do not show. One small call; the tests stay as they are.
|
|
973
|
+
* Returns the new blocks in order, or null when the answer does not have one
|
|
974
|
+
* per block.
|
|
975
|
+
*/
|
|
976
|
+
export async function rewriteSetupWithAI(input) {
|
|
977
|
+
const mark = (n) => `// ==== SETUP ${n} ====`;
|
|
978
|
+
const prompt = [
|
|
979
|
+
`You are a senior SDET rewriting the setup of one Jest test file: its hooks and helpers, not its tests.`,
|
|
980
|
+
`The tests below this setup stay exactly as they are and use what it sets.`,
|
|
981
|
+
``,
|
|
982
|
+
`Rules`,
|
|
983
|
+
`- Make the data the setup needs with kit.insert(table, row) and kit.update(table, match, values),`,
|
|
984
|
+
` using the tables and columns <code_facts> shows. kit.insert returns the new row's id itself.`,
|
|
985
|
+
`- Send a request only through a route <code_facts> shows, with its fields as written there.`,
|
|
986
|
+
`- Never call need(), and never use a constant that is not in <file_top> or set by this setup.`,
|
|
987
|
+
`- Keep each block what it is (the same hook, or a helper with the same name and parameters), and`,
|
|
988
|
+
` keep every variable it sets (ctx.*, outer let variables, maps) with the same name and meaning.`,
|
|
989
|
+
`- No comments except what was there.`,
|
|
990
|
+
``,
|
|
991
|
+
`Output: for each block, the line ${mark(1)} (with its number) and then its code. Nothing else.`,
|
|
992
|
+
``,
|
|
993
|
+
`<testkit>`, input.helpers, `</testkit>`,
|
|
994
|
+
``,
|
|
995
|
+
...(input.facts ? [`<code_facts>`, input.facts, `</code_facts>`, ``] : []),
|
|
996
|
+
`<file_top path="${input.filePath}">`, input.top, `</file_top>`,
|
|
997
|
+
``,
|
|
998
|
+
...input.blocks.flatMap((b, i) => [mark(i + 1), b.text, ``]),
|
|
999
|
+
].join('\n');
|
|
1000
|
+
const config = loadConfig();
|
|
1001
|
+
const { program, args } = withNoTools(splitCommand(config.aiCommand), 'Answer only with the setup blocks asked for.');
|
|
1002
|
+
// Split on the markers first: the comment cleaner would take them out.
|
|
1003
|
+
const out = await runWithStdin(program, args, prompt, config.aiTimeoutMs);
|
|
1004
|
+
const parts = out.split(/^\/\/ ==== SETUP (\d+) ====[ \t]*$/m);
|
|
1005
|
+
const got = [];
|
|
1006
|
+
for (let i = 1; i < parts.length; i += 2)
|
|
1007
|
+
got[Number(parts[i]) - 1] = parts[i + 1].replace(/^\s*```\w*\s*$/gm, '').trim();
|
|
1008
|
+
return got.length === input.blocks.length && got.every((g) => g && g.length > 0) ? got : null;
|
|
1009
|
+
}
|
|
@@ -82,6 +82,7 @@ const STAGES = [
|
|
|
82
82
|
[/^=== REPAIR ===$/m, 'repair'],
|
|
83
83
|
[/^You are a senior QA engineer\. Generate a complete/, 'write'],
|
|
84
84
|
[/^You are a senior QA engineer writing one .* test file/, 'write'],
|
|
85
|
+
[/^You are a senior SDET rewriting the setup/, 'setup'],
|
|
85
86
|
[/^You are a senior test architect/, 'plan'],
|
|
86
87
|
[/^You are a senior engineer establishing the MECHANICS/, 'grounding'],
|
|
87
88
|
[/^You review ONE claimed product defect/, 'defect-review'],
|
|
@@ -27,6 +27,12 @@ export function shortReason(why) {
|
|
|
27
27
|
export function withoutNames(content) {
|
|
28
28
|
return content.replace(/^[ \t]*(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])[^'"`\n]*\2\s*\);?[ \t]*\n/gm, (line, name) => new RegExp(`\\b${name}\\b`).test(content.replace(line, '')) ? line : '');
|
|
29
29
|
}
|
|
30
|
+
/** A file top without the `const X = need('…')` lines whose name is still blank. */
|
|
31
|
+
export function withoutBlankNames(head, isBlank) {
|
|
32
|
+
if (!isBlank)
|
|
33
|
+
return head;
|
|
34
|
+
return head.replace(/^[ \t]*(?:const|let)\s+[A-Za-z_$][\w$]*\s*=\s*need\(\s*(['"`])([^'"`\n]*)\1\s*\);?[ \t]*\n/gm, (line, _q, name) => (isBlank(name) ? '' : line));
|
|
35
|
+
}
|
|
30
36
|
/** The file without the tests that have a problem. */
|
|
31
37
|
function withoutBlocks(content, problems) {
|
|
32
38
|
const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
|
|
@@ -71,8 +77,10 @@ export async function completeFile(content, ctx) {
|
|
|
71
77
|
: [];
|
|
72
78
|
const redo = redoBlocks
|
|
73
79
|
.filter((b) => !first.problems.some((p) => p.block.start === b.start))
|
|
74
|
-
.map((b) => ({ title: b.title, rules: b.rules, reason:
|
|
80
|
+
.map((b) => ({ title: b.title, rules: b.rules, reason: `it waits on a name the code now has: write it again from the code with the same title, "${b.title}"`, block: b }));
|
|
75
81
|
const firstDrop = [...first.problems.filter(removable), ...redo];
|
|
82
|
+
let failedRedo = false;
|
|
83
|
+
const refusedRules = [];
|
|
76
84
|
const reported = first.problems.filter((p) => !removable(p));
|
|
77
85
|
if (reported.length) {
|
|
78
86
|
notes.push(`left ${reported.length} earlier test(s) in place that a check flags (${[...new Set(reported.map((p) => shortReason(p.reason)))].slice(0, 3).join('; ')}) — not removed, they were already in the file`);
|
|
@@ -82,24 +90,48 @@ export async function completeFile(content, ctx) {
|
|
|
82
90
|
drop(firstDrop);
|
|
83
91
|
// Only rules the doc has are written again: an id with no text makes the writer invent one.
|
|
84
92
|
const known = new Set(ctx.memory.rules.filter((r) => r.tag).map((r) => r.id));
|
|
85
|
-
const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])]
|
|
93
|
+
const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])]
|
|
94
|
+
.filter((id) => known.has(id) && ctx.rules.includes(id) && !(ctx.skip?.has(id) && !redo.some((p) => p.rules.includes(id))));
|
|
86
95
|
if (left.length) {
|
|
87
96
|
const blocks = findTestBlocks(out);
|
|
88
|
-
|
|
97
|
+
// The file's top as the writer sees it: a constant that still waits on a
|
|
98
|
+
// blank name is left out, or the new tests reuse it and wait again.
|
|
99
|
+
const head = withoutBlankNames(out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000), ctx.isBlank);
|
|
89
100
|
const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(out)?.[2] ?? ctx.journey.name;
|
|
90
101
|
const slice = journeySlice(ctx.memory, left);
|
|
91
102
|
calls = 1;
|
|
92
103
|
try {
|
|
93
104
|
const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(firstDrop, first.cov));
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
105
|
+
const fix = ctx.fix ?? ((c) => c);
|
|
106
|
+
let candidate = fix(appendTests(out, onlyNewTests(tests, out, left)));
|
|
107
|
+
let refused = ctx.check(candidate);
|
|
108
|
+
if (refused) {
|
|
109
|
+
// One bad test refuses the whole answer: keep the tests that pass the
|
|
110
|
+
// checks on their own, so the good ones are not thrown away with it.
|
|
111
|
+
const answer = onlyNewTests(tests, out, left);
|
|
112
|
+
let kept = out;
|
|
113
|
+
for (const b of findTestBlocks(answer)) {
|
|
114
|
+
const next = fix(appendTests(kept, answer.slice(b.start, b.end)));
|
|
115
|
+
if (!ctx.check(next))
|
|
116
|
+
kept = next;
|
|
117
|
+
}
|
|
118
|
+
if (kept !== out) {
|
|
119
|
+
notes.push(`some tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0].slice(0, 160)}); the others were kept`);
|
|
120
|
+
candidate = kept;
|
|
121
|
+
refused = null;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (refused) {
|
|
97
125
|
notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
|
|
126
|
+
failedRedo = true;
|
|
127
|
+
refusedRules.push(...left);
|
|
128
|
+
}
|
|
98
129
|
else
|
|
99
130
|
out = candidate;
|
|
100
131
|
}
|
|
101
132
|
catch (e) {
|
|
102
133
|
notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
|
|
134
|
+
failedRedo = true;
|
|
103
135
|
}
|
|
104
136
|
// What the second answer still gets wrong is removed, not sent again.
|
|
105
137
|
const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts }).filter(removable);
|
|
@@ -109,6 +141,13 @@ export async function completeFile(content, ctx) {
|
|
|
109
141
|
}
|
|
110
142
|
}
|
|
111
143
|
}
|
|
144
|
+
// A kept file whose tests were taken out to be written again, and the new
|
|
145
|
+
// ones did not come: it stays as it was. A waiting test beats no test.
|
|
146
|
+
if (ctx.kept && failedRedo && redo.length + firstDrop.length > 0) {
|
|
147
|
+
notes.push('kept the file as it was: the tests to write again did not come back');
|
|
148
|
+
out = content;
|
|
149
|
+
dropped.clear();
|
|
150
|
+
}
|
|
112
151
|
// The last word, checked mechanically: every test that passed is still here.
|
|
113
152
|
const lost = [...locked].filter((t) => before.has(t) && !findTestBlocks(out).some((b) => b.title === t));
|
|
114
153
|
if (lost.length) {
|
|
@@ -133,5 +172,5 @@ export async function completeFile(content, ctx) {
|
|
|
133
172
|
}
|
|
134
173
|
if (cov.missing.length)
|
|
135
174
|
notes.push(`no test for ${cov.missing.join(', ')}`);
|
|
136
|
-
return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes };
|
|
175
|
+
return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes, refusedRules: [...new Set([...refusedRules, ...(calls ? ruleCoverage(out, ctx.rules).missing.filter((id) => !refusedRules.includes(id) && !failedRedo) : [])])] };
|
|
137
176
|
}
|