@activate-core/test-forge 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/dist/core/docs/doc-guidance.js +9 -6
- package/dist/core/docs/excerpts.js +50 -1
- package/dist/core/docs/grounding.js +7 -1
- package/dist/core/gates/test-quality.js +295 -0
- package/dist/core/generate/ai-generator.js +73 -21
- package/dist/core/generate/complete-file.js +77 -0
- package/dist/core/generate/doc-diff.js +3 -2
- package/dist/core/generate/harness.js +15 -3
- package/dist/core/generate/journey-memory.js +72 -3
- package/dist/core/generate/run-generate.js +85 -18
- package/dist/core/generate/update-journey.js +4 -1
- package/dist/core/run/readiness.js +28 -1
- package/dist/core/run/run-tests.js +4 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -387,10 +387,26 @@ history, the plan, and the code facts. No later step sends the whole doc again.
|
|
|
387
387
|
with the reason, if it sleeps for a fixed time, skips because an element is
|
|
388
388
|
missing, has no test or no assertion, spends more than `maxTestsPerRule`
|
|
389
389
|
tests on one rule, or names a label found neither in the code nor the doc.
|
|
390
|
-
6. **
|
|
390
|
+
6. **Every rule, a test that can fail.** Each test is checked on its own:
|
|
391
|
+
every planned rule has a test, or a `// cannot-test: R7 — <the state this
|
|
392
|
+
area cannot set up>` line ("the locator is unknown" is not such a state);
|
|
393
|
+
no assertion accepts the good and the bad outcome alike, compares the page's
|
|
394
|
+
whole text, sits only inside an `if`, or repeats what its own stub returned;
|
|
395
|
+
`kit.find` is awaited and finds elements by the names the screen shows;
|
|
396
|
+
`kit.openStep` names a step the testkit has. The tests that fail a check are
|
|
397
|
+
taken out and their rules written again in ONE small call; what is still
|
|
398
|
+
wrong is removed and reported. The summary line says how many planned rules
|
|
399
|
+
have a test, which cannot be tested here and why, and which have none, and
|
|
400
|
+
`rules.json` / `plan-check.md` record the same.
|
|
401
|
+
7. **A changed doc.** Only what changed is touched: a renamed label is replaced
|
|
391
402
|
in its tests, a removed rule's tests are deleted (both without AI); a
|
|
392
403
|
reworded or new rule gets its tests written by one small call.
|
|
393
404
|
|
|
405
|
+
**A feature that is not live yet.** Put `Status: not built` or
|
|
406
|
+
`Status: behind flag <name>` in the journey doc's header. Every file of the
|
|
407
|
+
journey is stamped `// testforge-pending: …`, and `testforge run` (and `bind`)
|
|
408
|
+
hold it back until the line is removed; `--all` runs it anyway.
|
|
409
|
+
|
|
394
410
|
`generate` runs nothing. `testforge run` runs the suite; `testforge review`
|
|
395
411
|
judges failures when you ask for it.
|
|
396
412
|
|
|
@@ -127,15 +127,18 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
127
127
|
` becomes a test that fails on a correct product.`,
|
|
128
128
|
` An [observed] criterion is about what the customer sees or what the`,
|
|
129
129
|
` server accepts or refuses — never an implementation detail (a storage`,
|
|
130
|
-
` key name, a CSS class, an internal flag, a request field name
|
|
131
|
-
` source itself is about it.`,
|
|
132
|
-
` Only what a test can drive is a criterion. Behaviour no
|
|
133
|
-
` endpoint can reach (
|
|
134
|
-
`
|
|
130
|
+
` key name, a CSS class, an internal flag, a request field name, a log`,
|
|
131
|
+
` line, a lock, a cache) unless the source itself is about it.`,
|
|
132
|
+
` Only what a test in THIS area can drive is a criterion. Behaviour no`,
|
|
133
|
+
` screen or endpoint of this area can reach (a backend job, another app,`,
|
|
134
|
+
` an operator's action) is not tagged — list it under "## Not requirements"`,
|
|
135
|
+
` with where it is checked, or mention it in Test setup if the generator`,
|
|
136
|
+
` needs to know.`,
|
|
135
137
|
``,
|
|
136
138
|
`METHOD. Work through these steps for every journey, in order:`,
|
|
137
139
|
`1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
|
|
138
|
-
` interactive element and where it appears
|
|
140
|
+
` interactive element and where it appears, each row's first cell its id`,
|
|
141
|
+
` C1, C2, C3 …: each input, button, link,`,
|
|
139
142
|
` checkbox, code box, dropdown, pop-up, menu item, and the page itself.`,
|
|
140
143
|
` Secondary controls count as much as the main form: Resend, Back,`,
|
|
141
144
|
` show/hide, close, apply, sign-out, provider buttons. Apply every`,
|
|
@@ -83,7 +83,7 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
83
83
|
}
|
|
84
84
|
// A file named after a control ("Escalate button" → escalateDisputeButton/) is
|
|
85
85
|
// the component itself: found by its path, before any text search.
|
|
86
|
-
const controlWords = [...markdown.matchAll(/^\|\s*C
|
|
86
|
+
const controlWords = [...markdown.matchAll(/^\|\s*C?\d+[a-z]?\s*\|([^|]+)\|/gm)]
|
|
87
87
|
.flatMap((m) => m[1].replace(/["“”(][^"”)]*["”)]?/g, ' ').match(/[A-Za-z]{5,}/g) ?? [])
|
|
88
88
|
.map((w) => w.toLowerCase())
|
|
89
89
|
.filter((w) => !GENERIC.has(w));
|
|
@@ -198,6 +198,55 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
198
198
|
order.push(h.file);
|
|
199
199
|
}
|
|
200
200
|
}
|
|
201
|
+
// A page's own parts (its total, its operator card, its pickers) live in the
|
|
202
|
+
// components it imports from its own folder. Their visible texts and labels
|
|
203
|
+
// come right after it, ahead of files that only share a word with the doc.
|
|
204
|
+
const textLine = /(>\s*[A-Za-z£$€][^<>{}]{1,60}<)|\b(aria-label|placeholder|title|label|alt)\s*=\s*['"{]|\btitle:\s*['"`]|<(button|Button|a|Link|h[1-6]|label)\b/;
|
|
205
|
+
for (const f of order.slice(0, 6)) {
|
|
206
|
+
const dir = f.replace(/[\\/][^\\/]+$/, '');
|
|
207
|
+
let text;
|
|
208
|
+
try {
|
|
209
|
+
text = readFileSync(f, 'utf-8');
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
const children = [...text.matchAll(/from\s+['"]([^'"]+)['"]/g)]
|
|
215
|
+
.map((m) => m[1].replace(/^[@~.]+[\\/]?/, '').replace(/^src[\\/]/, ''))
|
|
216
|
+
.map((spec) => files.find((t) => t !== f && t.startsWith(dir) && t.replace(/\.\w+$/, '').replace(/[\\/]index$/, '').endsWith(spec)))
|
|
217
|
+
.filter((c) => !!c && /\.(tsx|jsx|vue|svelte)$/.test(c));
|
|
218
|
+
let at = order.indexOf(f);
|
|
219
|
+
for (const c of [...new Set(children)].slice(0, 12)) {
|
|
220
|
+
const lines = readFileSync(c, 'utf-8').split('\n');
|
|
221
|
+
lines.forEach((l, i) => { if (textLine.test(l))
|
|
222
|
+
add(c, i, i + 2); });
|
|
223
|
+
if (!windows.has(c))
|
|
224
|
+
continue;
|
|
225
|
+
const was = order.indexOf(c);
|
|
226
|
+
if (was !== -1 && was <= at)
|
|
227
|
+
continue;
|
|
228
|
+
if (was !== -1)
|
|
229
|
+
order.splice(was, 1);
|
|
230
|
+
order.splice(++at, 0, c);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
// A hook or API module says WHERE the app sends each request, often far below
|
|
234
|
+
// its first lines: every line that makes a call or names a path is kept.
|
|
235
|
+
for (const f of order) {
|
|
236
|
+
if (!/(^|[\\/])(use[A-Z]\w*|api|services?|dataProvider|queries|requests?)([\\/.]|$)/.test(f))
|
|
237
|
+
continue;
|
|
238
|
+
let lines;
|
|
239
|
+
try {
|
|
240
|
+
lines = readFileSync(f, 'utf-8').split('\n');
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
lines.forEach((l, i) => {
|
|
246
|
+
if (/\b(?:get|post|put|patch|del|delete|fetch|request|axios(?:\.\w+)?)\s*(?:<[^>]*>)?\s*\(|['"`]\/[\w-]+\//.test(l))
|
|
247
|
+
add(f, i + 1 - 3, i + 1 + 3);
|
|
248
|
+
});
|
|
249
|
+
}
|
|
201
250
|
const out = [];
|
|
202
251
|
let used = 0;
|
|
203
252
|
for (const file of order) {
|
|
@@ -37,7 +37,7 @@ const MAP_PATH = 'testforge.md';
|
|
|
37
37
|
* that upgrades keeps serving the facts it cached under the old version —
|
|
38
38
|
* so the new facts never reach a single prompt and the upgrade does nothing.
|
|
39
39
|
*/
|
|
40
|
-
const FACTS_VERSION =
|
|
40
|
+
const FACTS_VERSION = 5;
|
|
41
41
|
/** Where a journey's grounded facts live, mirroring the plan cache layout. */
|
|
42
42
|
function groundingPath(journey) {
|
|
43
43
|
return join(journeyDir(journey), 'facts.json');
|
|
@@ -167,6 +167,12 @@ reuseOnly = false) {
|
|
|
167
167
|
const excerpts = sourceExcerpts(journey.rawMarkdown || '', [srcDir, ...(config.relatedDirs ?? [])]);
|
|
168
168
|
if (!excerpts)
|
|
169
169
|
return null;
|
|
170
|
+
// Excerpts only from other packages (a backend matched on a quoted text) say
|
|
171
|
+
// nothing about this area's screens: a call on them returns "unknown" for
|
|
172
|
+
// every section and costs as much as a real one.
|
|
173
|
+
const files = [...excerpts.matchAll(/^### (.+)$/gm)].map((m) => m[1]);
|
|
174
|
+
if (files.length && files.every((f) => f.startsWith('..')))
|
|
175
|
+
return null;
|
|
170
176
|
const { program, args } = withNoTools(splitCommand(config.aiCommand));
|
|
171
177
|
let facts;
|
|
172
178
|
try {
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — does each written test really check its rule?
|
|
3
|
+
*
|
|
4
|
+
* The file-level gates see a file as text. These checks read each test on its
|
|
5
|
+
* own, against the rules the file was planned for:
|
|
6
|
+
*
|
|
7
|
+
* - every planned rule has a test, or a "// cannot-test: R7 — <what is
|
|
8
|
+
* missing>" line that says why not (Q1);
|
|
9
|
+
* - each test has an assertion that runs, not one only inside an if or a
|
|
10
|
+
* catch (Q2);
|
|
11
|
+
* - no assertion accepts the rule's success and its failure alike, like
|
|
12
|
+
* /confirm|time not available/ (Q3);
|
|
13
|
+
* - no test compares the page's whole text with itself (Q4);
|
|
14
|
+
* - a test does more than assert that something is absent, unless its rule
|
|
15
|
+
* says never / no / not (Q5);
|
|
16
|
+
* - kit.find is awaited: it returns a promise (Q6);
|
|
17
|
+
* - kit.openStep names a step the testkit has (Q7);
|
|
18
|
+
* - a run-time skip does not depend on what the page shows (Q8).
|
|
19
|
+
*
|
|
20
|
+
* Each problem names its test and its rules, so only those tests are written
|
|
21
|
+
* again; the rest of the file is kept.
|
|
22
|
+
*/
|
|
23
|
+
import { matchParen } from './validate.js';
|
|
24
|
+
import { findTestBlocks } from '../generate/test-blocks.js';
|
|
25
|
+
const CANNOT = /\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)$/gm;
|
|
26
|
+
/** Which planned rules have a test, which are explained, which are simply missing. */
|
|
27
|
+
/**
|
|
28
|
+
* Reasons that are not a state the area cannot set up: an element is found by
|
|
29
|
+
* the doc's name with kit.find, and two people at once is a run-time skip on
|
|
30
|
+
* kit.distinctRunAccount(). A rule given one of these is still missing.
|
|
31
|
+
*/
|
|
32
|
+
const NOT_A_REASON = [
|
|
33
|
+
/\b(?:locators?|selectors?|test ?ids?|elements?|controls?|handles?|buttons?|fields?)\b[^.;]*\b(?:unknown|not known|not in the code facts|not listed|not documented|not named|named nowhere|not exposed|not identified)\b/i,
|
|
34
|
+
/\bno (?:known |documented )?(?:locator|selector|element|control|handle)\b/i,
|
|
35
|
+
/\bknown (?:[\w-]+ )?(?:locator|selector|element|control|handle)\b/i,
|
|
36
|
+
/\b(?:second|two|distinct|another|different) (?:\w+ )?(?:customer|account|user|people|person)s?\b/i,
|
|
37
|
+
];
|
|
38
|
+
export const acceptedReason = (why) => !NOT_A_REASON.some((re) => re.test(why));
|
|
39
|
+
export function ruleCoverage(content, planned) {
|
|
40
|
+
const tested = new Set(findTestBlocks(content).flatMap((b) => b.rules));
|
|
41
|
+
const cannotTest = [];
|
|
42
|
+
for (const m of content.matchAll(CANNOT)) {
|
|
43
|
+
for (const id of m[1].split(/\s*,\s*/))
|
|
44
|
+
if (planned.includes(id) && !tested.has(id) && acceptedReason(m[2]))
|
|
45
|
+
cannotTest.push({ id, why: m[2].trim() });
|
|
46
|
+
}
|
|
47
|
+
const explained = new Set(cannotTest.map((c) => c.id));
|
|
48
|
+
return { missing: planned.filter((id) => !tested.has(id) && !explained.has(id)), cannotTest };
|
|
49
|
+
}
|
|
50
|
+
/** The index of the brace closing the one at `open`, skipping strings and comments. */
|
|
51
|
+
function matchBrace(s, open) {
|
|
52
|
+
let depth = 0;
|
|
53
|
+
for (let i = open; i < s.length; i++) {
|
|
54
|
+
const c = s[i];
|
|
55
|
+
if (c === "'" || c === '"' || c === '`') {
|
|
56
|
+
for (i++; i < s.length && s[i] !== c; i++)
|
|
57
|
+
if (s[i] === '\\')
|
|
58
|
+
i++;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (c === '/' && s[i + 1] === '/') {
|
|
62
|
+
while (i < s.length && s[i] !== '\n')
|
|
63
|
+
i++;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (c === '/' && s[i + 1] === '*') {
|
|
67
|
+
i = s.indexOf('*/', i + 2);
|
|
68
|
+
if (i === -1)
|
|
69
|
+
return -1;
|
|
70
|
+
i++;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (c === '{')
|
|
74
|
+
depth++;
|
|
75
|
+
else if (c === '}' && --depth === 0)
|
|
76
|
+
return i;
|
|
77
|
+
}
|
|
78
|
+
return -1;
|
|
79
|
+
}
|
|
80
|
+
/** The code with every if/else/try/catch body (and .catch(…) callback) taken out. */
|
|
81
|
+
function withoutBranches(code) {
|
|
82
|
+
let out = code;
|
|
83
|
+
for (;;) {
|
|
84
|
+
const m = /\b(?:if\s*\(|else\b|try\b|catch\b|\.catch\s*\()/.exec(out);
|
|
85
|
+
if (!m)
|
|
86
|
+
return out;
|
|
87
|
+
let at = m.index + m[0].length;
|
|
88
|
+
if (/\($/.test(m[0])) {
|
|
89
|
+
const close = matchParen(out, at - 1);
|
|
90
|
+
if (close === -1)
|
|
91
|
+
return out;
|
|
92
|
+
at = close + 1;
|
|
93
|
+
if (m[0].startsWith('.catch')) {
|
|
94
|
+
out = out.slice(0, m.index) + out.slice(at);
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
else if (/^catch$/.test(m[0]) && /^\s*\(/.test(out.slice(at))) {
|
|
99
|
+
const close = matchParen(out, out.indexOf('(', at));
|
|
100
|
+
if (close === -1)
|
|
101
|
+
return out;
|
|
102
|
+
at = close + 1;
|
|
103
|
+
}
|
|
104
|
+
const rest = out.slice(at);
|
|
105
|
+
const brace = /^\s*\{/.exec(rest);
|
|
106
|
+
if (brace) {
|
|
107
|
+
const open = at + brace[0].length - 1;
|
|
108
|
+
const close = matchBrace(out, open);
|
|
109
|
+
if (close === -1)
|
|
110
|
+
return out;
|
|
111
|
+
out = out.slice(0, m.index) + out.slice(close + 1);
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
// A one-statement branch: drop up to the end of the statement.
|
|
115
|
+
const end = out.indexOf(';', at);
|
|
116
|
+
out = out.slice(0, m.index) + (end === -1 ? '' : out.slice(end + 1));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const stripComments = (code) => code.replace(/^\s*\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '');
|
|
121
|
+
/** Words for a good outcome and for a bad one: one assertion that accepts both checks nothing. */
|
|
122
|
+
const GOOD = /confirm|success|placed|booked|saved|created|thank|welcome|done|complete|sent/i;
|
|
123
|
+
const BAD = /not available|unavailable|error|fail|declin|invalid|denied|refus|reject|sorry|cannot|can't|unable|wrong|taken/i;
|
|
124
|
+
/** Regex literals inside assertion-like calls that offer alternatives. */
|
|
125
|
+
function eitherOutcome(code) {
|
|
126
|
+
const calls = /(?:getByText|toHaveText|toContainText|toMatch|toHaveURL|toHaveTitle|test)\s*\(\s*\/((?:\\\/|[^/\n])+)\/[a-z]*/g;
|
|
127
|
+
for (const m of code.matchAll(calls)) {
|
|
128
|
+
const alts = m[1].split('|');
|
|
129
|
+
if (alts.length < 2)
|
|
130
|
+
continue;
|
|
131
|
+
if (alts.some((a) => GOOD.test(a)) && alts.some((a) => BAD.test(a)))
|
|
132
|
+
return `/${m[1]}/`;
|
|
133
|
+
}
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
/** Problems of each test in a file. `steps` are the testkit's openStep names, when known. */
|
|
137
|
+
/** The words of a regex name: its alternatives, without the regex syntax. */
|
|
138
|
+
const regexWords = (src) => src.split('|').map((a) => a.replace(/\\[sbdw]/g, ' ').replace(/[\\^$()[\]{}?*+.]/g, ' ').replace(/\s+/g, ' ').trim().toLowerCase()).filter((a) => /[a-z]{3}/.test(a));
|
|
139
|
+
/**
|
|
140
|
+
* Problems of each test in a file. `steps` are the testkit's openStep names,
|
|
141
|
+
* `knownTexts` the app's source and the doc, lower-cased: every name a test
|
|
142
|
+
* finds an element by must come from them.
|
|
143
|
+
*/
|
|
144
|
+
export function testProblems(content, opts = {}) {
|
|
145
|
+
const out = [];
|
|
146
|
+
for (const block of findTestBlocks(content)) {
|
|
147
|
+
const text = content.slice(block.start, block.end);
|
|
148
|
+
const oracle = (/\/\/\s*oracle:\s*(.+)$/m.exec(text)?.[1] ?? '').toLowerCase();
|
|
149
|
+
const code = stripComments(text);
|
|
150
|
+
const seen = new Set();
|
|
151
|
+
const add = (reason) => {
|
|
152
|
+
if (seen.has(reason))
|
|
153
|
+
return;
|
|
154
|
+
seen.add(reason);
|
|
155
|
+
out.push({ title: block.title, rules: block.rules, reason, block });
|
|
156
|
+
};
|
|
157
|
+
// A stub's own words asserted back: the test proves the stub, not the app.
|
|
158
|
+
const stubbed = [...code.matchAll(/\.fulfill\(\s*\{([\s\S]*?)\}\s*\)/g)].flatMap((m) => [...m[1].matchAll(/(['"`])([^'"`\n]{6,})\1/g)].map((s) => s[2].toLowerCase()).filter((s) => /[a-z]{4,}\s/i.test(s)));
|
|
159
|
+
if (stubbed.length) {
|
|
160
|
+
const expected = [...code.matchAll(/(?:getByText|toHaveText|toContainText|toMatch)\s*\(\s*(?:\/((?:\\\/|[^/\n])+)\/|(['"`])([^'"`\n]+)\2)/g)]
|
|
161
|
+
.flatMap((m) => (m[1] ? m[1].split('|') : [m[3]]).map((t) => t.toLowerCase().replace(/\\/g, '').trim()))
|
|
162
|
+
.filter((t) => t.length >= 4);
|
|
163
|
+
const echo = expected.find((t) => stubbed.some((s) => s.includes(t)));
|
|
164
|
+
if (echo)
|
|
165
|
+
add(`asserts "${echo}", which its own page.route stub returns — the test proves the stub, not the app: assert what the app does with that state (what the doc says the screen shows)`);
|
|
166
|
+
}
|
|
167
|
+
// …or a value handed to a stub and then looked for on the screen.
|
|
168
|
+
const handed = new Set([...code.matchAll(/\b(?:stub\w*|mock\w*|fulfill|route)\s*\(([^;]*?)\)\s*;/gi)]
|
|
169
|
+
.flatMap((m) => m[1].match(/\b[a-z_$][\w$]*\b/gi) ?? [])
|
|
170
|
+
.filter((v) => !/^(page|route|request|url|r|req|status|body|json|true|false|null|undefined|JSON|stringify|contentType|async|await|const)$/.test(v)));
|
|
171
|
+
const shown = [...code.matchAll(/(?:getByText|toHaveText|toContainText)\s*\(\s*([a-z_$][\w$]*)\s*[,)]/gi)].map((m) => m[1]);
|
|
172
|
+
const back = shown.find((v) => handed.has(v));
|
|
173
|
+
if (back)
|
|
174
|
+
add(`looks on the screen for ${back}, the value its own stub returned — the test proves the stub, not the app: assert what the app decides with that state`);
|
|
175
|
+
// Q6 — kit.find returns a promise.
|
|
176
|
+
// `await kit.find(…).first()` is wrong too: the await applies to the whole chain.
|
|
177
|
+
if (/\bkit\.find\([^)]*\)\s*\.(?!then\b)/.test(code) || /expect\(\s*kit\.find\(/.test(code)) {
|
|
178
|
+
add('uses kit.find(...) without await: it returns a promise — write `const x = await kit.find(page, kind, name)` first, then use x');
|
|
179
|
+
}
|
|
180
|
+
// …kept in a variable without await is the same mistake.
|
|
181
|
+
if (/\b(?:const|let|var)\s+\w+\s*=\s*kit\.find\(/.test(code)) {
|
|
182
|
+
add('keeps kit.find(...) in a variable without await: it is a promise, not the element — `const x = await kit.find(...)`');
|
|
183
|
+
}
|
|
184
|
+
// Q10 — the name an element is found by comes from the screen's source or the doc.
|
|
185
|
+
for (const m of code.matchAll(/\bkit\.find\(\s*\w+\s*,\s*['"`][\w-]+['"`]\s*,\s*/g)) {
|
|
186
|
+
const rest = code.slice(m.index + m[0].length);
|
|
187
|
+
const v = /^([A-Za-z_$][\w$]*)\s*[,)]/.exec(rest)?.[1];
|
|
188
|
+
// A name from a list written out in the test (a parametrised loop, a const) is fine.
|
|
189
|
+
const writtenOut = v && (new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
|
|
190
|
+
new RegExp(`for\\s*\\(\\s*(?:const|let)\\s+(?:${v}|\\[[^\\]]*\\b${v}\\b[^\\]]*\\])\\s+of\\s+\\[\\s*['"\`\\[]`).test(code));
|
|
191
|
+
if (/^[A-Za-z_$]/.test(rest) && !writtenOut) {
|
|
192
|
+
add("finds an element by a name held in a variable: take the name the screen shows from the CODE FACTS or the doc, written out — a value the test made up (kit.unique, a stub's id) is never a control's name");
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
const re = /^\/((?:\\\/|[^/\n])+)\//.exec(rest);
|
|
196
|
+
if (re && opts.knownTexts && !regexWords(re[1]).some((w) => opts.knownTexts.includes(w))) {
|
|
197
|
+
add(`finds an element by /${re[1]}/, which matches no text in the app's source or the doc: use the name the screen really shows (the CODE FACTS give the component's text)`);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
// Q11 — a logged-out state reached by wiping storage: the testkit's login seed puts the session back.
|
|
201
|
+
if (/kit\.login\(/.test(code) && /\b(?:clearCookies\s*\(|(?:localStorage|sessionStorage)\.clear\s*\(|storageState\s*\(\s*\{\s*\}|removeItem\s*\()/.test(code)) {
|
|
202
|
+
add("logs out by clearing storage or cookies after kit.login: the testkit's login seed restores the session — log out through the app's own Log Out control");
|
|
203
|
+
}
|
|
204
|
+
// Q12 — one tab's login reaching another: kit.login prepares one page, so it cannot be set up.
|
|
205
|
+
if (/\b(?:other|second|another|both)\s+tabs?\b|\belsewhere\b/.test(oracle) && /\bnewPage\s*\(/.test(code) && /kit\.login\(/.test(code)) {
|
|
206
|
+
add("tests a login or logout in one tab reaching another: kit.login prepares one page, so the testkit cannot set that up — give the rule a cannot-test line");
|
|
207
|
+
}
|
|
208
|
+
// Q7 — only steps the testkit has.
|
|
209
|
+
if (opts.steps?.length) {
|
|
210
|
+
for (const m of code.matchAll(/\bopenStep\(\s*\w+\s*,\s*(['"`])([^'"`]+)\1/g)) {
|
|
211
|
+
if (!opts.steps.includes(m[2]))
|
|
212
|
+
add(`calls kit.openStep with step "${m[2]}", which the testkit does not have (its steps: ${opts.steps.join(', ')})`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// Q8 — a skip decided by what the page shows.
|
|
216
|
+
for (const m of code.matchAll(/\btest\.skip\s*\(/g)) {
|
|
217
|
+
const close = matchParen(code, m.index + m[0].length - 1);
|
|
218
|
+
const cond = close === -1 ? '' : code.slice(m.index + m[0].length, close).split(/,\s*['"`]/)[0];
|
|
219
|
+
const flag = /^\s*!?\s*\(?\s*([A-Za-z_$][\w$]*)/.exec(cond)?.[1];
|
|
220
|
+
const fromText = flag && new RegExp(`\\b(?:const|let|var)\\s+${flag}\\s*=[^;\\n]*(?:innerText|textContent|allInnerTexts|allTextContents)\\(`).test(code);
|
|
221
|
+
if (/innerText\(|textContent\(|allInnerTexts\(|allTextContents\(/.test(cond) || fromText) {
|
|
222
|
+
add('skips itself because of what the page shows: what the doc expects on the page is asserted, never a reason to skip');
|
|
223
|
+
break;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
// Q9 — real data is made by the run account and recorded, never by the shared fixture.
|
|
227
|
+
const createsData = /(?:find\(\s*\w+\s*,\s*['"`]button['"`]\s*,\s*|getByRole\(\s*['"`]button['"`]\s*,\s*\{\s*name:\s*)['"`/]\s*(?:pay\b|book(?!\s*again)\b|place\b|confirm\s+(?:payment|booking)|complete\s+(?:payment|booking)|submit\s+order)/i.test(code);
|
|
228
|
+
if (createsData && /\.click\(|settledClick\(/.test(code) && !/\.route\(/.test(code) && !/['"`]run['"`]/.test(code) && !/kit\.(?:unique|created)\(/.test(code)) {
|
|
229
|
+
add("clicks a button that creates a booking or a payment, but not as the testkit's 'run' role with kit.unique/kit.created: real data is made by the run account and recorded so it can be cleaned up");
|
|
230
|
+
}
|
|
231
|
+
// Q2 — an assertion that always runs.
|
|
232
|
+
const top = withoutBranches(code);
|
|
233
|
+
if (!/\bexpect(?:\.soft)?\s*\(|\bexpect\.poll\s*\(/.test(top)) {
|
|
234
|
+
add(/\bexpect\s*\(/.test(code)
|
|
235
|
+
? 'its only assertions sit inside an if, else or catch, so the test can pass without checking anything — assert unconditionally'
|
|
236
|
+
: 'has no assertion');
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
// Q3 — one assertion that accepts success and failure.
|
|
240
|
+
const either = eitherOutcome(code);
|
|
241
|
+
if (either && !(GOOD.test(oracle) && BAD.test(oracle) && /\bor\b/.test(oracle))) {
|
|
242
|
+
add(`asserts ${either}, which matches both the good and the bad outcome — assert the one outcome the rule expects`);
|
|
243
|
+
}
|
|
244
|
+
// Q4 — the page's whole text compared with itself.
|
|
245
|
+
if (/locator\(\s*['"`](?:body|main|html)['"`]\s*\)\s*\.(?:innerText|textContent)\(/.test(code) && /\.split\(\s*['"`]\\n['"`]\s*\)|toContainText\(\s*\w+/.test(code)) {
|
|
246
|
+
add("compares the page's text with itself (body/main innerText): read the element the rule is about (the operator's name, the total) and compare that");
|
|
247
|
+
}
|
|
248
|
+
// Q5 — only that something is absent.
|
|
249
|
+
const asserts = [...top.matchAll(/expect(?:\.soft)?\s*\(([\s\S]*?)\)\s*\.\s*((?:not\.)?\w+)/g)];
|
|
250
|
+
const positive = asserts.filter((a) => !a[2].startsWith('not.') && !/^toHaveCount$/.test(a[2]) || /toHaveCount\(\s*[1-9]/.test(top));
|
|
251
|
+
if (asserts.length && !positive.length && !/\b(never|no|not|without|cannot|isn't|doesn't|none|nothing|hidden|absent|disappear|removed)\b/.test(oracle)) {
|
|
252
|
+
add('only asserts that something is absent: also assert the outcome the rule states is there');
|
|
253
|
+
}
|
|
254
|
+
// Always true, whatever the app does.
|
|
255
|
+
if (/expect\(\s*new Set\(\s*(\w+)\s*\)\.size\s*\)\.toBeLessThanOrEqual\(\s*\1\.length\s*\)/.test(code) || /expect\([^)]*\b(?:false\s*\|\||true\s*&&)/.test(code) || /\.toBeDefined\(\)/.test(top) && !/expect\([^)]*\)\.(?!toBeDefined)\w+/.test(top)) {
|
|
256
|
+
add('has an assertion that cannot fail (a set is never larger than its list; false || x; only toBeDefined) — assert the outcome itself');
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
return out;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* The step names a testkit's openStep accepts: the keys of the object its
|
|
263
|
+
* openStep looks the step up in (if (!STEPS[step]) …).
|
|
264
|
+
*/
|
|
265
|
+
export function harnessSteps(harnessSource) {
|
|
266
|
+
// The contract's own export first (STEPS), else the object openStep looks the step up in.
|
|
267
|
+
let map = /(?:const|let|var)\s+STEPS\s*=\s*\{/.test(harnessSource) ? 'STEPS' : undefined;
|
|
268
|
+
if (!map) {
|
|
269
|
+
const fn = /function\s+openStep\s*\(\s*\w+\s*,\s*(\w+)/.exec(harnessSource);
|
|
270
|
+
if (!fn)
|
|
271
|
+
return [];
|
|
272
|
+
const body = harnessSource.slice(fn.index, fn.index + 1500);
|
|
273
|
+
map = new RegExp(`!\\s*(\\w+)\\[\\s*${fn[1]}\\s*\\]`).exec(body)?.[1];
|
|
274
|
+
}
|
|
275
|
+
if (!map)
|
|
276
|
+
return [];
|
|
277
|
+
const decl = new RegExp(`(?:const|let|var)\\s+${map}\\s*=\\s*\\{`).exec(harnessSource);
|
|
278
|
+
if (!decl)
|
|
279
|
+
return [];
|
|
280
|
+
const open = decl.index + decl[0].length - 1;
|
|
281
|
+
const close = matchBrace(harnessSource, open);
|
|
282
|
+
if (close === -1)
|
|
283
|
+
return [];
|
|
284
|
+
return [...harnessSource.slice(open + 1, close).matchAll(/^\s*(?:(['"])([^'"]+)\1|([A-Za-z_$][\w$-]*))\s*:/gm)].map((m) => m[2] ?? m[3]);
|
|
285
|
+
}
|
|
286
|
+
/** One refusal naming every problem, so a single rewrite fixes them all. */
|
|
287
|
+
export function describeProblems(problems, coverage) {
|
|
288
|
+
const lines = [];
|
|
289
|
+
if (coverage?.missing.length) {
|
|
290
|
+
lines.push(`rules with no test: ${coverage.missing.join(', ')} — write a test for each, or a "// cannot-test: <id> — <the state this area cannot set up>" line. "The locator is unknown" is not such a state (find it with kit.find by the doc's name), and neither is "needs two customers" (skip unless kit.distinctRunAccount())`);
|
|
291
|
+
}
|
|
292
|
+
for (const p of problems)
|
|
293
|
+
lines.push(`"${p.title}"${p.rules.length ? ` (${p.rules.join(', ')})` : ''} ${p.reason}`);
|
|
294
|
+
return lines.join('\n');
|
|
295
|
+
}
|
|
@@ -127,9 +127,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
127
127
|
` how to establish preconditions — follow it.`,
|
|
128
128
|
` - Assert what the line states, at the loosest form that still proves it.`,
|
|
129
129
|
` Add no condition it does not state, and no exact copy, ordering, id,`,
|
|
130
|
-
` timestamp or count it does not fix.
|
|
131
|
-
` without a crash"
|
|
132
|
-
`
|
|
130
|
+
` timestamp or count it does not fix. Only a line that says exactly "opens`,
|
|
131
|
+
` without a crash" is proved by the page loading with no error and no 5xx;`,
|
|
132
|
+
` every other line is proved by its own outcome, never by "no 5xx". "The customer is`,
|
|
133
133
|
` told why" is proved by a non-empty message VISIBLE where the app shows`,
|
|
134
134
|
` it, unless the line fixes the words. Check where the page actually`,
|
|
135
135
|
` renders a value before asserting it there.`,
|
|
@@ -252,10 +252,25 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
252
252
|
``,
|
|
253
253
|
`A LATER STEP OF A FLOW IS OPENED THROUGH THE TESTKIT. Never page.goto() a`,
|
|
254
254
|
`later step of a multi-step flow by its URL alone: call`,
|
|
255
|
-
`kit.openStep(page, step, input)
|
|
256
|
-
`
|
|
255
|
+
`kit.openStep(page, step, input) with a step name the TESTKIT lists for it`,
|
|
256
|
+
`(no other name exists), and skip with its reason when it does not resolve`,
|
|
257
|
+
`true. A screen after the last listed step is reached from that step through`,
|
|
258
|
+
`the UI. Only a test that checks the guard itself (what a visitor who`,
|
|
257
259
|
`skipped the earlier steps sees) opens the bare URL.`,
|
|
258
260
|
``,
|
|
261
|
+
`kit.find RETURNS A PROMISE. Always \`const pay = await kit.find(page, 'button',`,
|
|
262
|
+
`'Pay')\` first, then use pay. Never chain on kit.find(...) (.first(), .click()),`,
|
|
263
|
+
`never keep it in a variable without await, and never pass it to expect()`,
|
|
264
|
+
`without await.`,
|
|
265
|
+
``,
|
|
266
|
+
`kit.login PREPARES ONE PAGE. It sets the session for the page it is given and`,
|
|
267
|
+
`restores it when that page loads again. So:`,
|
|
268
|
+
` - log out through the app's own Log Out control, never by clearing storage or`,
|
|
269
|
+
` cookies (the restored session comes straight back);`,
|
|
270
|
+
` - a second tab or page is not logged in by it: call kit.login on that page`,
|
|
271
|
+
` too. "Logged in in another tab" — one tab's login reaching another — is not`,
|
|
272
|
+
` something the testkit can set up: such a rule gets a cannot-test line.`,
|
|
273
|
+
``,
|
|
259
274
|
`DATA A TEST CREATES. Other runs use the same accounts, so what they left`,
|
|
260
275
|
`behind is already there when this test starts.`,
|
|
261
276
|
` - A test that creates a record (a booking, an order, a vehicle, an`,
|
|
@@ -268,6 +283,10 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
268
283
|
` or a list being empty. "Exactly one booking for this request" is checked`,
|
|
269
284
|
` among the records that carry this test's unique value.`,
|
|
270
285
|
` - A test that only reads data uses the fixture role and the seeded ids.`,
|
|
286
|
+
` - Two DIFFERENT people at once (two customers paying for one slot): log one`,
|
|
287
|
+
` in as the fixture role and one as "run", and skip at the top unless`,
|
|
288
|
+
` kit.distinctRunAccount() is true (the environment has no second account).`,
|
|
289
|
+
` A TESTKIT without distinctRunAccount gives that rule a cannot-test line.`,
|
|
271
290
|
``,
|
|
272
291
|
`AT MOST ${maxPerRule()} TESTS FOR ONE CRITERION. Two tests whose oracle quotes the same`,
|
|
273
292
|
`criterion must reach different branches; more inputs for that criterion go`,
|
|
@@ -294,11 +313,36 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
294
313
|
` declarations. A precondition the environment lacks is a run-time skip`,
|
|
295
314
|
` inside a real test, stating what was observed.`,
|
|
296
315
|
``,
|
|
316
|
+
`EVERY RULE GETS A TEST THAT CAN FAIL. Each rule id this file is given gets`,
|
|
317
|
+
`at least one test, and that test fails when the product breaks the rule.`,
|
|
318
|
+
` - A rule whose STATE this area cannot set up — data that lives in another`,
|
|
319
|
+
` system, a backend-only effect the screen never shows, an external service`,
|
|
320
|
+
` it cannot drive — gets ONE line instead of a test:`,
|
|
321
|
+
` // cannot-test: R7 — <the state this area cannot set up>`,
|
|
322
|
+
` An element, locator or route you were not shown is NEVER that reason:`,
|
|
323
|
+
` find the element with kit.find(page, kind, <the name the doc gives it>),`,
|
|
324
|
+
` which needs nothing from the CODE FACTS. Two different people at once`,
|
|
325
|
+
` is not that reason either: see kit.distinctRunAccount() below.`,
|
|
326
|
+
` Never leave a rule out silently, and never write a test that passes`,
|
|
327
|
+
` whatever the product does in its place.`,
|
|
328
|
+
` - Assert the ONE outcome the rule expects. Never a regex or a condition`,
|
|
329
|
+
` that accepts the good and the bad outcome alike (/confirm|not available/),`,
|
|
330
|
+
` never the page's whole text (body or main innerText) compared with`,
|
|
331
|
+
` itself, and never an expect() only inside an if, else or catch.`,
|
|
332
|
+
` - A rule about a specific thing (the operator, the total, the time, the`,
|
|
333
|
+
` link's address) reads THAT element and compares its value: "the same`,
|
|
334
|
+
` operator" means the operator's name read before and after.`,
|
|
335
|
+
` - "No 5xx", "the page loads", "body is not empty", "a £ sign is shown",`,
|
|
336
|
+
` "the count of a text that never appears is 0" prove only a rule that says`,
|
|
337
|
+
` exactly that.`,
|
|
338
|
+
` - A stub's own words are never the assertion: assert what the app does`,
|
|
339
|
+
` with the stubbed state.`,
|
|
340
|
+
``,
|
|
297
341
|
`SELF-CHECK BEFORE YOU OUTPUT. Go through every test you wrote and name, for`,
|
|
298
|
-
`each line, its evidence under E1–E5, and the
|
|
299
|
-
`
|
|
300
|
-
`
|
|
301
|
-
`
|
|
342
|
+
`each line, its evidence under E1–E5, and the outcome that makes it fail. A`,
|
|
343
|
+
`line with no evidence is fixed; a test that cannot fail is rewritten, or`,
|
|
344
|
+
`its rule gets a cannot-test line. Then check every rule id you were given`,
|
|
345
|
+
`has a test or a cannot-test line.`,
|
|
302
346
|
``,
|
|
303
347
|
...(memory
|
|
304
348
|
? [
|
|
@@ -307,9 +351,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
307
351
|
`once for the whole journey, the TESTKIT list is the harness's real`,
|
|
308
352
|
`exports, and the JOURNEY block holds exactly the rules this file tests`,
|
|
309
353
|
`with their context. Those are the MECHANICS: HOW to reach and drive`,
|
|
310
|
-
`the behaviour. Never invent a
|
|
311
|
-
`appears nowhere below
|
|
312
|
-
`
|
|
354
|
+
`the behaviour. Never invent a test id, route, request field or helper`,
|
|
355
|
+
`that appears nowhere below. An element is found by the name the doc`,
|
|
356
|
+
`gives it, with kit.find. But the EXPECTED OUTCOME you`,
|
|
313
357
|
]
|
|
314
358
|
: grounding
|
|
315
359
|
? [
|
|
@@ -411,7 +455,11 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
411
455
|
`FIND EVERY ELEMENT WITH kit.find(page, kind, name): kind is what the doc`,
|
|
412
456
|
`says it is (button, link, input, dropdown, checkbox, radio, toggle, tab,`,
|
|
413
457
|
`menuitem, dialog, heading, alert, toast, image, list, table …), name is`,
|
|
414
|
-
`the text the
|
|
458
|
+
`the text the SCREEN shows for it — read it from the CODE FACTS (the`,
|
|
459
|
+
`component's text, label or aria-label); the doc's word only when the facts`,
|
|
460
|
+
`do not show that element ("help link" in the doc may read "FAQs" on the`,
|
|
461
|
+
`screen). Written out, never a value the test made up (kit.unique, a stub's`,
|
|
462
|
+
`id). It tries the proper role first and falls back`,
|
|
415
463
|
`to the label, placeholder, aria-label, alt text and visible text, adding`,
|
|
416
464
|
`a finding when the element is not built as what it is — so a test written`,
|
|
417
465
|
`before the feature still finds it once it ships. Use page.getBy* only for`,
|
|
@@ -430,8 +478,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
430
478
|
`ground on role/label/text/name instead of inventing a test-id.`,
|
|
431
479
|
`NEVER invent a data-testid/data-cy: an unverified test-id is`,
|
|
432
480
|
`REJECTED by the selector-grounding gate and wastes the whole file.`,
|
|
433
|
-
`
|
|
434
|
-
`
|
|
481
|
+
`An element the doc names is found with kit.find(page, kind, <that name>)`,
|
|
482
|
+
`even when the source shows no handle for it; never guess a test id.`,
|
|
435
483
|
`NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
|
|
436
484
|
`never a description/comment, and never a route-file placeholder`,
|
|
437
485
|
`like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
|
|
@@ -546,12 +594,12 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
546
594
|
` ${focus.title}`,
|
|
547
595
|
` ${focus.covers}`,
|
|
548
596
|
`Do NOT test the journey's other concerns here — separate files`,
|
|
549
|
-
`cover those. Go DEEP on this one concern (happy path + its own`,
|
|
550
|
-
`edge/negative/security cases) rather than broad-but-shallow.`,
|
|
597
|
+
`cover those. ${memory?.rules.length ? 'Test each rule below; add no case the rules do not state.' : 'Go DEEP on this one concern (happy path + its own edge/negative/security cases) rather than broad-but-shallow.'}`,
|
|
551
598
|
`Use this exact describe() title: "${focus.title}".`,
|
|
552
599
|
...(memory?.rules.length
|
|
553
600
|
? [
|
|
554
|
-
`Test exactly these rules: ${memory.rules.join(', ')}.
|
|
601
|
+
`Test exactly these rules: ${memory.rules.join(', ')}. EVERY one gets a test or a`,
|
|
602
|
+
`"// cannot-test: <id> — <why>" line. Name the rule id AND its`,
|
|
555
603
|
`tag in each test's oracle comment, e.g. "// oracle: ${memory.rules[0]} [spec] …":`,
|
|
556
604
|
`the tag is how a failure is later graded as a defect or a finding.`,
|
|
557
605
|
]
|
|
@@ -575,11 +623,15 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
575
623
|
`- calls page.waitForTimeout() or sleeps for a fixed time;`,
|
|
576
624
|
`- skips a test because an element is missing (test.skip(true, …), or a`,
|
|
577
625
|
` condition that counts or looks for an element) — that is a failure to assert;`,
|
|
578
|
-
`- has a test with no expect(),
|
|
579
|
-
|
|
626
|
+
`- has a test with no expect(), an expect() only inside an if/else/catch, or more`,
|
|
627
|
+
` than ${maxPerRule()} tests for one rule;`,
|
|
628
|
+
`- leaves a rule with neither a test nor a "// cannot-test: <id> — <why>" line;`,
|
|
629
|
+
`- asserts a regex that accepts the good and the bad outcome, compares the page's`,
|
|
630
|
+
` whole text, or asserts the words its own stub returns;`,
|
|
631
|
+
`- uses kit.find(...) without await, or a kit.openStep step the TESTKIT does not list;`,
|
|
580
632
|
`- names a button, link, field or label that is in neither the CODE FACTS nor the rules;`,
|
|
581
633
|
`- defines its own helper functions (use the TESTKIT) or comments anything but`,
|
|
582
|
-
` the one "// oracle: <rule id> [tag] …" line above each test.`,
|
|
634
|
+
` the one "// oracle: <rule id> [tag] …" line above each test and cannot-test lines.`,
|
|
583
635
|
`Keep each test short: the steps, then the expect()s. No logging.`,
|
|
584
636
|
``,
|
|
585
637
|
]
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — finish a written file: every rule tested, every test able to fail.
|
|
3
|
+
*
|
|
4
|
+
* A file can pass every file-level gate and still leave rules out, or hold a
|
|
5
|
+
* test that passes whatever the app does. Those tests are taken out, and the
|
|
6
|
+
* rules left without a test are written again in ONE small call that sees only
|
|
7
|
+
* those rules, the file's top and why the earlier tests were refused. What
|
|
8
|
+
* that call still gets wrong is removed and reported, never sent again: a
|
|
9
|
+
* missing test is honest, a test that cannot fail is not.
|
|
10
|
+
*/
|
|
11
|
+
import { factsSlice, journeySlice } from './journey-memory.js';
|
|
12
|
+
import { appendTests, findTestBlocks, spliceBlocks } from './test-blocks.js';
|
|
13
|
+
import { acceptedReason, describeProblems, ruleCoverage, testProblems } from '../gates/test-quality.js';
|
|
14
|
+
const review = (content, rules, steps, knownTexts) => ({
|
|
15
|
+
cov: ruleCoverage(content, rules),
|
|
16
|
+
problems: testProblems(content, { steps, knownTexts }),
|
|
17
|
+
});
|
|
18
|
+
/** The file without the tests that have a problem. */
|
|
19
|
+
function withoutBlocks(content, problems) {
|
|
20
|
+
const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
|
|
21
|
+
return blocks.length ? spliceBlocks(content, blocks.map((block) => ({ block, text: '' }))) : content;
|
|
22
|
+
}
|
|
23
|
+
/** Is the file complete as it is? */
|
|
24
|
+
export function fileIsComplete(content, rules, steps, knownTexts) {
|
|
25
|
+
const { cov, problems } = review(content, rules, steps, knownTexts);
|
|
26
|
+
return !cov.missing.length && !problems.length;
|
|
27
|
+
}
|
|
28
|
+
export async function completeFile(content, ctx) {
|
|
29
|
+
const notes = [];
|
|
30
|
+
const first = review(content, ctx.rules, ctx.steps, ctx.knownTexts);
|
|
31
|
+
let calls = 0;
|
|
32
|
+
let out = content;
|
|
33
|
+
const dropped = new Set();
|
|
34
|
+
if (first.cov.missing.length || first.problems.length) {
|
|
35
|
+
out = withoutBlocks(content, first.problems);
|
|
36
|
+
for (const p of first.problems)
|
|
37
|
+
dropped.add(p.title);
|
|
38
|
+
const left = ruleCoverage(out, ctx.rules).missing;
|
|
39
|
+
if (left.length) {
|
|
40
|
+
const blocks = findTestBlocks(out);
|
|
41
|
+
const head = out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000);
|
|
42
|
+
const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(out)?.[2] ?? ctx.journey.name;
|
|
43
|
+
const slice = journeySlice(ctx.memory, left);
|
|
44
|
+
calls = 1;
|
|
45
|
+
try {
|
|
46
|
+
const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(first.problems, first.cov));
|
|
47
|
+
const candidate = appendTests(out, tests);
|
|
48
|
+
const refused = ctx.check(candidate);
|
|
49
|
+
if (refused)
|
|
50
|
+
notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
|
|
51
|
+
else
|
|
52
|
+
out = candidate;
|
|
53
|
+
}
|
|
54
|
+
catch (e) {
|
|
55
|
+
notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
|
|
56
|
+
}
|
|
57
|
+
// What the second answer still gets wrong is removed, not sent again.
|
|
58
|
+
const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts });
|
|
59
|
+
if (second.length) {
|
|
60
|
+
out = withoutBlocks(out, second);
|
|
61
|
+
for (const p of second)
|
|
62
|
+
dropped.add(p.title);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// A cannot-test line that is no longer true (its rule has a test now) or never
|
|
67
|
+
// was a reason ("the locator is unknown") does not stay in the file.
|
|
68
|
+
const has = new Set(findTestBlocks(out).flatMap((b) => b.rules));
|
|
69
|
+
out = out.replace(/^[ \t]*\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)\n/gm, (line, ids, why) => ids.split(/\s*,\s*/).every((id) => has.has(id)) || !acceptedReason(why) ? '' : line);
|
|
70
|
+
const cov = ruleCoverage(out, ctx.rules);
|
|
71
|
+
const tested = [...has].filter((id) => ctx.rules.includes(id));
|
|
72
|
+
if (dropped.size)
|
|
73
|
+
notes.push(`took out ${dropped.size} test(s) that could not check their rule`);
|
|
74
|
+
if (cov.missing.length)
|
|
75
|
+
notes.push(`no test for ${cov.missing.join(', ')}`);
|
|
76
|
+
return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped], calls, notes };
|
|
77
|
+
}
|
|
@@ -49,7 +49,7 @@ function similarity(a, b) {
|
|
|
49
49
|
* strings is a rename; a close match is a change; the rest is added or
|
|
50
50
|
* removed. Pure: no AI, no I/O.
|
|
51
51
|
*/
|
|
52
|
-
export function alignRules(prev, next) {
|
|
52
|
+
export function alignRules(prev, next, retired = []) {
|
|
53
53
|
const diff = { same: [], changed: [], renamed: [], added: [], removed: [] };
|
|
54
54
|
const free = new Set(prev.map((r) => r.id));
|
|
55
55
|
const byText = new Map();
|
|
@@ -117,7 +117,8 @@ export function alignRules(prev, next) {
|
|
|
117
117
|
diff.changed.push(best.id);
|
|
118
118
|
}
|
|
119
119
|
// 4. Everything else is new, numbered after the highest id ever used.
|
|
120
|
-
|
|
120
|
+
// A removed rule's id is never given to a new one: old results still name it.
|
|
121
|
+
let top = Math.max(0, ...prev.map((r) => Number(r.id.slice(1)) || 0), ...retired.map((id) => Number(id.slice(1)) || 0));
|
|
121
122
|
for (const r of next.rules) {
|
|
122
123
|
if (id.has(r))
|
|
123
124
|
continue;
|
|
@@ -43,9 +43,13 @@ export const HARNESS_CONTRACT = [
|
|
|
43
43
|
` - created(kind, id, cleanup?, role?): void record what the run made (H8).`,
|
|
44
44
|
` - the "run" role: token('run'), ensureToken('run'), login(page, 'run'),`,
|
|
45
45
|
` apiContext(request, 'run') — the account tests that CREATE data use (H8).`,
|
|
46
|
+
` - distinctRunAccount(): boolean true only when the "run" role is a`,
|
|
47
|
+
` different account from the fixture role (H8), so a test that needs two`,
|
|
48
|
+
` different people at once can tell whether it has them.`,
|
|
46
49
|
`Plus, for Playwright suites:`,
|
|
47
50
|
` - openStep(page, step, input?): Promise<true | string> open a later step`,
|
|
48
51
|
` of a multi-step flow with what the earlier steps would have left (H9).`,
|
|
52
|
+
` - STEPS: object the step names openStep accepts (H9).`,
|
|
49
53
|
` - settledClick, fillFrameInputs, watchDialogs: re-exported unchanged from`,
|
|
50
54
|
` require('@activate-core/test-forge/reporters/ui-helpers.cjs') — never`,
|
|
51
55
|
` rewritten here.`,
|
|
@@ -170,7 +174,9 @@ export const HARNESS_CONTRACT = [
|
|
|
170
174
|
` signup — create a fresh account through the product's own sign-up, only`,
|
|
171
175
|
` when this environment can complete it (no code it cannot read and`,
|
|
172
176
|
` cannot bypass under H7); otherwise fall back to fixture and say so once.`,
|
|
173
|
-
` Mint and cache its token like any role
|
|
177
|
+
` Mint and cache its token like any role. distinctRunAccount() returns true`,
|
|
178
|
+
` only in pool or signup mode, when the run account really differs from`,
|
|
179
|
+
` the fixture one.`,
|
|
174
180
|
`H9. STEPS OF A FLOW. A later step of a multi-step flow (onboarding, a`,
|
|
175
181
|
` booking or checkout wizard, a sign-up in stages) depends on what the`,
|
|
176
182
|
` earlier steps stored — a choice, an address, a cart, a plan — in URL`,
|
|
@@ -185,8 +191,14 @@ export const HARNESS_CONTRACT = [
|
|
|
185
191
|
` - It waits until the step is really shown and resolves true; when the`,
|
|
186
192
|
` product still sends the visitor elsewhere it resolves a string naming`,
|
|
187
193
|
` the step it landed on, so the test can skip with that reason.`,
|
|
188
|
-
` - Offer
|
|
189
|
-
`
|
|
194
|
+
` - Offer EVERY step the flow has, up to and including its last page before`,
|
|
195
|
+
` the result — the summary, review or checkout page where the visitor`,
|
|
196
|
+
` confirms and pays — reaching a page that follows a choice (an operator,`,
|
|
197
|
+
` a plan, a slot) by making that choice through the UI, as a visitor would.`,
|
|
198
|
+
` Keep the names in one exported object STEPS (name → what identifies the`,
|
|
199
|
+
` page), and look the step up in it: an unknown name resolves`,
|
|
200
|
+
` 'unknown step "<name>" (steps: …)'. When the project has no such flow,`,
|
|
201
|
+
` export STEPS = {} and openStep resolving 'this product has no multi-step flow'.`,
|
|
190
202
|
].join('\n');
|
|
191
203
|
/**
|
|
192
204
|
* Stamp written into every generated testkit: the contract it was built
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* repair was most of 9.6M tokens; the code below needs no AI at all.
|
|
10
10
|
*/
|
|
11
11
|
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { harnessSteps } from '../gates/test-quality.js';
|
|
12
13
|
import { dirname, join } from 'node:path';
|
|
13
14
|
import { journeyDir } from '../paths.js';
|
|
14
15
|
/** Sections that are indexes or open questions, never something to test. */
|
|
@@ -73,9 +74,10 @@ export function readRules(markdown) {
|
|
|
73
74
|
continue;
|
|
74
75
|
}
|
|
75
76
|
if (mode === 'controls') {
|
|
76
|
-
|
|
77
|
+
// "| C3 |" or a plain "| 3 |": both are control C3.
|
|
78
|
+
const row = /^\|\s*C?(\d+[a-z]?)\s*\|(.*)\|\s*$/.exec(line);
|
|
77
79
|
if (row)
|
|
78
|
-
controls[row[1]] = row[2].split('|').map((c) => c.trim()).filter(Boolean).join(' · ');
|
|
80
|
+
controls[`C${row[1]}`] = row[2].split('|').map((c) => c.trim()).filter(Boolean).join(' · ');
|
|
79
81
|
continue;
|
|
80
82
|
}
|
|
81
83
|
if (mode !== 'rules')
|
|
@@ -147,6 +149,13 @@ function sliceText(memory, own, context, controlIds) {
|
|
|
147
149
|
* the file, not asked of the AI, so every writer gets the same true list
|
|
148
150
|
* without opening the testkit itself.
|
|
149
151
|
*/
|
|
152
|
+
/** TestForge's own runtime helpers, as a testkit re-exports them (find: uiHelpers.find). */
|
|
153
|
+
const RUNTIME_HELPERS = {
|
|
154
|
+
find: "async find(page, kind, name, { exact, timeout, within }) → Promise<Locator> — ALWAYS await it before use: `const b = await kit.find(page, 'button', 'Pay')`; never kit.find(...).first() and never expect(kit.find(...)). kind: button, link, input, dropdown, checkbox, radio, toggle, tab, menuitem, dialog, heading, alert, toast, image …; name: the text the doc gives it (or a RegExp).",
|
|
155
|
+
settledClick: 'async settledClick(locator, { settleMs, tries }) — clicks once the element has stopped moving (pop-ups, growing forms).',
|
|
156
|
+
fillFrameInputs: 'async fillFrameInputs(page, framePrefix, { name: value }) — fills inputs inside an embedded iframe (card fields).',
|
|
157
|
+
watchDialogs: 'watchDialogs(page) → string[] — accepts browser alerts and records their messages.',
|
|
158
|
+
};
|
|
150
159
|
export function helperApi(harnessSource) {
|
|
151
160
|
const names = new Set();
|
|
152
161
|
const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(harnessSource);
|
|
@@ -166,10 +175,16 @@ export function helperApi(harnessSource) {
|
|
|
166
175
|
}
|
|
167
176
|
const lines = harnessSource.split('\n');
|
|
168
177
|
const out = [];
|
|
178
|
+
const steps = harnessSteps(harnessSource);
|
|
169
179
|
for (const name of names) {
|
|
170
180
|
const at = lines.findIndex((l) => new RegExp(`^\\s*(?:export\\s+)?(?:async\\s+)?(?:function\\*?\\s+${name}\\s*\\(|(?:const|let|var)\\s+${name}\\s*=)`).test(l));
|
|
171
181
|
if (at === -1) {
|
|
172
|
-
|
|
182
|
+
// Re-exported from TestForge's runtime (find: uiHelpers.find): its contract, not just its name.
|
|
183
|
+
out.push(RUNTIME_HELPERS[name] ? `- ${RUNTIME_HELPERS[name]}` : `- ${name}`);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (name === 'openStep' && steps.length) {
|
|
187
|
+
out.push(`- ${lines[at].trim().replace(/^export\s+/, '').replace(/\s*\{\s*\}?\s*$/, '')} — step is EXACTLY one of: ${steps.join(', ')}. Any other name does not exist. Resolves true when the step is open, else a string saying why.`);
|
|
173
188
|
continue;
|
|
174
189
|
}
|
|
175
190
|
let sig = lines[at].trim().replace(/^export\s+/, '').replace(/\s*\{\s*\}?\s*$/, '');
|
|
@@ -278,6 +293,60 @@ export function saveMemory(journey, docHash, memory, helpers, extra) {
|
|
|
278
293
|
}
|
|
279
294
|
return path;
|
|
280
295
|
}
|
|
296
|
+
/**
|
|
297
|
+
* Put what the files really test into the journey's memory and its plan
|
|
298
|
+
* check: which file tests each rule, which rules cannot be tested here and
|
|
299
|
+
* why, and which have no test. Returns the one-line summary.
|
|
300
|
+
*/
|
|
301
|
+
export function recordWritten(journey, files) {
|
|
302
|
+
const dir = journeyDir(journey);
|
|
303
|
+
const path = join(dir, 'rules.json');
|
|
304
|
+
const testedBy = new Map();
|
|
305
|
+
const why = new Map();
|
|
306
|
+
const missing = new Set();
|
|
307
|
+
for (const f of files) {
|
|
308
|
+
for (const id of f.tested)
|
|
309
|
+
testedBy.set(id, [...(testedBy.get(id) ?? []), f.slug]);
|
|
310
|
+
for (const c of f.cannotTest)
|
|
311
|
+
why.set(c.id, `cannot be tested here: ${c.why}`);
|
|
312
|
+
for (const id of f.missing)
|
|
313
|
+
missing.add(id);
|
|
314
|
+
}
|
|
315
|
+
const planned = new Set(files.flatMap((f) => [...f.tested, ...f.cannotTest.map((c) => c.id), ...f.missing]));
|
|
316
|
+
try {
|
|
317
|
+
const record = JSON.parse(readFileSync(path, 'utf-8'));
|
|
318
|
+
for (const r of record.rules) {
|
|
319
|
+
if (!planned.has(r.id))
|
|
320
|
+
continue;
|
|
321
|
+
r.testedBy = testedBy.get(r.id) ?? [];
|
|
322
|
+
r.skipped = why.get(r.id) ?? (missing.has(r.id) ? 'no test was written for it' : undefined);
|
|
323
|
+
}
|
|
324
|
+
writeFileSync(path, JSON.stringify(record, null, 2), 'utf-8');
|
|
325
|
+
}
|
|
326
|
+
catch {
|
|
327
|
+
// no memory file: the summary below still reports it
|
|
328
|
+
}
|
|
329
|
+
const dropped = files.reduce((n, f) => n + f.dropped.length, 0);
|
|
330
|
+
const lines = [
|
|
331
|
+
`${testedBy.size}/${planned.size} planned rules have a test`,
|
|
332
|
+
why.size ? `${why.size} cannot be tested here (${[...why.keys()].join(', ')})` : '',
|
|
333
|
+
missing.size ? `NO TEST for ${[...missing].join(', ')}` : '',
|
|
334
|
+
dropped ? `${dropped} test(s) taken out because they could not check their rule` : '',
|
|
335
|
+
].filter(Boolean);
|
|
336
|
+
const check = join(dir, 'plan-check.md');
|
|
337
|
+
try {
|
|
338
|
+
const before = existsSync(check) ? readFileSync(check, 'utf-8').replace(/\n## Written\n[\s\S]*$/, '') : '';
|
|
339
|
+
const detail = [
|
|
340
|
+
...[...why].map(([id, w]) => `- ${id}: ${w}`),
|
|
341
|
+
...[...missing].map((id) => `- ${id}: no test was written for it`),
|
|
342
|
+
];
|
|
343
|
+
writeFileSync(check, `${before.trimEnd()}\n\n## Written\n\n${lines.join('; ')}.\n${detail.length ? `\n${detail.join('\n')}\n` : ''}`, 'utf-8');
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
// best-effort
|
|
347
|
+
}
|
|
348
|
+
return lines.join('; ');
|
|
349
|
+
}
|
|
281
350
|
/**
|
|
282
351
|
* Rules of this journey that another journey already tests, found without the
|
|
283
352
|
* AI: the same tag and nearly the same words (≥ 85 % of the words shared).
|
|
@@ -26,15 +26,18 @@ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, is
|
|
|
26
26
|
import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
|
-
import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, } from './ai-generator.js';
|
|
29
|
+
import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
|
+
import { completeFile, fileIsComplete } from './complete-file.js';
|
|
31
|
+
import { harnessSteps } from '../gates/test-quality.js';
|
|
30
32
|
import { normalisePending } from './pending.js';
|
|
31
33
|
import { packageRoot } from '../run/run-tests.js';
|
|
34
|
+
import { pendingReason, withPendingStamp } from '../run/readiness.js';
|
|
32
35
|
import { groundJourney } from '../docs/grounding.js';
|
|
33
36
|
import { generateSeed } from './seed-generator.js';
|
|
34
37
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
35
38
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
36
39
|
import { testedTags } from './suite-planner.js';
|
|
37
|
-
import { factsSlice, helperApi, journeySlice, readRules, saveMemory, testedElsewhere } from './journey-memory.js';
|
|
40
|
+
import { factsSlice, helperApi, journeySlice, readRules, recordWritten, saveMemory, testedElsewhere } from './journey-memory.js';
|
|
38
41
|
import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
39
42
|
import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
|
|
40
43
|
import { updateJourney } from './update-journey.js';
|
|
@@ -424,19 +427,36 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
424
427
|
// What the app can show: its source and the doc. A fresh test's names and
|
|
425
428
|
// labels are checked against it before the test is run.
|
|
426
429
|
let knownTexts;
|
|
430
|
+
const appTexts = () => framework === 'playwright'
|
|
431
|
+
? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
432
|
+
: undefined;
|
|
427
433
|
const genGate = (content, filePath) => testGate(content) ??
|
|
428
434
|
checkTestGates(content, ext, {
|
|
429
435
|
framework, forbidPlaceholders: true, filePath,
|
|
430
436
|
maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 2,
|
|
431
|
-
knownTexts:
|
|
432
|
-
? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
433
|
-
: undefined,
|
|
437
|
+
knownTexts: appTexts(),
|
|
434
438
|
});
|
|
435
439
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
436
440
|
// to retry (a resume pass fills those in without touching real files).
|
|
441
|
+
// A suite whose tests leave rules out or cannot fail is not done either:
|
|
442
|
+
// it goes on, and only those tests are written again.
|
|
443
|
+
const suiteComplete = () => {
|
|
444
|
+
const plan = readPlan(journey);
|
|
445
|
+
if (!plan?.some((f) => f.rules?.length))
|
|
446
|
+
return true;
|
|
447
|
+
const kit = harnessInfo(areaDir, framework, ext).path;
|
|
448
|
+
const steps = existsSync(kit) ? harnessSteps(readFileSync(kit, 'utf-8')) : [];
|
|
449
|
+
return plan.every((f) => {
|
|
450
|
+
const p = join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`);
|
|
451
|
+
if (!f.rules?.length || !existsSync(p))
|
|
452
|
+
return true;
|
|
453
|
+
return fileIsComplete(readFileSync(p, 'utf-8'), f.rules, steps, appTexts());
|
|
454
|
+
});
|
|
455
|
+
};
|
|
437
456
|
if (!force &&
|
|
438
457
|
cache[file] === hash &&
|
|
439
|
-
!hasDeferredStubs(testDir, ext)
|
|
458
|
+
!hasDeferredStubs(testDir, ext) &&
|
|
459
|
+
suiteComplete()) {
|
|
440
460
|
results.push({
|
|
441
461
|
journeyName: journey.name,
|
|
442
462
|
testDir,
|
|
@@ -452,8 +472,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
452
472
|
// A feature written up before it is built: its tests come from the doc
|
|
453
473
|
// alone. There is no code to read for facts and no app to run them on.
|
|
454
474
|
const notBuilt = /^status:\s*not built/im.test(markdown);
|
|
455
|
-
|
|
456
|
-
|
|
475
|
+
// Not live yet (not built, or behind a flag): every file is stamped, and
|
|
476
|
+
// `testforge run` holds it back until the doc's Status line goes.
|
|
477
|
+
const pending = pendingReason(markdown);
|
|
478
|
+
if (pending)
|
|
479
|
+
warnings.push(`${pending}: its files are held back by \`testforge run\` until the doc's Status line is removed`);
|
|
457
480
|
const collision = collisionWarning(areaDir, journey.id, ext);
|
|
458
481
|
if (collision)
|
|
459
482
|
warnings.push(collision);
|
|
@@ -493,17 +516,42 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
493
516
|
const tested = testedTags();
|
|
494
517
|
let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
|
|
495
518
|
const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
|
|
519
|
+
const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
|
|
520
|
+
// What each file really tests once written: the memory and the report come from this, not the plan.
|
|
521
|
+
const written = [];
|
|
522
|
+
const finish = async (content, pf, filePath) => {
|
|
523
|
+
if (!memory || !pf.rules?.length)
|
|
524
|
+
return content;
|
|
525
|
+
const { mem, facts } = await fileMemory(pf);
|
|
526
|
+
const done = await completeFile(content, {
|
|
527
|
+
journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
|
|
528
|
+
harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
|
|
529
|
+
check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
|
|
530
|
+
});
|
|
531
|
+
written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
|
|
532
|
+
for (const n of done.notes)
|
|
533
|
+
warnings.push(`${pf.slug}: ${n}`);
|
|
534
|
+
return done.content;
|
|
535
|
+
};
|
|
496
536
|
// A doc that changed since its suite was written: only what changed is
|
|
497
537
|
// touched. Renames and removals are edits without AI; reworded and added
|
|
498
538
|
// rules get one small call each file; new sections become new files below.
|
|
499
539
|
const prevRecord = memory ? loadMemory(journey) : undefined;
|
|
500
|
-
|
|
501
|
-
|
|
540
|
+
// Rule ids always follow the last written version, even for --force and
|
|
541
|
+
// --plan-only: an id that moved would point the tests' oracle comments at
|
|
542
|
+
// another rule. Only the in-place update of the tests is skipped then.
|
|
502
543
|
let changes;
|
|
503
|
-
|
|
504
|
-
|
|
544
|
+
let docDiff;
|
|
545
|
+
if (memory && prevRecord && prevRecord.hash !== hash) {
|
|
546
|
+
const { memory: aligned, diff } = alignRules(prevRecord.rules, memory, (prevRecord.removed ?? []).map((r) => r.id));
|
|
505
547
|
memory = aligned;
|
|
548
|
+
docDiff = diff;
|
|
506
549
|
changes = { renamed: diff.renamed.map((r) => r.id), changed: diff.changed, removed: diff.removed };
|
|
550
|
+
}
|
|
551
|
+
const prevPlan = docDiff && !force && !opts.planOnly ? readPlan(journey) : undefined;
|
|
552
|
+
if (memory && docDiff && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
|
|
553
|
+
const aligned = memory;
|
|
554
|
+
const diff = docDiff;
|
|
507
555
|
console.log(` ${journey.id}: doc changed — ${describeDiff(diff)}`);
|
|
508
556
|
const facts = notBuilt ? undefined : (await groundJourney(journey, hash, 'src', markdown, true)) ?? undefined;
|
|
509
557
|
const up = await updateJourney({
|
|
@@ -519,7 +567,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
519
567
|
const planned = useAI
|
|
520
568
|
? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
|
|
521
569
|
: [{ slug: journey.id, title: journey.name, covers: '' }];
|
|
522
|
-
|
|
570
|
+
// A plan-only run leaves the memory as the tests were last written, so the
|
|
571
|
+
// next generate still sees what changed in the doc and updates those tests.
|
|
572
|
+
if (memory && !opts.planOnly) {
|
|
523
573
|
const choice = loadSourceChoice();
|
|
524
574
|
saveMemory(journey, hash, memory, helpers, {
|
|
525
575
|
doc: file, reference: choice?.journeys?.[file] ?? choice?.reference,
|
|
@@ -593,8 +643,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
593
643
|
if (useAI && memory && perCall > 1) {
|
|
594
644
|
const toWrite = planned.filter((pf) => !keeps(pathOf(pf)));
|
|
595
645
|
const batches = [];
|
|
596
|
-
|
|
597
|
-
|
|
646
|
+
// A few files to one call, but never so many rules that the writer drops
|
|
647
|
+
// some: 21 rules in one answer left 14 of them without a test.
|
|
648
|
+
const maxRules = 12;
|
|
649
|
+
for (const pf of toWrite) {
|
|
650
|
+
const last = batches[batches.length - 1];
|
|
651
|
+
const rules = (b) => b.reduce((n, f) => n + (f.rules?.length ?? 0), 0);
|
|
652
|
+
if (last && last.length < perCall && rules(last) + (pf.rules?.length ?? 0) <= maxRules)
|
|
653
|
+
last.push(pf);
|
|
654
|
+
else
|
|
655
|
+
batches.push([pf]);
|
|
656
|
+
}
|
|
598
657
|
const writeBatch = async (batch) => {
|
|
599
658
|
const files = await Promise.all(batch.map(async (pf) => {
|
|
600
659
|
const { mem, facts } = await fileMemory(pf);
|
|
@@ -637,8 +696,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
637
696
|
// A suite written before pending tests carried their reason gets
|
|
638
697
|
// it now, deterministically — no regeneration, no AI.
|
|
639
698
|
const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
|
|
640
|
-
|
|
641
|
-
|
|
699
|
+
// A kept file that leaves rules out or holds tests that cannot fail:
|
|
700
|
+
// only those tests are written again.
|
|
701
|
+
const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath) : normalised, pending);
|
|
702
|
+
if (finished !== existing)
|
|
703
|
+
writeFileSync(filePath, finished, 'utf-8');
|
|
642
704
|
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
643
705
|
}
|
|
644
706
|
}
|
|
@@ -654,7 +716,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
654
716
|
const { mem, facts } = await fileMemory(pf);
|
|
655
717
|
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
|
|
656
718
|
if (ai !== null) {
|
|
657
|
-
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
719
|
+
content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
|
|
658
720
|
status = 'ai';
|
|
659
721
|
}
|
|
660
722
|
else {
|
|
@@ -673,6 +735,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
673
735
|
if (!content.includes(GEN_MARKER)) {
|
|
674
736
|
content = `// ${GEN_MARKER}\n${content}`;
|
|
675
737
|
}
|
|
738
|
+
content = withPendingStamp(content, pending);
|
|
676
739
|
// Write IMMEDIATELY (not batched at the end of the journey), so an
|
|
677
740
|
// interruption preserves every file generated so far.
|
|
678
741
|
// keepDeferredStubs:false (config) — do NOT write a DEFERRED stub file;
|
|
@@ -685,6 +748,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
685
748
|
}
|
|
686
749
|
return { slug: pf.slug, path: filePath, status };
|
|
687
750
|
});
|
|
751
|
+
if (memory && written.length) {
|
|
752
|
+
const sum = recordWritten(journey, written);
|
|
753
|
+
console.log(` ${journey.id}: ${sum}`);
|
|
754
|
+
}
|
|
688
755
|
const testFiles = generated.map((g) => ({
|
|
689
756
|
slug: g.slug,
|
|
690
757
|
path: g.path,
|
|
@@ -74,6 +74,8 @@ export async function updateJourney(ctx) {
|
|
|
74
74
|
const base = old.length ? spliceBlocks(content, old.map((block) => ({ block, text: '' }))) : content;
|
|
75
75
|
let refusal;
|
|
76
76
|
let next = null;
|
|
77
|
+
// Only the new tests are judged: a problem the file already had is not theirs.
|
|
78
|
+
const already = ctx.check(base, path);
|
|
77
79
|
for (let attempt = 0; attempt < 2 && next === null; attempt++) {
|
|
78
80
|
let tests = '';
|
|
79
81
|
try {
|
|
@@ -85,7 +87,8 @@ export async function updateJourney(ctx) {
|
|
|
85
87
|
break;
|
|
86
88
|
}
|
|
87
89
|
const candidate = appendTests(base, tests);
|
|
88
|
-
|
|
90
|
+
const why = findTestBlocks(tests).length ? ctx.check(candidate, path) : 'it contained no test(...) block';
|
|
91
|
+
refusal = why && why !== already ? why : undefined;
|
|
89
92
|
if (!refusal)
|
|
90
93
|
next = candidate;
|
|
91
94
|
}
|
|
@@ -77,7 +77,12 @@ function hasPage(path, haystack, appRoots) {
|
|
|
77
77
|
export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
|
|
78
78
|
let texts;
|
|
79
79
|
return files.map((file) => {
|
|
80
|
-
const
|
|
80
|
+
const content = readFileSync(file, 'utf-8');
|
|
81
|
+
// The doc says the feature is not live: held back whatever the code has.
|
|
82
|
+
const pending = PENDING_STAMP.exec(content)?.[1];
|
|
83
|
+
if (pending)
|
|
84
|
+
return { file, state: 'not-built', missing: [pending.trim()], renamed: [] };
|
|
85
|
+
const { names, pages } = namesInTest(content);
|
|
81
86
|
const missing = [];
|
|
82
87
|
const renamed = [];
|
|
83
88
|
for (const n of names) {
|
|
@@ -100,6 +105,28 @@ export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
|
|
|
100
105
|
return { file, state, missing, renamed };
|
|
101
106
|
});
|
|
102
107
|
}
|
|
108
|
+
/** The line that holds back a file whose feature is not live yet. */
|
|
109
|
+
export const PENDING_STAMP = /^\/\/\s*testforge-pending:\s*(.+)$/m;
|
|
110
|
+
/**
|
|
111
|
+
* The status a journey doc gives its feature: "Status: not built", or
|
|
112
|
+
* "Status: behind flag <name>" for code that ships switched off. Undefined
|
|
113
|
+
* when the feature is live.
|
|
114
|
+
*/
|
|
115
|
+
export function pendingReason(markdown) {
|
|
116
|
+
const m = /^status:\s*(not built(?: yet)?|behind (?:a )?flag\b[^\n]*)$/im.exec(markdown);
|
|
117
|
+
if (!m)
|
|
118
|
+
return undefined;
|
|
119
|
+
return /^not built/i.test(m[1]) ? 'the feature is not built yet' : `the feature is ${m[1].trim().replace(/^behind (?:a )?flag\s*/i, 'behind the flag ').trim()} — run it once that is switched on`;
|
|
120
|
+
}
|
|
121
|
+
/** The file with the pending stamp set to `reason`, or removed when there is none. */
|
|
122
|
+
export function withPendingStamp(content, reason) {
|
|
123
|
+
const without = content.replace(/^\/\/\s*testforge-pending:.*\n/m, '');
|
|
124
|
+
if (!reason)
|
|
125
|
+
return without;
|
|
126
|
+
const marker = /^\/\/ @generated by testforge.*\n/m.exec(without);
|
|
127
|
+
const at = marker ? marker.index + marker[0].length : 0;
|
|
128
|
+
return `${without.slice(0, at)}// testforge-pending: ${reason}\n${without.slice(at)}`;
|
|
129
|
+
}
|
|
103
130
|
/** The test files under a directory. */
|
|
104
131
|
export function testFilesIn(dir) {
|
|
105
132
|
const out = [];
|
|
@@ -140,7 +140,7 @@ export function runTests(opts = {}) {
|
|
|
140
140
|
const states = checkReadiness(files, sourceText(roots), () => [], [root]);
|
|
141
141
|
const held = states.filter((s) => s.state === 'not-built');
|
|
142
142
|
if (held.length) {
|
|
143
|
-
console.log(`\n${held.length} file(s) pending —
|
|
143
|
+
console.log(`\n${held.length} file(s) pending — their feature is not live yet (\`testforge bind\` shows why; --all runs them anyway):`);
|
|
144
144
|
for (const h of held.slice(0, 15))
|
|
145
145
|
console.log(` not built ${relative(root, h.file)}: ${h.missing.slice(0, 3).map((m) => `"${m}"`).join(', ')}`);
|
|
146
146
|
targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
|
|
@@ -164,6 +164,8 @@ export function runTests(opts = {}) {
|
|
|
164
164
|
console.log('\nNothing to run — generate tests first (`testforge generate`).');
|
|
165
165
|
return 1;
|
|
166
166
|
}
|
|
167
|
-
|
|
167
|
+
// The runner's own summary above says how many ran and how many skipped; a run where
|
|
168
|
+
// everything skipped has no failure, but it is not a pass.
|
|
169
|
+
console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
|
|
168
170
|
return anyFail ? 1 : 0;
|
|
169
171
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|