@activate-core/test-forge 1.3.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +70 -29
- package/dist/core/docs/excerpts.js +294 -0
- package/dist/core/docs/grounding.js +55 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/test-quality.js +295 -0
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +267 -117
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/complete-file.js +77 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +140 -0
- package/dist/core/generate/harness.js +18 -34
- package/dist/core/generate/journey-memory.js +394 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +240 -82
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +118 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +146 -0
- package/dist/core/run/run-tests.js +26 -2
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
9
9
|
import { basename, extname, join } from 'node:path';
|
|
10
|
-
import {
|
|
11
|
-
export const TICKETS_REFERENCE = join(
|
|
10
|
+
import { REFERENCE_DIR } from '../paths.js';
|
|
11
|
+
export const TICKETS_REFERENCE = join(REFERENCE_DIR, 'tickets.md');
|
|
12
12
|
const TEXT = /\.(md|markdown|txt|text|json|csv|html?|xml|ya?ml)$/i;
|
|
13
13
|
const ID = /\b([A-Z][A-Z0-9]+-\d+)\b/;
|
|
14
14
|
const MAX_CHARS = 20_000;
|
|
@@ -50,7 +50,7 @@ export function bundleTickets(folder) {
|
|
|
50
50
|
'',
|
|
51
51
|
...tickets.flatMap((t) => [`## ${t.id} — ${basename(t.file)}`, '', t.content.trim().slice(0, MAX_CHARS), '']),
|
|
52
52
|
].join('\n');
|
|
53
|
-
mkdirSync(
|
|
53
|
+
mkdirSync(REFERENCE_DIR, { recursive: true });
|
|
54
54
|
writeFileSync(TICKETS_REFERENCE, body, 'utf-8');
|
|
55
55
|
return { path: TICKETS_REFERENCE, ids: tickets.map((t) => t.id) };
|
|
56
56
|
}
|
|
@@ -13,7 +13,7 @@ import { COVERAGE_PATH, STATE_DIR } from '../paths.js';
|
|
|
13
13
|
* F8 — write an honest coverage manifest. For every generated file it
|
|
14
14
|
* records real (has assertions) vs deferred (pending only), so a "green"
|
|
15
15
|
* run is never mistaken for full coverage. Written to
|
|
16
|
-
*
|
|
16
|
+
* testforge/coverage.json and summarised to the console.
|
|
17
17
|
*/
|
|
18
18
|
export function writeCoverageManifest(results) {
|
|
19
19
|
const journeys = results
|
|
@@ -38,10 +38,6 @@ export function writeCoverageManifest(results) {
|
|
|
38
38
|
journey: r.journeyName,
|
|
39
39
|
real: files.filter((f) => f.real).length,
|
|
40
40
|
deferred: files.filter((f) => !f.real).length,
|
|
41
|
-
// Whether these files were RUN and passed. A file can assert plenty
|
|
42
|
-
// and still target something the app does not have, so "real" alone
|
|
43
|
-
// was never enough to call the coverage honest.
|
|
44
|
-
verified: r.verified,
|
|
45
41
|
tiers,
|
|
46
42
|
files,
|
|
47
43
|
};
|
|
@@ -55,9 +51,6 @@ export function writeCoverageManifest(results) {
|
|
|
55
51
|
writeFileSync(COVERAGE_PATH, JSON.stringify({
|
|
56
52
|
real: totalReal,
|
|
57
53
|
deferred: totalDeferred,
|
|
58
|
-
verified: journeys.filter((j) => j.verified === true).length,
|
|
59
|
-
failedVerify: journeys.filter((j) => j.verified === false).length,
|
|
60
|
-
notVerified: journeys.filter((j) => j.verified === undefined).length,
|
|
61
54
|
journeys,
|
|
62
55
|
}, null, 2), 'utf-8');
|
|
63
56
|
}
|
|
@@ -67,14 +60,6 @@ export function writeCoverageManifest(results) {
|
|
|
67
60
|
console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see ${COVERAGE_PATH}`);
|
|
68
61
|
const t = journeys.reduce((a, j) => ({ spec: a.spec + j.tiers.spec, baseline: a.baseline + j.tiers.baseline, observed: a.observed + j.tiers.observed, untagged: a.untagged + j.tiers.untagged }), { spec: 0, baseline: 0, observed: 0, untagged: 0 });
|
|
69
62
|
console.log(`Tests by the rule they check: ${t.spec} client requirement [spec], ${t.baseline} robustness [baseline], ${t.observed} current behaviour [observed]${t.untagged ? `, ${t.untagged} with no tagged rule` : ''}.`);
|
|
70
|
-
const ranGreen = journeys.filter((j) => j.verified === true).length;
|
|
71
|
-
const ranRed = journeys.filter((j) => j.verified === false).length;
|
|
72
|
-
const unrun = journeys.filter((j) => j.verified === undefined).length;
|
|
73
|
-
if (ranGreen || ranRed || unrun) {
|
|
74
|
-
console.log(`Verified: ${ranGreen} journey(s) ran green` +
|
|
75
|
-
(ranRed ? `, ${ranRed} still failing` : '') +
|
|
76
|
-
(unrun ? `, ${unrun} not run` : ''));
|
|
77
|
-
}
|
|
78
63
|
if (totalDeferred) {
|
|
79
64
|
console.log(` (deferred files are pending, NOT real coverage — regenerate or implement them.)`);
|
|
80
65
|
}
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — does each written test really check its rule?
|
|
3
|
+
*
|
|
4
|
+
* The file-level gates see a file as text. These checks read each test on its
|
|
5
|
+
* own, against the rules the file was planned for:
|
|
6
|
+
*
|
|
7
|
+
* - every planned rule has a test, or a "// cannot-test: R7 — <what is
|
|
8
|
+
* missing>" line that says why not (Q1);
|
|
9
|
+
* - each test has an assertion that runs, not one only inside an if or a
|
|
10
|
+
* catch (Q2);
|
|
11
|
+
* - no assertion accepts the rule's success and its failure alike, like
|
|
12
|
+
* /confirm|time not available/ (Q3);
|
|
13
|
+
* - no test compares the page's whole text with itself (Q4);
|
|
14
|
+
* - a test does more than assert that something is absent, unless its rule
|
|
15
|
+
* says never / no / not (Q5);
|
|
16
|
+
* - kit.find is awaited: it returns a promise (Q6);
|
|
17
|
+
* - kit.openStep names a step the testkit has (Q7);
|
|
18
|
+
* - a run-time skip does not depend on what the page shows (Q8).
|
|
19
|
+
*
|
|
20
|
+
* Each problem names its test and its rules, so only those tests are written
|
|
21
|
+
* again; the rest of the file is kept.
|
|
22
|
+
*/
|
|
23
|
+
import { matchParen } from './validate.js';
|
|
24
|
+
import { findTestBlocks } from '../generate/test-blocks.js';
|
|
25
|
+
const CANNOT = /\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)$/gm;
|
|
26
|
+
/** Which planned rules have a test, which are explained, which are simply missing. */
|
|
27
|
+
/**
|
|
28
|
+
* Reasons that are not a state the area cannot set up: an element is found by
|
|
29
|
+
* the doc's name with kit.find, and two people at once is a run-time skip on
|
|
30
|
+
* kit.distinctRunAccount(). A rule given one of these is still missing.
|
|
31
|
+
*/
|
|
32
|
+
const NOT_A_REASON = [
|
|
33
|
+
/\b(?:locators?|selectors?|test ?ids?|elements?|controls?|handles?|buttons?|fields?)\b[^.;]*\b(?:unknown|not known|not in the code facts|not listed|not documented|not named|named nowhere|not exposed|not identified)\b/i,
|
|
34
|
+
/\bno (?:known |documented )?(?:locator|selector|element|control|handle)\b/i,
|
|
35
|
+
/\bknown (?:[\w-]+ )?(?:locator|selector|element|control|handle)\b/i,
|
|
36
|
+
/\b(?:second|two|distinct|another|different) (?:\w+ )?(?:customer|account|user|people|person)s?\b/i,
|
|
37
|
+
];
|
|
38
|
+
export const acceptedReason = (why) => !NOT_A_REASON.some((re) => re.test(why));
|
|
39
|
+
export function ruleCoverage(content, planned) {
|
|
40
|
+
const tested = new Set(findTestBlocks(content).flatMap((b) => b.rules));
|
|
41
|
+
const cannotTest = [];
|
|
42
|
+
for (const m of content.matchAll(CANNOT)) {
|
|
43
|
+
for (const id of m[1].split(/\s*,\s*/))
|
|
44
|
+
if (planned.includes(id) && !tested.has(id) && acceptedReason(m[2]))
|
|
45
|
+
cannotTest.push({ id, why: m[2].trim() });
|
|
46
|
+
}
|
|
47
|
+
const explained = new Set(cannotTest.map((c) => c.id));
|
|
48
|
+
return { missing: planned.filter((id) => !tested.has(id) && !explained.has(id)), cannotTest };
|
|
49
|
+
}
|
|
50
|
+
/** The index of the brace closing the one at `open`, skipping strings and comments. */
|
|
51
|
+
function matchBrace(s, open) {
|
|
52
|
+
let depth = 0;
|
|
53
|
+
for (let i = open; i < s.length; i++) {
|
|
54
|
+
const c = s[i];
|
|
55
|
+
if (c === "'" || c === '"' || c === '`') {
|
|
56
|
+
for (i++; i < s.length && s[i] !== c; i++)
|
|
57
|
+
if (s[i] === '\\')
|
|
58
|
+
i++;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (c === '/' && s[i + 1] === '/') {
|
|
62
|
+
while (i < s.length && s[i] !== '\n')
|
|
63
|
+
i++;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (c === '/' && s[i + 1] === '*') {
|
|
67
|
+
i = s.indexOf('*/', i + 2);
|
|
68
|
+
if (i === -1)
|
|
69
|
+
return -1;
|
|
70
|
+
i++;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (c === '{')
|
|
74
|
+
depth++;
|
|
75
|
+
else if (c === '}' && --depth === 0)
|
|
76
|
+
return i;
|
|
77
|
+
}
|
|
78
|
+
return -1;
|
|
79
|
+
}
|
|
80
|
+
/** The code with every if/else/try/catch body (and .catch(…) callback) taken out. */
|
|
81
|
+
function withoutBranches(code) {
|
|
82
|
+
let out = code;
|
|
83
|
+
for (;;) {
|
|
84
|
+
const m = /\b(?:if\s*\(|else\b|try\b|catch\b|\.catch\s*\()/.exec(out);
|
|
85
|
+
if (!m)
|
|
86
|
+
return out;
|
|
87
|
+
let at = m.index + m[0].length;
|
|
88
|
+
if (/\($/.test(m[0])) {
|
|
89
|
+
const close = matchParen(out, at - 1);
|
|
90
|
+
if (close === -1)
|
|
91
|
+
return out;
|
|
92
|
+
at = close + 1;
|
|
93
|
+
if (m[0].startsWith('.catch')) {
|
|
94
|
+
out = out.slice(0, m.index) + out.slice(at);
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
else if (/^catch$/.test(m[0]) && /^\s*\(/.test(out.slice(at))) {
|
|
99
|
+
const close = matchParen(out, out.indexOf('(', at));
|
|
100
|
+
if (close === -1)
|
|
101
|
+
return out;
|
|
102
|
+
at = close + 1;
|
|
103
|
+
}
|
|
104
|
+
const rest = out.slice(at);
|
|
105
|
+
const brace = /^\s*\{/.exec(rest);
|
|
106
|
+
if (brace) {
|
|
107
|
+
const open = at + brace[0].length - 1;
|
|
108
|
+
const close = matchBrace(out, open);
|
|
109
|
+
if (close === -1)
|
|
110
|
+
return out;
|
|
111
|
+
out = out.slice(0, m.index) + out.slice(close + 1);
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
// A one-statement branch: drop up to the end of the statement.
|
|
115
|
+
const end = out.indexOf(';', at);
|
|
116
|
+
out = out.slice(0, m.index) + (end === -1 ? '' : out.slice(end + 1));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const stripComments = (code) => code.replace(/^\s*\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '');
|
|
121
|
+
/** Words for a good outcome and for a bad one: one assertion that accepts both checks nothing. */
|
|
122
|
+
const GOOD = /confirm|success|placed|booked|saved|created|thank|welcome|done|complete|sent/i;
|
|
123
|
+
const BAD = /not available|unavailable|error|fail|declin|invalid|denied|refus|reject|sorry|cannot|can't|unable|wrong|taken/i;
|
|
124
|
+
/** Regex literals inside assertion-like calls that offer alternatives. */
|
|
125
|
+
function eitherOutcome(code) {
|
|
126
|
+
const calls = /(?:getByText|toHaveText|toContainText|toMatch|toHaveURL|toHaveTitle|test)\s*\(\s*\/((?:\\\/|[^/\n])+)\/[a-z]*/g;
|
|
127
|
+
for (const m of code.matchAll(calls)) {
|
|
128
|
+
const alts = m[1].split('|');
|
|
129
|
+
if (alts.length < 2)
|
|
130
|
+
continue;
|
|
131
|
+
if (alts.some((a) => GOOD.test(a)) && alts.some((a) => BAD.test(a)))
|
|
132
|
+
return `/${m[1]}/`;
|
|
133
|
+
}
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
/** Problems of each test in a file. `steps` are the testkit's openStep names, when known. */
|
|
137
|
+
/** The words of a regex name: its alternatives, without the regex syntax. */
|
|
138
|
+
const regexWords = (src) => src.split('|').map((a) => a.replace(/\\[sbdw]/g, ' ').replace(/[\\^$()[\]{}?*+.]/g, ' ').replace(/\s+/g, ' ').trim().toLowerCase()).filter((a) => /[a-z]{3}/.test(a));
|
|
139
|
+
/**
|
|
140
|
+
* Problems of each test in a file. `steps` are the testkit's openStep names,
|
|
141
|
+
* `knownTexts` the app's source and the doc, lower-cased: every name a test
|
|
142
|
+
* finds an element by must come from them.
|
|
143
|
+
*/
|
|
144
|
+
export function testProblems(content, opts = {}) {
|
|
145
|
+
const out = [];
|
|
146
|
+
for (const block of findTestBlocks(content)) {
|
|
147
|
+
const text = content.slice(block.start, block.end);
|
|
148
|
+
const oracle = (/\/\/\s*oracle:\s*(.+)$/m.exec(text)?.[1] ?? '').toLowerCase();
|
|
149
|
+
const code = stripComments(text);
|
|
150
|
+
const seen = new Set();
|
|
151
|
+
const add = (reason) => {
|
|
152
|
+
if (seen.has(reason))
|
|
153
|
+
return;
|
|
154
|
+
seen.add(reason);
|
|
155
|
+
out.push({ title: block.title, rules: block.rules, reason, block });
|
|
156
|
+
};
|
|
157
|
+
// A stub's own words asserted back: the test proves the stub, not the app.
|
|
158
|
+
const stubbed = [...code.matchAll(/\.fulfill\(\s*\{([\s\S]*?)\}\s*\)/g)].flatMap((m) => [...m[1].matchAll(/(['"`])([^'"`\n]{6,})\1/g)].map((s) => s[2].toLowerCase()).filter((s) => /[a-z]{4,}\s/i.test(s)));
|
|
159
|
+
if (stubbed.length) {
|
|
160
|
+
const expected = [...code.matchAll(/(?:getByText|toHaveText|toContainText|toMatch)\s*\(\s*(?:\/((?:\\\/|[^/\n])+)\/|(['"`])([^'"`\n]+)\2)/g)]
|
|
161
|
+
.flatMap((m) => (m[1] ? m[1].split('|') : [m[3]]).map((t) => t.toLowerCase().replace(/\\/g, '').trim()))
|
|
162
|
+
.filter((t) => t.length >= 4);
|
|
163
|
+
const echo = expected.find((t) => stubbed.some((s) => s.includes(t)));
|
|
164
|
+
if (echo)
|
|
165
|
+
add(`asserts "${echo}", which its own page.route stub returns — the test proves the stub, not the app: assert what the app does with that state (what the doc says the screen shows)`);
|
|
166
|
+
}
|
|
167
|
+
// …or a value handed to a stub and then looked for on the screen.
|
|
168
|
+
const handed = new Set([...code.matchAll(/\b(?:stub\w*|mock\w*|fulfill|route)\s*\(([^;]*?)\)\s*;/gi)]
|
|
169
|
+
.flatMap((m) => m[1].match(/\b[a-z_$][\w$]*\b/gi) ?? [])
|
|
170
|
+
.filter((v) => !/^(page|route|request|url|r|req|status|body|json|true|false|null|undefined|JSON|stringify|contentType|async|await|const)$/.test(v)));
|
|
171
|
+
const shown = [...code.matchAll(/(?:getByText|toHaveText|toContainText)\s*\(\s*([a-z_$][\w$]*)\s*[,)]/gi)].map((m) => m[1]);
|
|
172
|
+
const back = shown.find((v) => handed.has(v));
|
|
173
|
+
if (back)
|
|
174
|
+
add(`looks on the screen for ${back}, the value its own stub returned — the test proves the stub, not the app: assert what the app decides with that state`);
|
|
175
|
+
// Q6 — kit.find returns a promise.
|
|
176
|
+
// `await kit.find(…).first()` is wrong too: the await applies to the whole chain.
|
|
177
|
+
if (/\bkit\.find\([^)]*\)\s*\.(?!then\b)/.test(code) || /expect\(\s*kit\.find\(/.test(code)) {
|
|
178
|
+
add('uses kit.find(...) without await: it returns a promise — write `const x = await kit.find(page, kind, name)` first, then use x');
|
|
179
|
+
}
|
|
180
|
+
// …kept in a variable without await is the same mistake.
|
|
181
|
+
if (/\b(?:const|let|var)\s+\w+\s*=\s*kit\.find\(/.test(code)) {
|
|
182
|
+
add('keeps kit.find(...) in a variable without await: it is a promise, not the element — `const x = await kit.find(...)`');
|
|
183
|
+
}
|
|
184
|
+
// Q10 — the name an element is found by comes from the screen's source or the doc.
|
|
185
|
+
for (const m of code.matchAll(/\bkit\.find\(\s*\w+\s*,\s*['"`][\w-]+['"`]\s*,\s*/g)) {
|
|
186
|
+
const rest = code.slice(m.index + m[0].length);
|
|
187
|
+
const v = /^([A-Za-z_$][\w$]*)\s*[,)]/.exec(rest)?.[1];
|
|
188
|
+
// A name from a list written out in the test (a parametrised loop, a const) is fine.
|
|
189
|
+
const writtenOut = v && (new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
|
|
190
|
+
new RegExp(`for\\s*\\(\\s*(?:const|let)\\s+(?:${v}|\\[[^\\]]*\\b${v}\\b[^\\]]*\\])\\s+of\\s+\\[\\s*['"\`\\[]`).test(code));
|
|
191
|
+
if (/^[A-Za-z_$]/.test(rest) && !writtenOut) {
|
|
192
|
+
add("finds an element by a name held in a variable: take the name the screen shows from the CODE FACTS or the doc, written out — a value the test made up (kit.unique, a stub's id) is never a control's name");
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
const re = /^\/((?:\\\/|[^/\n])+)\//.exec(rest);
|
|
196
|
+
if (re && opts.knownTexts && !regexWords(re[1]).some((w) => opts.knownTexts.includes(w))) {
|
|
197
|
+
add(`finds an element by /${re[1]}/, which matches no text in the app's source or the doc: use the name the screen really shows (the CODE FACTS give the component's text)`);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
// Q11 — a logged-out state reached by wiping storage: the testkit's login seed puts the session back.
|
|
201
|
+
if (/kit\.login\(/.test(code) && /\b(?:clearCookies\s*\(|(?:localStorage|sessionStorage)\.clear\s*\(|storageState\s*\(\s*\{\s*\}|removeItem\s*\()/.test(code)) {
|
|
202
|
+
add("logs out by clearing storage or cookies after kit.login: the testkit's login seed restores the session — log out through the app's own Log Out control");
|
|
203
|
+
}
|
|
204
|
+
// Q12 — one tab's login reaching another: kit.login prepares one page, so it cannot be set up.
|
|
205
|
+
if (/\b(?:other|second|another|both)\s+tabs?\b|\belsewhere\b/.test(oracle) && /\bnewPage\s*\(/.test(code) && /kit\.login\(/.test(code)) {
|
|
206
|
+
add("tests a login or logout in one tab reaching another: kit.login prepares one page, so the testkit cannot set that up — give the rule a cannot-test line");
|
|
207
|
+
}
|
|
208
|
+
// Q7 — only steps the testkit has.
|
|
209
|
+
if (opts.steps?.length) {
|
|
210
|
+
for (const m of code.matchAll(/\bopenStep\(\s*\w+\s*,\s*(['"`])([^'"`]+)\1/g)) {
|
|
211
|
+
if (!opts.steps.includes(m[2]))
|
|
212
|
+
add(`calls kit.openStep with step "${m[2]}", which the testkit does not have (its steps: ${opts.steps.join(', ')})`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// Q8 — a skip decided by what the page shows.
|
|
216
|
+
for (const m of code.matchAll(/\btest\.skip\s*\(/g)) {
|
|
217
|
+
const close = matchParen(code, m.index + m[0].length - 1);
|
|
218
|
+
const cond = close === -1 ? '' : code.slice(m.index + m[0].length, close).split(/,\s*['"`]/)[0];
|
|
219
|
+
const flag = /^\s*!?\s*\(?\s*([A-Za-z_$][\w$]*)/.exec(cond)?.[1];
|
|
220
|
+
const fromText = flag && new RegExp(`\\b(?:const|let|var)\\s+${flag}\\s*=[^;\\n]*(?:innerText|textContent|allInnerTexts|allTextContents)\\(`).test(code);
|
|
221
|
+
if (/innerText\(|textContent\(|allInnerTexts\(|allTextContents\(/.test(cond) || fromText) {
|
|
222
|
+
add('skips itself because of what the page shows: what the doc expects on the page is asserted, never a reason to skip');
|
|
223
|
+
break;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
// Q9 — real data is made by the run account and recorded, never by the shared fixture.
|
|
227
|
+
const createsData = /(?:find\(\s*\w+\s*,\s*['"`]button['"`]\s*,\s*|getByRole\(\s*['"`]button['"`]\s*,\s*\{\s*name:\s*)['"`/]\s*(?:pay\b|book(?!\s*again)\b|place\b|confirm\s+(?:payment|booking)|complete\s+(?:payment|booking)|submit\s+order)/i.test(code);
|
|
228
|
+
if (createsData && /\.click\(|settledClick\(/.test(code) && !/\.route\(/.test(code) && !/['"`]run['"`]/.test(code) && !/kit\.(?:unique|created)\(/.test(code)) {
|
|
229
|
+
add("clicks a button that creates a booking or a payment, but not as the testkit's 'run' role with kit.unique/kit.created: real data is made by the run account and recorded so it can be cleaned up");
|
|
230
|
+
}
|
|
231
|
+
// Q2 — an assertion that always runs.
|
|
232
|
+
const top = withoutBranches(code);
|
|
233
|
+
if (!/\bexpect(?:\.soft)?\s*\(|\bexpect\.poll\s*\(/.test(top)) {
|
|
234
|
+
add(/\bexpect\s*\(/.test(code)
|
|
235
|
+
? 'its only assertions sit inside an if, else or catch, so the test can pass without checking anything — assert unconditionally'
|
|
236
|
+
: 'has no assertion');
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
// Q3 — one assertion that accepts success and failure.
|
|
240
|
+
const either = eitherOutcome(code);
|
|
241
|
+
if (either && !(GOOD.test(oracle) && BAD.test(oracle) && /\bor\b/.test(oracle))) {
|
|
242
|
+
add(`asserts ${either}, which matches both the good and the bad outcome — assert the one outcome the rule expects`);
|
|
243
|
+
}
|
|
244
|
+
// Q4 — the page's whole text compared with itself.
|
|
245
|
+
if (/locator\(\s*['"`](?:body|main|html)['"`]\s*\)\s*\.(?:innerText|textContent)\(/.test(code) && /\.split\(\s*['"`]\\n['"`]\s*\)|toContainText\(\s*\w+/.test(code)) {
|
|
246
|
+
add("compares the page's text with itself (body/main innerText): read the element the rule is about (the operator's name, the total) and compare that");
|
|
247
|
+
}
|
|
248
|
+
// Q5 — only that something is absent.
|
|
249
|
+
const asserts = [...top.matchAll(/expect(?:\.soft)?\s*\(([\s\S]*?)\)\s*\.\s*((?:not\.)?\w+)/g)];
|
|
250
|
+
const positive = asserts.filter((a) => !a[2].startsWith('not.') && !/^toHaveCount$/.test(a[2]) || /toHaveCount\(\s*[1-9]/.test(top));
|
|
251
|
+
if (asserts.length && !positive.length && !/\b(never|no|not|without|cannot|isn't|doesn't|none|nothing|hidden|absent|disappear|removed)\b/.test(oracle)) {
|
|
252
|
+
add('only asserts that something is absent: also assert the outcome the rule states is there');
|
|
253
|
+
}
|
|
254
|
+
// Always true, whatever the app does.
|
|
255
|
+
if (/expect\(\s*new Set\(\s*(\w+)\s*\)\.size\s*\)\.toBeLessThanOrEqual\(\s*\1\.length\s*\)/.test(code) || /expect\([^)]*\b(?:false\s*\|\||true\s*&&)/.test(code) || /\.toBeDefined\(\)/.test(top) && !/expect\([^)]*\)\.(?!toBeDefined)\w+/.test(top)) {
|
|
256
|
+
add('has an assertion that cannot fail (a set is never larger than its list; false || x; only toBeDefined) — assert the outcome itself');
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
return out;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* The step names a testkit's openStep accepts: the keys of the object its
|
|
263
|
+
* openStep looks the step up in (if (!STEPS[step]) …).
|
|
264
|
+
*/
|
|
265
|
+
export function harnessSteps(harnessSource) {
|
|
266
|
+
// The contract's own export first (STEPS), else the object openStep looks the step up in.
|
|
267
|
+
let map = /(?:const|let|var)\s+STEPS\s*=\s*\{/.test(harnessSource) ? 'STEPS' : undefined;
|
|
268
|
+
if (!map) {
|
|
269
|
+
const fn = /function\s+openStep\s*\(\s*\w+\s*,\s*(\w+)/.exec(harnessSource);
|
|
270
|
+
if (!fn)
|
|
271
|
+
return [];
|
|
272
|
+
const body = harnessSource.slice(fn.index, fn.index + 1500);
|
|
273
|
+
map = new RegExp(`!\\s*(\\w+)\\[\\s*${fn[1]}\\s*\\]`).exec(body)?.[1];
|
|
274
|
+
}
|
|
275
|
+
if (!map)
|
|
276
|
+
return [];
|
|
277
|
+
const decl = new RegExp(`(?:const|let|var)\\s+${map}\\s*=\\s*\\{`).exec(harnessSource);
|
|
278
|
+
if (!decl)
|
|
279
|
+
return [];
|
|
280
|
+
const open = decl.index + decl[0].length - 1;
|
|
281
|
+
const close = matchBrace(harnessSource, open);
|
|
282
|
+
if (close === -1)
|
|
283
|
+
return [];
|
|
284
|
+
return [...harnessSource.slice(open + 1, close).matchAll(/^\s*(?:(['"])([^'"]+)\1|([A-Za-z_$][\w$-]*))\s*:/gm)].map((m) => m[2] ?? m[3]);
|
|
285
|
+
}
|
|
286
|
+
/** One refusal naming every problem, so a single rewrite fixes them all. */
|
|
287
|
+
export function describeProblems(problems, coverage) {
|
|
288
|
+
const lines = [];
|
|
289
|
+
if (coverage?.missing.length) {
|
|
290
|
+
lines.push(`rules with no test: ${coverage.missing.join(', ')} — write a test for each, or a "// cannot-test: <id> — <the state this area cannot set up>" line. "The locator is unknown" is not such a state (find it with kit.find by the doc's name), and neither is "needs two customers" (skip unless kit.distinctRunAccount())`);
|
|
291
|
+
}
|
|
292
|
+
for (const p of problems)
|
|
293
|
+
lines.push(`"${p.title}"${p.rules.length ? ` (${p.rules.join(', ')})` : ''} ${p.reason}`);
|
|
294
|
+
return lines.join('\n');
|
|
295
|
+
}
|
|
@@ -169,6 +169,21 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
169
169
|
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
170
170
|
return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
|
|
171
171
|
}
|
|
172
|
+
// F22 — a fresh file exists to test its rules: one with no test at all says
|
|
173
|
+
// nothing, and passes. It goes back once, told to stub what it cannot create.
|
|
174
|
+
if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
|
|
175
|
+
return 'the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out';
|
|
176
|
+
}
|
|
177
|
+
// F21 — a run-time skip is for what the ENVIRONMENT lacks (an account, a
|
|
178
|
+
// variable, seeded data). Skipping because an element is not on the page
|
|
179
|
+
// hides exactly what a test written from the doc exists to report: the
|
|
180
|
+
// element the doc names is missing, and that is a failure.
|
|
181
|
+
if (ctx.forbidPlaceholders) {
|
|
182
|
+
const why = elementSkips(content);
|
|
183
|
+
if (why.length) {
|
|
184
|
+
return `a test skips itself because an element is not on the page (${why[0]}): a missing element the doc names is a failure — assert it instead. Skip only for an account, a variable or data the environment lacks`;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
172
187
|
// F17 — relative imports resolve from where the file is written.
|
|
173
188
|
if (ctx.filePath) {
|
|
174
189
|
const missing = unresolvedImports(content, ctx.filePath);
|
|
@@ -234,9 +249,59 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
234
249
|
}
|
|
235
250
|
}
|
|
236
251
|
}
|
|
252
|
+
// (c) A name, label or placeholder the app never renders is a guess too.
|
|
253
|
+
if (ctx.knownTexts) {
|
|
254
|
+
const missing = usedLabels(active).filter((l) => !ctx.knownTexts.includes(l.toLowerCase()));
|
|
255
|
+
if (missing.length) {
|
|
256
|
+
return `uses ${missing.map((m) => `"${m}"`).slice(0, 3).join(', ')} as an accessible name, label or placeholder, but that text appears nowhere in the app's source or the journey doc — take every name from the code facts or the rules`;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
237
259
|
}
|
|
238
260
|
return null;
|
|
239
261
|
}
|
|
262
|
+
/**
|
|
263
|
+
* The run-time skips whose condition looks at the page: test.skip(true, …),
|
|
264
|
+
* or a condition that counts, checks the visibility of, or locates an element.
|
|
265
|
+
* Returns each condition, shortened.
|
|
266
|
+
*/
|
|
267
|
+
export function elementSkips(content) {
|
|
268
|
+
const out = [];
|
|
269
|
+
for (const m of content.matchAll(/\btest\.skip\s*\(/g)) {
|
|
270
|
+
const open = m.index + m[0].length - 1;
|
|
271
|
+
const close = matchParen(content, open);
|
|
272
|
+
if (close === -1)
|
|
273
|
+
continue;
|
|
274
|
+
const args = content.slice(open + 1, close);
|
|
275
|
+
// A declaration skip (test.skip('title', fn)) is the placeholder gate's job.
|
|
276
|
+
if (/^\s*['"`]/.test(args))
|
|
277
|
+
continue;
|
|
278
|
+
const cond = args.split(/,\s*['"`]/)[0];
|
|
279
|
+
const looksAtPage = /^\s*true\s*$/.test(cond) || /\.count\(|\.isVisible\(|\.isHidden\(|getBy\w+\(|\bfind\(|\.locator\(|\$\(/.test(cond);
|
|
280
|
+
// …or a flag set from the page a few lines above (const burger = await …isVisible()).
|
|
281
|
+
const flag = /^\s*!?\s*([A-Za-z_$][\w$]*)\s*$/.exec(cond)?.[1];
|
|
282
|
+
const flagFromPage = flag && new RegExp(`\\b(?:const|let|var)\\s+${flag}\\s*=[^;\\n]*(?:\\.count\\(|\\.isVisible\\(|getBy\\w+\\(|\\bfind\\(|\\.locator\\()`).test(content);
|
|
283
|
+
if (looksAtPage || flagFromPage)
|
|
284
|
+
out.push(cond.replace(/\s+/g, ' ').trim().slice(0, 80));
|
|
285
|
+
}
|
|
286
|
+
return out;
|
|
287
|
+
}
|
|
288
|
+
/** Literal names a test locates by: getByRole(…, { name }), getByLabel, getByPlaceholder. */
|
|
289
|
+
export function usedLabels(content) {
|
|
290
|
+
const out = new Set();
|
|
291
|
+
const res = [
|
|
292
|
+
/getByRole\(\s*['"`][\w-]+['"`]\s*,\s*\{[^}]*?\bname\s*:\s*(['"`])((?:(?!\1).)+)\1/g,
|
|
293
|
+
/getBy(?:Label|Placeholder)\(\s*(['"`])((?:(?!\1).)+)\1/g,
|
|
294
|
+
/\bfind\(\s*\w+\s*,\s*['"`][\w-]+['"`]\s*,\s*(['"`])((?:(?!\1).)+)\1/g,
|
|
295
|
+
];
|
|
296
|
+
for (const re of res) {
|
|
297
|
+
for (const m of content.matchAll(re)) {
|
|
298
|
+
const name = m[2].replace(/\s+/g, ' ').trim();
|
|
299
|
+
if (name.length >= 3 && !name.includes('${') && !name.includes('\\'))
|
|
300
|
+
out.add(name);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return [...out];
|
|
304
|
+
}
|
|
240
305
|
/**
|
|
241
306
|
* Remove pending test blocks (`it/test/describe.skip|fixme|todo(...)`) so the
|
|
242
307
|
* F9 selector check only ever inspects RUNNABLE code. A guessed selector
|
|
@@ -410,10 +475,104 @@ function looksStructurallyClean(content) {
|
|
|
410
475
|
t.startsWith('"use strict"');
|
|
411
476
|
if (!startsWithCode)
|
|
412
477
|
return false;
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
478
|
+
return unbalancedAt(content) === null;
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Where the brackets of a TS file stop matching, or null when they all do.
|
|
482
|
+
* Strings, template literals (with their \${…}), comments and regex literals
|
|
483
|
+
* are skipped: a "(" inside a test's text is not code. Counting them was what
|
|
484
|
+
* refused correct files as "not valid syntax".
|
|
485
|
+
*/
|
|
486
|
+
export function unbalancedAt(content) {
|
|
487
|
+
const stack = [];
|
|
488
|
+
const pairs = { ')': '(', '}': '{', ']': '[' };
|
|
489
|
+
let line = 1;
|
|
490
|
+
let prev = ''; // last significant character, to tell a regex from a division
|
|
491
|
+
const templates = []; // brace depth at which each open template's \${ began
|
|
492
|
+
for (let i = 0; i < content.length; i++) {
|
|
493
|
+
const c = content[i];
|
|
494
|
+
if (c === '\n') {
|
|
495
|
+
line++;
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
if (/\s/.test(c))
|
|
499
|
+
continue;
|
|
500
|
+
if (c === '/' && content[i + 1] === '/') {
|
|
501
|
+
while (i < content.length && content[i] !== '\n')
|
|
502
|
+
i++;
|
|
503
|
+
line++;
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
if (c === '/' && content[i + 1] === '*') {
|
|
507
|
+
i += 2;
|
|
508
|
+
while (i < content.length && !(content[i] === '*' && content[i + 1] === '/')) {
|
|
509
|
+
if (content[i] === '\n')
|
|
510
|
+
line++;
|
|
511
|
+
i++;
|
|
512
|
+
}
|
|
513
|
+
i++;
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
if (c === "'" || c === '"') {
|
|
517
|
+
for (i++; i < content.length && content[i] !== c && content[i] !== '\n'; i++)
|
|
518
|
+
if (content[i] === '\\')
|
|
519
|
+
i++;
|
|
520
|
+
prev = c;
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
if (c === '`' || (c === '}' && templates.length && templates[templates.length - 1] === stack.length)) {
|
|
524
|
+
if (c === '}')
|
|
525
|
+
templates.pop();
|
|
526
|
+
// Inside a template: up to its end, or to a \${ that opens code.
|
|
527
|
+
for (i++; i < content.length; i++) {
|
|
528
|
+
if (content[i] === '\\') {
|
|
529
|
+
i++;
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
if (content[i] === '\n')
|
|
533
|
+
line++;
|
|
534
|
+
if (content[i] === '`')
|
|
535
|
+
break;
|
|
536
|
+
if (content[i] === '$' && content[i + 1] === '{') {
|
|
537
|
+
i++;
|
|
538
|
+
templates.push(stack.length);
|
|
539
|
+
break;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
prev = '`';
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
if (c === '/' && (!prev || /[(,=:[!&|?{};+\-*%<>~^]/.test(prev))) {
|
|
546
|
+
// A regex literal: skip to its closing slash, minding classes and escapes.
|
|
547
|
+
let inClass = false;
|
|
548
|
+
for (i++; i < content.length && content[i] !== '\n'; i++) {
|
|
549
|
+
if (content[i] === '\\') {
|
|
550
|
+
i++;
|
|
551
|
+
continue;
|
|
552
|
+
}
|
|
553
|
+
if (content[i] === '[')
|
|
554
|
+
inClass = true;
|
|
555
|
+
else if (content[i] === ']')
|
|
556
|
+
inClass = false;
|
|
557
|
+
else if (content[i] === '/' && !inClass)
|
|
558
|
+
break;
|
|
559
|
+
}
|
|
560
|
+
while (/[a-z]/i.test(content[i + 1] ?? ''))
|
|
561
|
+
i++;
|
|
562
|
+
prev = 'r';
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
if (c === '(' || c === '{' || c === '[')
|
|
566
|
+
stack.push({ ch: c, line });
|
|
567
|
+
else if (c in pairs) {
|
|
568
|
+
const open = stack.pop();
|
|
569
|
+
if (!open || open.ch !== pairs[c])
|
|
570
|
+
return `line ${line}: "${c}" does not close ${open ? `the "${open.ch}" from line ${open.line}` : 'anything'}`;
|
|
571
|
+
}
|
|
572
|
+
prev = c;
|
|
573
|
+
}
|
|
574
|
+
const open = stack.pop();
|
|
575
|
+
return open ? `the "${open.ch}" from line ${open.line} is never closed` : null;
|
|
417
576
|
}
|
|
418
577
|
const RESOLVE_EXTS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json'];
|
|
419
578
|
/** Relative module specifiers (import / require / dynamic import) that reach no file from `filePath`. */
|
|
@@ -491,13 +650,18 @@ function testCriteria(content) {
|
|
|
491
650
|
starts.push(i); });
|
|
492
651
|
return starts.map((start, n) => {
|
|
493
652
|
const end = starts[n + 1] ?? lines.length;
|
|
494
|
-
|
|
653
|
+
// The oracle comment sits right above the test or inside it.
|
|
654
|
+
const above = [];
|
|
655
|
+
for (let i = start - 1; i > (starts[n - 1] ?? -1) && /^\s*\/\//.test(lines[i] ?? ''); i--)
|
|
656
|
+
above.push(i);
|
|
657
|
+
for (const i of [...above, ...Array.from({ length: end - start }, (_, k) => start + k)]) {
|
|
495
658
|
const m = /\/\/\s*oracle:\s*(.+)$/.exec(lines[i]);
|
|
496
659
|
if (m) {
|
|
497
660
|
const tier = (/\[(spec|baseline|observed)\]/i.exec(m[1])?.[1]?.toLowerCase() ?? 'untagged');
|
|
498
|
-
// The criterion is the
|
|
661
|
+
// The criterion is the rule id when the oracle names one, else the quoted rule, else the oracle text.
|
|
662
|
+
const id = /^\s*(R\d+)\b/.exec(m[1])?.[1];
|
|
499
663
|
const quoted = /["“]([^"”]{12,})["”]/.exec(m[1]);
|
|
500
|
-
return { criterion: (quoted ? quoted[1] : m[1]).trim().toLowerCase(), tier };
|
|
664
|
+
return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
|
|
501
665
|
}
|
|
502
666
|
}
|
|
503
667
|
return { criterion: '', tier: 'untagged' };
|