@activate-core/test-forge 2.0.0 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -387,10 +387,26 @@ history, the plan, and the code facts. No later step sends the whole doc again.
387
387
  with the reason, if it sleeps for a fixed time, skips because an element is
388
388
  missing, has no test or no assertion, spends more than `maxTestsPerRule`
389
389
  tests on one rule, or names a label found neither in the code nor the doc.
390
- 6. **A changed doc.** Only what changed is touched: a renamed label is replaced
390
+ 6. **Every rule, a test that can fail.** Each test is checked on its own:
391
+ every planned rule has a test, or a `// cannot-test: R7 — <the state this
392
+ area cannot set up>` line ("the locator is unknown" is not such a state);
393
+ no assertion accepts the good and the bad outcome alike, compares the page's
394
+ whole text, sits only inside an `if`, or repeats what its own stub returned;
395
+ `kit.find` is awaited and finds elements by the names the screen shows;
396
+ `kit.openStep` names a step the testkit has. The tests that fail a check are
397
+ taken out and their rules written again in ONE small call; what is still
398
+ wrong is removed and reported. The summary line says how many planned rules
399
+ have a test, which cannot be tested here and why, and which have none, and
400
+ `rules.json` / `plan-check.md` record the same.
401
+ 7. **A changed doc.** Only what changed is touched: a renamed label is replaced
391
402
  in its tests, a removed rule's tests are deleted (both without AI); a
392
403
  reworded or new rule gets its tests written by one small call.
393
404
 
405
+ **A feature that is not live yet.** Put `Status: not built` or
406
+ `Status: behind flag <name>` in the journey doc's header. Every file of the
407
+ journey is stamped `// testforge-pending: …`, and `testforge run` (and `bind`)
408
+ hold it back until the line is removed; `--all` runs it anyway.
409
+
394
410
  `generate` runs nothing. `testforge run` runs the suite; `testforge review`
395
411
  judges failures when you ask for it.
396
412
 
@@ -127,15 +127,18 @@ export const DEEP_DOC_GUIDANCE = [
127
127
  ` becomes a test that fails on a correct product.`,
128
128
  ` An [observed] criterion is about what the customer sees or what the`,
129
129
  ` server accepts or refuses — never an implementation detail (a storage`,
130
- ` key name, a CSS class, an internal flag, a request field name) unless the`,
131
- ` source itself is about it.`,
132
- ` Only what a test can drive is a criterion. Behaviour no screen or`,
133
- ` endpoint can reach ("not tested through the screen") is not tagged —`,
134
- ` leave it out, or mention it in Test setup if the generator needs to know.`,
130
+ ` key name, a CSS class, an internal flag, a request field name, a log`,
131
+ ` line, a lock, a cache) unless the source itself is about it.`,
132
+ ` Only what a test in THIS area can drive is a criterion. Behaviour no`,
133
+ ` screen or endpoint of this area can reach (a backend job, another app,`,
134
+ ` an operator's action) is not tagged — list it under "## Not requirements"`,
135
+ ` with where it is checked, or mention it in Test setup if the generator`,
136
+ ` needs to know.`,
135
137
  ``,
136
138
  `METHOD. Work through these steps for every journey, in order:`,
137
139
  `1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
138
- ` interactive element and where it appears: each input, button, link,`,
140
+ ` interactive element and where it appears, each row's first cell its id`,
141
+ ` C1, C2, C3 …: each input, button, link,`,
139
142
  ` checkbox, code box, dropdown, pop-up, menu item, and the page itself.`,
140
143
  ` Secondary controls count as much as the main form: Resend, Back,`,
141
144
  ` show/hide, close, apply, sign-out, provider buttons. Apply every`,
@@ -9,7 +9,11 @@
9
9
  import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
10
10
  import { join, relative, resolve } from 'node:path';
11
11
  const CODE = /\.(tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go)$/;
12
- const SKIP_DIR = /^(node_modules|\.next|dist|build|coverage|vendor|storage|\.git|tests?|__tests__|__mocks__|test-artifacts|testforge|generated-tests)$/;
12
+ const SKIP_DIR = /^(node_modules|\.next|dist|build|coverage|vendor|storage|public|\.git|tests?|__tests__|__mocks__|test-artifacts|testforge|generated-tests)$/;
13
+ /** Where a package keeps its code: src/ when it has one, else the package itself (Laravel's app/, a flat repo). */
14
+ export function sourceRoot(base = '.') {
15
+ return existsSync(join(base, 'src')) ? join(base, 'src') : base;
16
+ }
13
17
  const FILE_REF = /((?:[\w.-]+\/)+[\w.-]+\.(?:tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go))(?::(\d+)(?:[-–](\d+))?)?/g;
14
18
  const QUOTED = /"([^"\n]{4,60})"/g;
15
19
  /** Words in a control's name that name no component in particular. */
@@ -83,7 +87,7 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
83
87
  }
84
88
  // A file named after a control ("Escalate button" → escalateDisputeButton/) is
85
89
  // the component itself: found by its path, before any text search.
86
- const controlWords = [...markdown.matchAll(/^\|\s*C\d+[a-z]?\s*\|([^|]+)\|/gm)]
90
+ const controlWords = [...markdown.matchAll(/^\|\s*C?\d+[a-z]?\s*\|([^|]+)\|/gm)]
87
91
  .flatMap((m) => m[1].replace(/["“”(][^"”)]*["”)]?/g, ' ').match(/[A-Za-z]{5,}/g) ?? [])
88
92
  .map((w) => w.toLowerCase())
89
93
  .filter((w) => !GENERIC.has(w));
@@ -198,6 +202,55 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
198
202
  order.push(h.file);
199
203
  }
200
204
  }
205
+ // A page's own parts (its total, its operator card, its pickers) live in the
206
+ // components it imports from its own folder. Their visible texts and labels
207
+ // come right after it, ahead of files that only share a word with the doc.
208
+ const textLine = /(>\s*[A-Za-z£$€][^<>{}]{1,60}<)|\b(aria-label|placeholder|title|label|alt)\s*=\s*['"{]|\btitle:\s*['"`]|<(button|Button|a|Link|h[1-6]|label)\b/;
209
+ for (const f of order.slice(0, 6)) {
210
+ const dir = f.replace(/[\\/][^\\/]+$/, '');
211
+ let text;
212
+ try {
213
+ text = readFileSync(f, 'utf-8');
214
+ }
215
+ catch {
216
+ continue;
217
+ }
218
+ const children = [...text.matchAll(/from\s+['"]([^'"]+)['"]/g)]
219
+ .map((m) => m[1].replace(/^[@~.]+[\\/]?/, '').replace(/^src[\\/]/, ''))
220
+ .map((spec) => files.find((t) => t !== f && t.startsWith(dir) && t.replace(/\.\w+$/, '').replace(/[\\/]index$/, '').endsWith(spec)))
221
+ .filter((c) => !!c && /\.(tsx|jsx|vue|svelte)$/.test(c));
222
+ let at = order.indexOf(f);
223
+ for (const c of [...new Set(children)].slice(0, 12)) {
224
+ const lines = readFileSync(c, 'utf-8').split('\n');
225
+ lines.forEach((l, i) => { if (textLine.test(l))
226
+ add(c, i, i + 2); });
227
+ if (!windows.has(c))
228
+ continue;
229
+ const was = order.indexOf(c);
230
+ if (was !== -1 && was <= at)
231
+ continue;
232
+ if (was !== -1)
233
+ order.splice(was, 1);
234
+ order.splice(++at, 0, c);
235
+ }
236
+ }
237
+ // A hook or API module says WHERE the app sends each request, often far below
238
+ // its first lines: every line that makes a call or names a path is kept.
239
+ for (const f of order) {
240
+ if (!/(^|[\\/])(use[A-Z]\w*|api|services?|dataProvider|queries|requests?)([\\/.]|$)/.test(f))
241
+ continue;
242
+ let lines;
243
+ try {
244
+ lines = readFileSync(f, 'utf-8').split('\n');
245
+ }
246
+ catch {
247
+ continue;
248
+ }
249
+ lines.forEach((l, i) => {
250
+ if (/\b(?:get|post|put|patch|del|delete|fetch|request|axios(?:\.\w+)?)\s*(?:<[^>]*>)?\s*\(|['"`]\/[\w-]+\//.test(l))
251
+ add(f, i + 1 - 3, i + 1 + 3);
252
+ });
253
+ }
201
254
  const out = [];
202
255
  let used = 0;
203
256
  for (const file of order) {
@@ -37,7 +37,7 @@ const MAP_PATH = 'testforge.md';
37
37
  * that upgrades keeps serving the facts it cached under the old version —
38
38
  * so the new facts never reach a single prompt and the upgrade does nothing.
39
39
  */
40
- const FACTS_VERSION = 4;
40
+ const FACTS_VERSION = 5;
41
41
  /** Where a journey's grounded facts live, mirroring the plan cache layout. */
42
42
  function groundingPath(journey) {
43
43
  return join(journeyDir(journey), 'facts.json');
@@ -167,6 +167,12 @@ reuseOnly = false) {
167
167
  const excerpts = sourceExcerpts(journey.rawMarkdown || '', [srcDir, ...(config.relatedDirs ?? [])]);
168
168
  if (!excerpts)
169
169
  return null;
170
+ // Excerpts only from other packages (a backend matched on a quoted text) say
171
+ // nothing about this area's screens: a call on them returns "unknown" for
172
+ // every section and costs as much as a real one.
173
+ const files = [...excerpts.matchAll(/^### (.+)$/gm)].map((m) => m[1]);
174
+ if (files.length && files.every((f) => f.startsWith('..')))
175
+ return null;
170
176
  const { program, args } = withNoTools(splitCommand(config.aiCommand));
171
177
  let facts;
172
178
  try {
@@ -108,10 +108,22 @@ export function buildLeanReviewPrompt(doc, section, excerpts, otherDocs) {
108
108
  export function enforceSpecSources(markdown) {
109
109
  let inFence = false;
110
110
  let retagged = 0;
111
- const lines = markdown.split('\n').map((line) => {
111
+ const all = markdown.split('\n');
112
+ // A rule's source may sit on a later line of its own block (after its
113
+ // steps), up to the next rule or heading.
114
+ const sourcedBelow = (i) => {
115
+ for (let j = i + 1; j < all.length && j <= i + 30; j++) {
116
+ if (/^#{1,6}\s/.test(all[j]) || /\[(?:spec|baseline|observed)\]/i.test(all[j]))
117
+ return false;
118
+ if (HAS_SOURCE.test(all[j]))
119
+ return true;
120
+ }
121
+ return false;
122
+ };
123
+ const lines = all.map((line, i) => {
112
124
  if (/^\s*```/.test(line))
113
125
  inFence = !inFence;
114
- if (inFence || !/\[spec\]/i.test(line) || HAS_SOURCE.test(line))
126
+ if (inFence || !/\[spec\]/i.test(line) || HAS_SOURCE.test(line) || sourcedBelow(i))
115
127
  return line;
116
128
  retagged += 1;
117
129
  return line.replace(/\[spec\]/gi, '[observed]');
@@ -0,0 +1,295 @@
1
+ /**
2
+ * testforge — does each written test really check its rule?
3
+ *
4
+ * The file-level gates see a file as text. These checks read each test on its
5
+ * own, against the rules the file was planned for:
6
+ *
7
+ * - every planned rule has a test, or a "// cannot-test: R7 — <what is
8
+ * missing>" line that says why not (Q1);
9
+ * - each test has an assertion that runs, not one only inside an if or a
10
+ * catch (Q2);
11
+ * - no assertion accepts the rule's success and its failure alike, like
12
+ * /confirm|time not available/ (Q3);
13
+ * - no test compares the page's whole text with itself (Q4);
14
+ * - a test does more than assert that something is absent, unless its rule
15
+ * says never / no / not (Q5);
16
+ * - kit.find is awaited: it returns a promise (Q6);
17
+ * - kit.openStep names a step the testkit has (Q7);
18
+ * - a run-time skip does not depend on what the page shows (Q8).
19
+ *
20
+ * Each problem names its test and its rules, so only those tests are written
21
+ * again; the rest of the file is kept.
22
+ */
23
+ import { matchParen } from './validate.js';
24
+ import { findTestBlocks } from '../generate/test-blocks.js';
25
+ const CANNOT = /\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)$/gm;
26
+ /** Which planned rules have a test, which are explained, which are simply missing. */
27
+ /**
28
+ * Reasons that are not a state the area cannot set up: an element is found by
29
+ * the doc's name with kit.find, and two people at once is a run-time skip on
30
+ * kit.distinctRunAccount(). A rule given one of these is still missing.
31
+ */
32
+ const NOT_A_REASON = [
33
+ /\b(?:locators?|selectors?|test ?ids?|elements?|controls?|handles?|buttons?|fields?)\b[^.;]*\b(?:unknown|not known|not in the code facts|not listed|not documented|not named|named nowhere|not exposed|not identified)\b/i,
34
+ /\bno (?:known |documented )?(?:locator|selector|element|control|handle)\b/i,
35
+ /\bknown (?:[\w-]+ )?(?:locator|selector|element|control|handle)\b/i,
36
+ /\b(?:second|two|distinct|another|different) (?:\w+ )?(?:customer|account|user|people|person)s?\b/i,
37
+ ];
38
+ export const acceptedReason = (why) => !NOT_A_REASON.some((re) => re.test(why));
39
+ export function ruleCoverage(content, planned) {
40
+ const tested = new Set(findTestBlocks(content).flatMap((b) => b.rules));
41
+ const cannotTest = [];
42
+ for (const m of content.matchAll(CANNOT)) {
43
+ for (const id of m[1].split(/\s*,\s*/))
44
+ if (planned.includes(id) && !tested.has(id) && acceptedReason(m[2]))
45
+ cannotTest.push({ id, why: m[2].trim() });
46
+ }
47
+ const explained = new Set(cannotTest.map((c) => c.id));
48
+ return { missing: planned.filter((id) => !tested.has(id) && !explained.has(id)), cannotTest };
49
+ }
50
+ /** The index of the brace closing the one at `open`, skipping strings and comments. */
51
+ function matchBrace(s, open) {
52
+ let depth = 0;
53
+ for (let i = open; i < s.length; i++) {
54
+ const c = s[i];
55
+ if (c === "'" || c === '"' || c === '`') {
56
+ for (i++; i < s.length && s[i] !== c; i++)
57
+ if (s[i] === '\\')
58
+ i++;
59
+ continue;
60
+ }
61
+ if (c === '/' && s[i + 1] === '/') {
62
+ while (i < s.length && s[i] !== '\n')
63
+ i++;
64
+ continue;
65
+ }
66
+ if (c === '/' && s[i + 1] === '*') {
67
+ i = s.indexOf('*/', i + 2);
68
+ if (i === -1)
69
+ return -1;
70
+ i++;
71
+ continue;
72
+ }
73
+ if (c === '{')
74
+ depth++;
75
+ else if (c === '}' && --depth === 0)
76
+ return i;
77
+ }
78
+ return -1;
79
+ }
80
+ /** The code with every if/else/try/catch body (and .catch(…) callback) taken out. */
81
+ function withoutBranches(code) {
82
+ let out = code;
83
+ for (;;) {
84
+ const m = /\b(?:if\s*\(|else\b|try\b|catch\b|\.catch\s*\()/.exec(out);
85
+ if (!m)
86
+ return out;
87
+ let at = m.index + m[0].length;
88
+ if (/\($/.test(m[0])) {
89
+ const close = matchParen(out, at - 1);
90
+ if (close === -1)
91
+ return out;
92
+ at = close + 1;
93
+ if (m[0].startsWith('.catch')) {
94
+ out = out.slice(0, m.index) + out.slice(at);
95
+ continue;
96
+ }
97
+ }
98
+ else if (/^catch$/.test(m[0]) && /^\s*\(/.test(out.slice(at))) {
99
+ const close = matchParen(out, out.indexOf('(', at));
100
+ if (close === -1)
101
+ return out;
102
+ at = close + 1;
103
+ }
104
+ const rest = out.slice(at);
105
+ const brace = /^\s*\{/.exec(rest);
106
+ if (brace) {
107
+ const open = at + brace[0].length - 1;
108
+ const close = matchBrace(out, open);
109
+ if (close === -1)
110
+ return out;
111
+ out = out.slice(0, m.index) + out.slice(close + 1);
112
+ }
113
+ else {
114
+ // A one-statement branch: drop up to the end of the statement.
115
+ const end = out.indexOf(';', at);
116
+ out = out.slice(0, m.index) + (end === -1 ? '' : out.slice(end + 1));
117
+ }
118
+ }
119
+ }
120
+ const stripComments = (code) => code.replace(/^\s*\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '');
121
+ /** Words for a good outcome and for a bad one: one assertion that accepts both checks nothing. */
122
+ const GOOD = /confirm|success|placed|booked|saved|created|thank|welcome|done|complete|sent/i;
123
+ const BAD = /not available|unavailable|error|fail|declin|invalid|denied|refus|reject|sorry|cannot|can't|unable|wrong|taken/i;
124
+ /** Regex literals inside assertion-like calls that offer alternatives. */
125
+ function eitherOutcome(code) {
126
+ const calls = /(?:getByText|toHaveText|toContainText|toMatch|toHaveURL|toHaveTitle|test)\s*\(\s*\/((?:\\\/|[^/\n])+)\/[a-z]*/g;
127
+ for (const m of code.matchAll(calls)) {
128
+ const alts = m[1].split('|');
129
+ if (alts.length < 2)
130
+ continue;
131
+ if (alts.some((a) => GOOD.test(a)) && alts.some((a) => BAD.test(a)))
132
+ return `/${m[1]}/`;
133
+ }
134
+ return undefined;
135
+ }
136
+ /** Problems of each test in a file. `steps` are the testkit's openStep names, when known. */
137
+ /** The words of a regex name: its alternatives, without the regex syntax. */
138
+ const regexWords = (src) => src.split('|').map((a) => a.replace(/\\[sbdw]/g, ' ').replace(/[\\^$()[\]{}?*+.]/g, ' ').replace(/\s+/g, ' ').trim().toLowerCase()).filter((a) => /[a-z]{3}/.test(a));
139
+ /**
140
+ * Problems of each test in a file. `steps` are the testkit's openStep names,
141
+ * `knownTexts` the app's source and the doc, lower-cased: every name a test
142
+ * finds an element by must come from them.
143
+ */
144
+ export function testProblems(content, opts = {}) {
145
+ const out = [];
146
+ for (const block of findTestBlocks(content)) {
147
+ const text = content.slice(block.start, block.end);
148
+ const oracle = (/\/\/\s*oracle:\s*(.+)$/m.exec(text)?.[1] ?? '').toLowerCase();
149
+ const code = stripComments(text);
150
+ const seen = new Set();
151
+ const add = (reason) => {
152
+ if (seen.has(reason))
153
+ return;
154
+ seen.add(reason);
155
+ out.push({ title: block.title, rules: block.rules, reason, block });
156
+ };
157
+ // A stub's own words asserted back: the test proves the stub, not the app.
158
+ const stubbed = [...code.matchAll(/\.fulfill\(\s*\{([\s\S]*?)\}\s*\)/g)].flatMap((m) => [...m[1].matchAll(/(['"`])([^'"`\n]{6,})\1/g)].map((s) => s[2].toLowerCase()).filter((s) => /[a-z]{4,}\s/i.test(s)));
159
+ if (stubbed.length) {
160
+ const expected = [...code.matchAll(/(?:getByText|toHaveText|toContainText|toMatch)\s*\(\s*(?:\/((?:\\\/|[^/\n])+)\/|(['"`])([^'"`\n]+)\2)/g)]
161
+ .flatMap((m) => (m[1] ? m[1].split('|') : [m[3]]).map((t) => t.toLowerCase().replace(/\\/g, '').trim()))
162
+ .filter((t) => t.length >= 4);
163
+ const echo = expected.find((t) => stubbed.some((s) => s.includes(t)));
164
+ if (echo)
165
+ add(`asserts "${echo}", which its own page.route stub returns — the test proves the stub, not the app: assert what the app does with that state (what the doc says the screen shows)`);
166
+ }
167
+ // …or a value handed to a stub and then looked for on the screen.
168
+ const handed = new Set([...code.matchAll(/\b(?:stub\w*|mock\w*|fulfill|route)\s*\(([^;]*?)\)\s*;/gi)]
169
+ .flatMap((m) => m[1].match(/\b[a-z_$][\w$]*\b/gi) ?? [])
170
+ .filter((v) => !/^(page|route|request|url|r|req|status|body|json|true|false|null|undefined|JSON|stringify|contentType|async|await|const)$/.test(v)));
171
+ const shown = [...code.matchAll(/(?:getByText|toHaveText|toContainText)\s*\(\s*([a-z_$][\w$]*)\s*[,)]/gi)].map((m) => m[1]);
172
+ const back = shown.find((v) => handed.has(v));
173
+ if (back)
174
+ add(`looks on the screen for ${back}, the value its own stub returned — the test proves the stub, not the app: assert what the app decides with that state`);
175
+ // Q6 — kit.find returns a promise.
176
+ // `await kit.find(…).first()` is wrong too: the await applies to the whole chain.
177
+ if (/\bkit\.find\([^)]*\)\s*\.(?!then\b)/.test(code) || /expect\(\s*kit\.find\(/.test(code)) {
178
+ add('uses kit.find(...) without await: it returns a promise — write `const x = await kit.find(page, kind, name)` first, then use x');
179
+ }
180
+ // …kept in a variable without await is the same mistake.
181
+ if (/\b(?:const|let|var)\s+\w+\s*=\s*kit\.find\(/.test(code)) {
182
+ add('keeps kit.find(...) in a variable without await: it is a promise, not the element — `const x = await kit.find(...)`');
183
+ }
184
+ // Q10 — the name an element is found by comes from the screen's source or the doc.
185
+ for (const m of code.matchAll(/\bkit\.find\(\s*\w+\s*,\s*['"`][\w-]+['"`]\s*,\s*/g)) {
186
+ const rest = code.slice(m.index + m[0].length);
187
+ const v = /^([A-Za-z_$][\w$]*)\s*[,)]/.exec(rest)?.[1];
188
+ // A name from a list written out in the test (a parametrised loop, a const) is fine.
189
+ const writtenOut = v && (new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
190
+ new RegExp(`for\\s*\\(\\s*(?:const|let)\\s+(?:${v}|\\[[^\\]]*\\b${v}\\b[^\\]]*\\])\\s+of\\s+\\[\\s*['"\`\\[]`).test(code));
191
+ if (/^[A-Za-z_$]/.test(rest) && !writtenOut) {
192
+ add("finds an element by a name held in a variable: take the name the screen shows from the CODE FACTS or the doc, written out — a value the test made up (kit.unique, a stub's id) is never a control's name");
193
+ continue;
194
+ }
195
+ const re = /^\/((?:\\\/|[^/\n])+)\//.exec(rest);
196
+ if (re && opts.knownTexts && !regexWords(re[1]).some((w) => opts.knownTexts.includes(w))) {
197
+ add(`finds an element by /${re[1]}/, which matches no text in the app's source or the doc: use the name the screen really shows (the CODE FACTS give the component's text)`);
198
+ }
199
+ }
200
+ // Q11 — a logged-out state reached by wiping storage: the testkit's login seed puts the session back.
201
+ if (/kit\.login\(/.test(code) && /\b(?:clearCookies\s*\(|(?:localStorage|sessionStorage)\.clear\s*\(|storageState\s*\(\s*\{\s*\}|removeItem\s*\()/.test(code)) {
202
+ add("logs out by clearing storage or cookies after kit.login: the testkit's login seed restores the session — log out through the app's own Log Out control");
203
+ }
204
+ // Q12 — one tab's login reaching another: kit.login prepares one page, so it cannot be set up.
205
+ if (/\b(?:other|second|another|both)\s+tabs?\b|\belsewhere\b/.test(oracle) && /\bnewPage\s*\(/.test(code) && /kit\.login\(/.test(code)) {
206
+ add("tests a login or logout in one tab reaching another: kit.login prepares one page, so the testkit cannot set that up — give the rule a cannot-test line");
207
+ }
208
+ // Q7 — only steps the testkit has.
209
+ if (opts.steps?.length) {
210
+ for (const m of code.matchAll(/\bopenStep\(\s*\w+\s*,\s*(['"`])([^'"`]+)\1/g)) {
211
+ if (!opts.steps.includes(m[2]))
212
+ add(`calls kit.openStep with step "${m[2]}", which the testkit does not have (its steps: ${opts.steps.join(', ')})`);
213
+ }
214
+ }
215
+ // Q8 — a skip decided by what the page shows.
216
+ for (const m of code.matchAll(/\btest\.skip\s*\(/g)) {
217
+ const close = matchParen(code, m.index + m[0].length - 1);
218
+ const cond = close === -1 ? '' : code.slice(m.index + m[0].length, close).split(/,\s*['"`]/)[0];
219
+ const flag = /^\s*!?\s*\(?\s*([A-Za-z_$][\w$]*)/.exec(cond)?.[1];
220
+ const fromText = flag && new RegExp(`\\b(?:const|let|var)\\s+${flag}\\s*=[^;\\n]*(?:innerText|textContent|allInnerTexts|allTextContents)\\(`).test(code);
221
+ if (/innerText\(|textContent\(|allInnerTexts\(|allTextContents\(/.test(cond) || fromText) {
222
+ add('skips itself because of what the page shows: what the doc expects on the page is asserted, never a reason to skip');
223
+ break;
224
+ }
225
+ }
226
+ // Q9 — real data is made by the run account and recorded, never by the shared fixture.
227
+ const createsData = /(?:find\(\s*\w+\s*,\s*['"`]button['"`]\s*,\s*|getByRole\(\s*['"`]button['"`]\s*,\s*\{\s*name:\s*)['"`/]\s*(?:pay\b|book(?!\s*again)\b|place\b|confirm\s+(?:payment|booking)|complete\s+(?:payment|booking)|submit\s+order)/i.test(code);
228
+ if (createsData && /\.click\(|settledClick\(/.test(code) && !/\.route\(/.test(code) && !/['"`]run['"`]/.test(code) && !/kit\.(?:unique|created)\(/.test(code)) {
229
+ add("clicks a button that creates a booking or a payment, but not as the testkit's 'run' role with kit.unique/kit.created: real data is made by the run account and recorded so it can be cleaned up");
230
+ }
231
+ // Q2 — an assertion that always runs.
232
+ const top = withoutBranches(code);
233
+ if (!/\bexpect(?:\.soft)?\s*\(|\bexpect\.poll\s*\(/.test(top)) {
234
+ add(/\bexpect\s*\(/.test(code)
235
+ ? 'its only assertions sit inside an if, else or catch, so the test can pass without checking anything — assert unconditionally'
236
+ : 'has no assertion');
237
+ continue;
238
+ }
239
+ // Q3 — one assertion that accepts success and failure.
240
+ const either = eitherOutcome(code);
241
+ if (either && !(GOOD.test(oracle) && BAD.test(oracle) && /\bor\b/.test(oracle))) {
242
+ add(`asserts ${either}, which matches both the good and the bad outcome — assert the one outcome the rule expects`);
243
+ }
244
+ // Q4 — the page's whole text compared with itself.
245
+ if (/locator\(\s*['"`](?:body|main|html)['"`]\s*\)\s*\.(?:innerText|textContent)\(/.test(code) && /\.split\(\s*['"`]\\n['"`]\s*\)|toContainText\(\s*\w+/.test(code)) {
246
+ add("compares the page's text with itself (body/main innerText): read the element the rule is about (the operator's name, the total) and compare that");
247
+ }
248
+ // Q5 — only that something is absent.
249
+ const asserts = [...top.matchAll(/expect(?:\.soft)?\s*\(([\s\S]*?)\)\s*\.\s*((?:not\.)?\w+)/g)];
250
+ const positive = asserts.filter((a) => !a[2].startsWith('not.') && !/^toHaveCount$/.test(a[2]) || /toHaveCount\(\s*[1-9]/.test(top));
251
+ if (asserts.length && !positive.length && !/\b(never|no|not|without|cannot|isn't|doesn't|none|nothing|hidden|absent|disappear|removed)\b/.test(oracle)) {
252
+ add('only asserts that something is absent: also assert the outcome the rule states is there');
253
+ }
254
+ // Always true, whatever the app does.
255
+ if (/expect\(\s*new Set\(\s*(\w+)\s*\)\.size\s*\)\.toBeLessThanOrEqual\(\s*\1\.length\s*\)/.test(code) || /expect\([^)]*\b(?:false\s*\|\||true\s*&&)/.test(code) || /\.toBeDefined\(\)/.test(top) && !/expect\([^)]*\)\.(?!toBeDefined)\w+/.test(top)) {
256
+ add('has an assertion that cannot fail (a set is never larger than its list; false || x; only toBeDefined) — assert the outcome itself');
257
+ }
258
+ }
259
+ return out;
260
+ }
261
+ /**
262
+ * The step names a testkit's openStep accepts: the keys of the object its
263
+ * openStep looks the step up in (if (!STEPS[step]) …).
264
+ */
265
+ export function harnessSteps(harnessSource) {
266
+ // The contract's own export first (STEPS), else the object openStep looks the step up in.
267
+ let map = /(?:const|let|var)\s+STEPS\s*=\s*\{/.test(harnessSource) ? 'STEPS' : undefined;
268
+ if (!map) {
269
+ const fn = /function\s+openStep\s*\(\s*\w+\s*,\s*(\w+)/.exec(harnessSource);
270
+ if (!fn)
271
+ return [];
272
+ const body = harnessSource.slice(fn.index, fn.index + 1500);
273
+ map = new RegExp(`!\\s*(\\w+)\\[\\s*${fn[1]}\\s*\\]`).exec(body)?.[1];
274
+ }
275
+ if (!map)
276
+ return [];
277
+ const decl = new RegExp(`(?:const|let|var)\\s+${map}\\s*=\\s*\\{`).exec(harnessSource);
278
+ if (!decl)
279
+ return [];
280
+ const open = decl.index + decl[0].length - 1;
281
+ const close = matchBrace(harnessSource, open);
282
+ if (close === -1)
283
+ return [];
284
+ return [...harnessSource.slice(open + 1, close).matchAll(/^\s*(?:(['"])([^'"]+)\1|([A-Za-z_$][\w$-]*))\s*:/gm)].map((m) => m[2] ?? m[3]);
285
+ }
286
+ /** One refusal naming every problem, so a single rewrite fixes them all. */
287
+ export function describeProblems(problems, coverage) {
288
+ const lines = [];
289
+ if (coverage?.missing.length) {
290
+ lines.push(`rules with no test: ${coverage.missing.join(', ')} — write a test for each, or a "// cannot-test: <id> — <the state this area cannot set up>" line. "The locator is unknown" is not such a state (find it with kit.find by the doc's name), and neither is "needs two customers" (skip unless kit.distinctRunAccount())`);
291
+ }
292
+ for (const p of problems)
293
+ lines.push(`"${p.title}"${p.rules.length ? ` (${p.rules.join(', ')})` : ''} ${p.reason}`);
294
+ return lines.join('\n');
295
+ }