@activate-core/test-forge 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -387,10 +387,26 @@ history, the plan, and the code facts. No later step sends the whole doc again.
387
387
  with the reason, if it sleeps for a fixed time, skips because an element is
388
388
  missing, has no test or no assertion, spends more than `maxTestsPerRule`
389
389
  tests on one rule, or names a label found neither in the code nor the doc.
390
- 6. **A changed doc.** Only what changed is touched: a renamed label is replaced
390
+ 6. **Every rule, a test that can fail.** Each test is checked on its own:
391
+ every planned rule has a test, or a `// cannot-test: R7 — <the state this
392
+ area cannot set up>` line ("the locator is unknown" is not such a state);
393
+ no assertion accepts the good and the bad outcome alike, compares the page's
394
+ whole text, sits only inside an `if`, or repeats what its own stub returned;
395
+ `kit.find` is awaited and finds elements by the names the screen shows;
396
+ `kit.openStep` names a step the testkit has. The tests that fail a check are
397
+ taken out and their rules written again in ONE small call; what is still
398
+ wrong is removed and reported. The summary line says how many planned rules
399
+ have a test, which cannot be tested here and why, and which have none, and
400
+ `rules.json` / `plan-check.md` record the same.
401
+ 7. **A changed doc.** Only what changed is touched: a renamed label is replaced
391
402
  in its tests, a removed rule's tests are deleted (both without AI); a
392
403
  reworded or new rule gets its tests written by one small call.
393
404
 
405
+ **A feature that is not live yet.** Put `Status: not built` or
406
+ `Status: behind flag <name>` in the journey doc's header. Every file of the
407
+ journey is stamped `// testforge-pending: …`, and `testforge run` (and `bind`)
408
+ hold it back until the line is removed; `--all` runs it anyway.
409
+
394
410
  `generate` runs nothing. `testforge run` runs the suite; `testforge review`
395
411
  judges failures when you ask for it.
396
412
 
@@ -127,15 +127,18 @@ export const DEEP_DOC_GUIDANCE = [
127
127
  ` becomes a test that fails on a correct product.`,
128
128
  ` An [observed] criterion is about what the customer sees or what the`,
129
129
  ` server accepts or refuses — never an implementation detail (a storage`,
130
- ` key name, a CSS class, an internal flag, a request field name) unless the`,
131
- ` source itself is about it.`,
132
- ` Only what a test can drive is a criterion. Behaviour no screen or`,
133
- ` endpoint can reach ("not tested through the screen") is not tagged —`,
134
- ` leave it out, or mention it in Test setup if the generator needs to know.`,
130
+ ` key name, a CSS class, an internal flag, a request field name, a log`,
131
+ ` line, a lock, a cache) unless the source itself is about it.`,
132
+ ` Only what a test in THIS area can drive is a criterion. Behaviour no`,
133
+ ` screen or endpoint of this area can reach (a backend job, another app,`,
134
+ ` an operator's action) is not tagged — list it under "## Not requirements"`,
135
+ ` with where it is checked, or mention it in Test setup if the generator`,
136
+ ` needs to know.`,
135
137
  ``,
136
138
  `METHOD. Work through these steps for every journey, in order:`,
137
139
  `1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
138
- ` interactive element and where it appears: each input, button, link,`,
140
+ ` interactive element and where it appears, each row's first cell its id`,
141
+ ` C1, C2, C3 …: each input, button, link,`,
139
142
  ` checkbox, code box, dropdown, pop-up, menu item, and the page itself.`,
140
143
  ` Secondary controls count as much as the main form: Resend, Back,`,
141
144
  ` show/hide, close, apply, sign-out, provider buttons. Apply every`,
@@ -83,7 +83,7 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
83
83
  }
84
84
  // A file named after a control ("Escalate button" → escalateDisputeButton/) is
85
85
  // the component itself: found by its path, before any text search.
86
- const controlWords = [...markdown.matchAll(/^\|\s*C\d+[a-z]?\s*\|([^|]+)\|/gm)]
86
+ const controlWords = [...markdown.matchAll(/^\|\s*C?\d+[a-z]?\s*\|([^|]+)\|/gm)]
87
87
  .flatMap((m) => m[1].replace(/["“”(][^"”)]*["”)]?/g, ' ').match(/[A-Za-z]{5,}/g) ?? [])
88
88
  .map((w) => w.toLowerCase())
89
89
  .filter((w) => !GENERIC.has(w));
@@ -198,6 +198,55 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
198
198
  order.push(h.file);
199
199
  }
200
200
  }
201
+ // A page's own parts (its total, its operator card, its pickers) live in the
202
+ // components it imports from its own folder. Their visible texts and labels
203
+ // come right after it, ahead of files that only share a word with the doc.
204
+ const textLine = /(>\s*[A-Za-z£$€][^<>{}]{1,60}<)|\b(aria-label|placeholder|title|label|alt)\s*=\s*['"{]|\btitle:\s*['"`]|<(button|Button|a|Link|h[1-6]|label)\b/;
205
+ for (const f of order.slice(0, 6)) {
206
+ const dir = f.replace(/[\\/][^\\/]+$/, '');
207
+ let text;
208
+ try {
209
+ text = readFileSync(f, 'utf-8');
210
+ }
211
+ catch {
212
+ continue;
213
+ }
214
+ const children = [...text.matchAll(/from\s+['"]([^'"]+)['"]/g)]
215
+ .map((m) => m[1].replace(/^[@~.]+[\\/]?/, '').replace(/^src[\\/]/, ''))
216
+ .map((spec) => files.find((t) => t !== f && t.startsWith(dir) && t.replace(/\.\w+$/, '').replace(/[\\/]index$/, '').endsWith(spec)))
217
+ .filter((c) => !!c && /\.(tsx|jsx|vue|svelte)$/.test(c));
218
+ let at = order.indexOf(f);
219
+ for (const c of [...new Set(children)].slice(0, 12)) {
220
+ const lines = readFileSync(c, 'utf-8').split('\n');
221
+ lines.forEach((l, i) => { if (textLine.test(l))
222
+ add(c, i, i + 2); });
223
+ if (!windows.has(c))
224
+ continue;
225
+ const was = order.indexOf(c);
226
+ if (was !== -1 && was <= at)
227
+ continue;
228
+ if (was !== -1)
229
+ order.splice(was, 1);
230
+ order.splice(++at, 0, c);
231
+ }
232
+ }
233
+ // A hook or API module says WHERE the app sends each request, often far below
234
+ // its first lines: every line that makes a call or names a path is kept.
235
+ for (const f of order) {
236
+ if (!/(^|[\\/])(use[A-Z]\w*|api|services?|dataProvider|queries|requests?)([\\/.]|$)/.test(f))
237
+ continue;
238
+ let lines;
239
+ try {
240
+ lines = readFileSync(f, 'utf-8').split('\n');
241
+ }
242
+ catch {
243
+ continue;
244
+ }
245
+ lines.forEach((l, i) => {
246
+ if (/\b(?:get|post|put|patch|del|delete|fetch|request|axios(?:\.\w+)?)\s*(?:<[^>]*>)?\s*\(|['"`]\/[\w-]+\//.test(l))
247
+ add(f, i + 1 - 3, i + 1 + 3);
248
+ });
249
+ }
201
250
  const out = [];
202
251
  let used = 0;
203
252
  for (const file of order) {
@@ -37,7 +37,7 @@ const MAP_PATH = 'testforge.md';
37
37
  * that upgrades keeps serving the facts it cached under the old version —
38
38
  * so the new facts never reach a single prompt and the upgrade does nothing.
39
39
  */
40
- const FACTS_VERSION = 4;
40
+ const FACTS_VERSION = 5;
41
41
  /** Where a journey's grounded facts live, mirroring the plan cache layout. */
42
42
  function groundingPath(journey) {
43
43
  return join(journeyDir(journey), 'facts.json');
@@ -167,6 +167,12 @@ reuseOnly = false) {
167
167
  const excerpts = sourceExcerpts(journey.rawMarkdown || '', [srcDir, ...(config.relatedDirs ?? [])]);
168
168
  if (!excerpts)
169
169
  return null;
170
+ // Excerpts only from other packages (a backend matched on a quoted text) say
171
+ // nothing about this area's screens: a call on them returns "unknown" for
172
+ // every section and costs as much as a real one.
173
+ const files = [...excerpts.matchAll(/^### (.+)$/gm)].map((m) => m[1]);
174
+ if (files.length && files.every((f) => f.startsWith('..')))
175
+ return null;
170
176
  const { program, args } = withNoTools(splitCommand(config.aiCommand));
171
177
  let facts;
172
178
  try {
@@ -0,0 +1,295 @@
1
+ /**
2
+ * testforge — does each written test really check its rule?
3
+ *
4
+ * The file-level gates see a file as text. These checks read each test on its
5
+ * own, against the rules the file was planned for:
6
+ *
7
+ * - every planned rule has a test, or a "// cannot-test: R7 — <what is
8
+ * missing>" line that says why not (Q1);
9
+ * - each test has an assertion that runs, not one only inside an if or a
10
+ * catch (Q2);
11
+ * - no assertion accepts the rule's success and its failure alike, like
12
+ * /confirm|time not available/ (Q3);
13
+ * - no test compares the page's whole text with itself (Q4);
14
+ * - a test does more than assert that something is absent, unless its rule
15
+ * says never / no / not (Q5);
16
+ * - kit.find is awaited: it returns a promise (Q6);
17
+ * - kit.openStep names a step the testkit has (Q7);
18
+ * - a run-time skip does not depend on what the page shows (Q8).
19
+ *
20
+ * Each problem names its test and its rules, so only those tests are written
21
+ * again; the rest of the file is kept.
22
+ */
23
+ import { matchParen } from './validate.js';
24
+ import { findTestBlocks } from '../generate/test-blocks.js';
25
+ const CANNOT = /\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)$/gm;
26
+ /** Which planned rules have a test, which are explained, which are simply missing. */
27
+ /**
28
+ * Reasons that are not a state the area cannot set up: an element is found by
29
+ * the doc's name with kit.find, and two people at once is a run-time skip on
30
+ * kit.distinctRunAccount(). A rule given one of these is still missing.
31
+ */
32
+ const NOT_A_REASON = [
33
+ /\b(?:locators?|selectors?|test ?ids?|elements?|controls?|handles?|buttons?|fields?)\b[^.;]*\b(?:unknown|not known|not in the code facts|not listed|not documented|not named|named nowhere|not exposed|not identified)\b/i,
34
+ /\bno (?:known |documented )?(?:locator|selector|element|control|handle)\b/i,
35
+ /\bknown (?:[\w-]+ )?(?:locator|selector|element|control|handle)\b/i,
36
+ /\b(?:second|two|distinct|another|different) (?:\w+ )?(?:customer|account|user|people|person)s?\b/i,
37
+ ];
38
+ export const acceptedReason = (why) => !NOT_A_REASON.some((re) => re.test(why));
39
+ export function ruleCoverage(content, planned) {
40
+ const tested = new Set(findTestBlocks(content).flatMap((b) => b.rules));
41
+ const cannotTest = [];
42
+ for (const m of content.matchAll(CANNOT)) {
43
+ for (const id of m[1].split(/\s*,\s*/))
44
+ if (planned.includes(id) && !tested.has(id) && acceptedReason(m[2]))
45
+ cannotTest.push({ id, why: m[2].trim() });
46
+ }
47
+ const explained = new Set(cannotTest.map((c) => c.id));
48
+ return { missing: planned.filter((id) => !tested.has(id) && !explained.has(id)), cannotTest };
49
+ }
50
+ /** The index of the brace closing the one at `open`, skipping strings and comments. */
51
+ function matchBrace(s, open) {
52
+ let depth = 0;
53
+ for (let i = open; i < s.length; i++) {
54
+ const c = s[i];
55
+ if (c === "'" || c === '"' || c === '`') {
56
+ for (i++; i < s.length && s[i] !== c; i++)
57
+ if (s[i] === '\\')
58
+ i++;
59
+ continue;
60
+ }
61
+ if (c === '/' && s[i + 1] === '/') {
62
+ while (i < s.length && s[i] !== '\n')
63
+ i++;
64
+ continue;
65
+ }
66
+ if (c === '/' && s[i + 1] === '*') {
67
+ i = s.indexOf('*/', i + 2);
68
+ if (i === -1)
69
+ return -1;
70
+ i++;
71
+ continue;
72
+ }
73
+ if (c === '{')
74
+ depth++;
75
+ else if (c === '}' && --depth === 0)
76
+ return i;
77
+ }
78
+ return -1;
79
+ }
80
+ /** The code with every if/else/try/catch body (and .catch(…) callback) taken out. */
81
+ function withoutBranches(code) {
82
+ let out = code;
83
+ for (;;) {
84
+ const m = /\b(?:if\s*\(|else\b|try\b|catch\b|\.catch\s*\()/.exec(out);
85
+ if (!m)
86
+ return out;
87
+ let at = m.index + m[0].length;
88
+ if (/\($/.test(m[0])) {
89
+ const close = matchParen(out, at - 1);
90
+ if (close === -1)
91
+ return out;
92
+ at = close + 1;
93
+ if (m[0].startsWith('.catch')) {
94
+ out = out.slice(0, m.index) + out.slice(at);
95
+ continue;
96
+ }
97
+ }
98
+ else if (/^catch$/.test(m[0]) && /^\s*\(/.test(out.slice(at))) {
99
+ const close = matchParen(out, out.indexOf('(', at));
100
+ if (close === -1)
101
+ return out;
102
+ at = close + 1;
103
+ }
104
+ const rest = out.slice(at);
105
+ const brace = /^\s*\{/.exec(rest);
106
+ if (brace) {
107
+ const open = at + brace[0].length - 1;
108
+ const close = matchBrace(out, open);
109
+ if (close === -1)
110
+ return out;
111
+ out = out.slice(0, m.index) + out.slice(close + 1);
112
+ }
113
+ else {
114
+ // A one-statement branch: drop up to the end of the statement.
115
+ const end = out.indexOf(';', at);
116
+ out = out.slice(0, m.index) + (end === -1 ? '' : out.slice(end + 1));
117
+ }
118
+ }
119
+ }
120
+ const stripComments = (code) => code.replace(/^\s*\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '');
121
+ /** Words for a good outcome and for a bad one: one assertion that accepts both checks nothing. */
122
+ const GOOD = /confirm|success|placed|booked|saved|created|thank|welcome|done|complete|sent/i;
123
+ const BAD = /not available|unavailable|error|fail|declin|invalid|denied|refus|reject|sorry|cannot|can't|unable|wrong|taken/i;
124
+ /** Regex literals inside assertion-like calls that offer alternatives. */
125
+ function eitherOutcome(code) {
126
+ const calls = /(?:getByText|toHaveText|toContainText|toMatch|toHaveURL|toHaveTitle|test)\s*\(\s*\/((?:\\\/|[^/\n])+)\/[a-z]*/g;
127
+ for (const m of code.matchAll(calls)) {
128
+ const alts = m[1].split('|');
129
+ if (alts.length < 2)
130
+ continue;
131
+ if (alts.some((a) => GOOD.test(a)) && alts.some((a) => BAD.test(a)))
132
+ return `/${m[1]}/`;
133
+ }
134
+ return undefined;
135
+ }
136
+ /** Problems of each test in a file. `steps` are the testkit's openStep names, when known. */
137
+ /** The words of a regex name: its alternatives, without the regex syntax. */
138
+ const regexWords = (src) => src.split('|').map((a) => a.replace(/\\[sbdw]/g, ' ').replace(/[\\^$()[\]{}?*+.]/g, ' ').replace(/\s+/g, ' ').trim().toLowerCase()).filter((a) => /[a-z]{3}/.test(a));
139
+ /**
140
+ * Problems of each test in a file. `steps` are the testkit's openStep names,
141
+ * `knownTexts` the app's source and the doc, lower-cased: every name a test
142
+ * finds an element by must come from them.
143
+ */
144
+ export function testProblems(content, opts = {}) {
145
+ const out = [];
146
+ for (const block of findTestBlocks(content)) {
147
+ const text = content.slice(block.start, block.end);
148
+ const oracle = (/\/\/\s*oracle:\s*(.+)$/m.exec(text)?.[1] ?? '').toLowerCase();
149
+ const code = stripComments(text);
150
+ const seen = new Set();
151
+ const add = (reason) => {
152
+ if (seen.has(reason))
153
+ return;
154
+ seen.add(reason);
155
+ out.push({ title: block.title, rules: block.rules, reason, block });
156
+ };
157
+ // A stub's own words asserted back: the test proves the stub, not the app.
158
+ const stubbed = [...code.matchAll(/\.fulfill\(\s*\{([\s\S]*?)\}\s*\)/g)].flatMap((m) => [...m[1].matchAll(/(['"`])([^'"`\n]{6,})\1/g)].map((s) => s[2].toLowerCase()).filter((s) => /[a-z]{4,}\s/i.test(s)));
159
+ if (stubbed.length) {
160
+ const expected = [...code.matchAll(/(?:getByText|toHaveText|toContainText|toMatch)\s*\(\s*(?:\/((?:\\\/|[^/\n])+)\/|(['"`])([^'"`\n]+)\2)/g)]
161
+ .flatMap((m) => (m[1] ? m[1].split('|') : [m[3]]).map((t) => t.toLowerCase().replace(/\\/g, '').trim()))
162
+ .filter((t) => t.length >= 4);
163
+ const echo = expected.find((t) => stubbed.some((s) => s.includes(t)));
164
+ if (echo)
165
+ add(`asserts "${echo}", which its own page.route stub returns — the test proves the stub, not the app: assert what the app does with that state (what the doc says the screen shows)`);
166
+ }
167
+ // …or a value handed to a stub and then looked for on the screen.
168
+ const handed = new Set([...code.matchAll(/\b(?:stub\w*|mock\w*|fulfill|route)\s*\(([^;]*?)\)\s*;/gi)]
169
+ .flatMap((m) => m[1].match(/\b[a-z_$][\w$]*\b/gi) ?? [])
170
+ .filter((v) => !/^(page|route|request|url|r|req|status|body|json|true|false|null|undefined|JSON|stringify|contentType|async|await|const)$/.test(v)));
171
+ const shown = [...code.matchAll(/(?:getByText|toHaveText|toContainText)\s*\(\s*([a-z_$][\w$]*)\s*[,)]/gi)].map((m) => m[1]);
172
+ const back = shown.find((v) => handed.has(v));
173
+ if (back)
174
+ add(`looks on the screen for ${back}, the value its own stub returned — the test proves the stub, not the app: assert what the app decides with that state`);
175
+ // Q6 — kit.find returns a promise.
176
+ // `await kit.find(…).first()` is wrong too: the await applies to the whole chain.
177
+ if (/\bkit\.find\([^)]*\)\s*\.(?!then\b)/.test(code) || /expect\(\s*kit\.find\(/.test(code)) {
178
+ add('uses kit.find(...) without await: it returns a promise — write `const x = await kit.find(page, kind, name)` first, then use x');
179
+ }
180
+ // …kept in a variable without await is the same mistake.
181
+ if (/\b(?:const|let|var)\s+\w+\s*=\s*kit\.find\(/.test(code)) {
182
+ add('keeps kit.find(...) in a variable without await: it is a promise, not the element — `const x = await kit.find(...)`');
183
+ }
184
+ // Q10 — the name an element is found by comes from the screen's source or the doc.
185
+ for (const m of code.matchAll(/\bkit\.find\(\s*\w+\s*,\s*['"`][\w-]+['"`]\s*,\s*/g)) {
186
+ const rest = code.slice(m.index + m[0].length);
187
+ const v = /^([A-Za-z_$][\w$]*)\s*[,)]/.exec(rest)?.[1];
188
+ // A name from a list written out in the test (a parametrised loop, a const) is fine.
189
+ const writtenOut = v && (new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
190
+ new RegExp(`for\\s*\\(\\s*(?:const|let)\\s+(?:${v}|\\[[^\\]]*\\b${v}\\b[^\\]]*\\])\\s+of\\s+\\[\\s*['"\`\\[]`).test(code));
191
+ if (/^[A-Za-z_$]/.test(rest) && !writtenOut) {
192
+ add("finds an element by a name held in a variable: take the name the screen shows from the CODE FACTS or the doc, written out — a value the test made up (kit.unique, a stub's id) is never a control's name");
193
+ continue;
194
+ }
195
+ const re = /^\/((?:\\\/|[^/\n])+)\//.exec(rest);
196
+ if (re && opts.knownTexts && !regexWords(re[1]).some((w) => opts.knownTexts.includes(w))) {
197
+ add(`finds an element by /${re[1]}/, which matches no text in the app's source or the doc: use the name the screen really shows (the CODE FACTS give the component's text)`);
198
+ }
199
+ }
200
+ // Q11 — a logged-out state reached by wiping storage: the testkit's login seed puts the session back.
201
+ if (/kit\.login\(/.test(code) && /\b(?:clearCookies\s*\(|(?:localStorage|sessionStorage)\.clear\s*\(|storageState\s*\(\s*\{\s*\}|removeItem\s*\()/.test(code)) {
202
+ add("logs out by clearing storage or cookies after kit.login: the testkit's login seed restores the session — log out through the app's own Log Out control");
203
+ }
204
+ // Q12 — one tab's login reaching another: kit.login prepares one page, so it cannot be set up.
205
+ if (/\b(?:other|second|another|both)\s+tabs?\b|\belsewhere\b/.test(oracle) && /\bnewPage\s*\(/.test(code) && /kit\.login\(/.test(code)) {
206
+ add("tests a login or logout in one tab reaching another: kit.login prepares one page, so the testkit cannot set that up — give the rule a cannot-test line");
207
+ }
208
+ // Q7 — only steps the testkit has.
209
+ if (opts.steps?.length) {
210
+ for (const m of code.matchAll(/\bopenStep\(\s*\w+\s*,\s*(['"`])([^'"`]+)\1/g)) {
211
+ if (!opts.steps.includes(m[2]))
212
+ add(`calls kit.openStep with step "${m[2]}", which the testkit does not have (its steps: ${opts.steps.join(', ')})`);
213
+ }
214
+ }
215
+ // Q8 — a skip decided by what the page shows.
216
+ for (const m of code.matchAll(/\btest\.skip\s*\(/g)) {
217
+ const close = matchParen(code, m.index + m[0].length - 1);
218
+ const cond = close === -1 ? '' : code.slice(m.index + m[0].length, close).split(/,\s*['"`]/)[0];
219
+ const flag = /^\s*!?\s*\(?\s*([A-Za-z_$][\w$]*)/.exec(cond)?.[1];
220
+ const fromText = flag && new RegExp(`\\b(?:const|let|var)\\s+${flag}\\s*=[^;\\n]*(?:innerText|textContent|allInnerTexts|allTextContents)\\(`).test(code);
221
+ if (/innerText\(|textContent\(|allInnerTexts\(|allTextContents\(/.test(cond) || fromText) {
222
+ add('skips itself because of what the page shows: what the doc expects on the page is asserted, never a reason to skip');
223
+ break;
224
+ }
225
+ }
226
+ // Q9 — real data is made by the run account and recorded, never by the shared fixture.
227
+ const createsData = /(?:find\(\s*\w+\s*,\s*['"`]button['"`]\s*,\s*|getByRole\(\s*['"`]button['"`]\s*,\s*\{\s*name:\s*)['"`/]\s*(?:pay\b|book(?!\s*again)\b|place\b|confirm\s+(?:payment|booking)|complete\s+(?:payment|booking)|submit\s+order)/i.test(code);
228
+ if (createsData && /\.click\(|settledClick\(/.test(code) && !/\.route\(/.test(code) && !/['"`]run['"`]/.test(code) && !/kit\.(?:unique|created)\(/.test(code)) {
229
+ add("clicks a button that creates a booking or a payment, but not as the testkit's 'run' role with kit.unique/kit.created: real data is made by the run account and recorded so it can be cleaned up");
230
+ }
231
+ // Q2 — an assertion that always runs.
232
+ const top = withoutBranches(code);
233
+ if (!/\bexpect(?:\.soft)?\s*\(|\bexpect\.poll\s*\(/.test(top)) {
234
+ add(/\bexpect\s*\(/.test(code)
235
+ ? 'its only assertions sit inside an if, else or catch, so the test can pass without checking anything — assert unconditionally'
236
+ : 'has no assertion');
237
+ continue;
238
+ }
239
+ // Q3 — one assertion that accepts success and failure.
240
+ const either = eitherOutcome(code);
241
+ if (either && !(GOOD.test(oracle) && BAD.test(oracle) && /\bor\b/.test(oracle))) {
242
+ add(`asserts ${either}, which matches both the good and the bad outcome — assert the one outcome the rule expects`);
243
+ }
244
+ // Q4 — the page's whole text compared with itself.
245
+ if (/locator\(\s*['"`](?:body|main|html)['"`]\s*\)\s*\.(?:innerText|textContent)\(/.test(code) && /\.split\(\s*['"`]\\n['"`]\s*\)|toContainText\(\s*\w+/.test(code)) {
246
+ add("compares the page's text with itself (body/main innerText): read the element the rule is about (the operator's name, the total) and compare that");
247
+ }
248
+ // Q5 — only that something is absent.
249
+ const asserts = [...top.matchAll(/expect(?:\.soft)?\s*\(([\s\S]*?)\)\s*\.\s*((?:not\.)?\w+)/g)];
250
+ const positive = asserts.filter((a) => !a[2].startsWith('not.') && !/^toHaveCount$/.test(a[2]) || /toHaveCount\(\s*[1-9]/.test(top));
251
+ if (asserts.length && !positive.length && !/\b(never|no|not|without|cannot|isn't|doesn't|none|nothing|hidden|absent|disappear|removed)\b/.test(oracle)) {
252
+ add('only asserts that something is absent: also assert the outcome the rule states is there');
253
+ }
254
+ // Always true, whatever the app does.
255
+ if (/expect\(\s*new Set\(\s*(\w+)\s*\)\.size\s*\)\.toBeLessThanOrEqual\(\s*\1\.length\s*\)/.test(code) || /expect\([^)]*\b(?:false\s*\|\||true\s*&&)/.test(code) || /\.toBeDefined\(\)/.test(top) && !/expect\([^)]*\)\.(?!toBeDefined)\w+/.test(top)) {
256
+ add('has an assertion that cannot fail (a set is never larger than its list; false || x; only toBeDefined) — assert the outcome itself');
257
+ }
258
+ }
259
+ return out;
260
+ }
261
+ /**
262
+ * The step names a testkit's openStep accepts: the keys of the object its
263
+ * openStep looks the step up in (if (!STEPS[step]) …).
264
+ */
265
+ export function harnessSteps(harnessSource) {
266
+ // The contract's own export first (STEPS), else the object openStep looks the step up in.
267
+ let map = /(?:const|let|var)\s+STEPS\s*=\s*\{/.test(harnessSource) ? 'STEPS' : undefined;
268
+ if (!map) {
269
+ const fn = /function\s+openStep\s*\(\s*\w+\s*,\s*(\w+)/.exec(harnessSource);
270
+ if (!fn)
271
+ return [];
272
+ const body = harnessSource.slice(fn.index, fn.index + 1500);
273
+ map = new RegExp(`!\\s*(\\w+)\\[\\s*${fn[1]}\\s*\\]`).exec(body)?.[1];
274
+ }
275
+ if (!map)
276
+ return [];
277
+ const decl = new RegExp(`(?:const|let|var)\\s+${map}\\s*=\\s*\\{`).exec(harnessSource);
278
+ if (!decl)
279
+ return [];
280
+ const open = decl.index + decl[0].length - 1;
281
+ const close = matchBrace(harnessSource, open);
282
+ if (close === -1)
283
+ return [];
284
+ return [...harnessSource.slice(open + 1, close).matchAll(/^\s*(?:(['"])([^'"]+)\1|([A-Za-z_$][\w$-]*))\s*:/gm)].map((m) => m[2] ?? m[3]);
285
+ }
286
+ /** One refusal naming every problem, so a single rewrite fixes them all. */
287
+ export function describeProblems(problems, coverage) {
288
+ const lines = [];
289
+ if (coverage?.missing.length) {
290
+ lines.push(`rules with no test: ${coverage.missing.join(', ')} — write a test for each, or a "// cannot-test: <id> — <the state this area cannot set up>" line. "The locator is unknown" is not such a state (find it with kit.find by the doc's name), and neither is "needs two customers" (skip unless kit.distinctRunAccount())`);
291
+ }
292
+ for (const p of problems)
293
+ lines.push(`"${p.title}"${p.rules.length ? ` (${p.rules.join(', ')})` : ''} ${p.reason}`);
294
+ return lines.join('\n');
295
+ }
@@ -127,9 +127,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
127
127
  ` how to establish preconditions — follow it.`,
128
128
  ` - Assert what the line states, at the loosest form that still proves it.`,
129
129
  ` Add no condition it does not state, and no exact copy, ordering, id,`,
130
- ` timestamp or count it does not fix. A [baseline] robustness line ("opens`,
131
- ` without a crash") is proved by the page loading with no error and no`,
132
- ` 5xx, not by a URL, heading or element it never names. "The customer is`,
130
+ ` timestamp or count it does not fix. Only a line that says exactly "opens`,
131
+ ` without a crash" is proved by the page loading with no error and no 5xx;`,
132
+ ` every other line is proved by its own outcome, never by "no 5xx". "The customer is`,
133
133
  ` told why" is proved by a non-empty message VISIBLE where the app shows`,
134
134
  ` it, unless the line fixes the words. Check where the page actually`,
135
135
  ` renders a value before asserting it there.`,
@@ -252,10 +252,25 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
252
252
  ``,
253
253
  `A LATER STEP OF A FLOW IS OPENED THROUGH THE TESTKIT. Never page.goto() a`,
254
254
  `later step of a multi-step flow by its URL alone: call`,
255
- `kit.openStep(page, step, input) and skip with its reason when it does not`,
256
- `resolve true. Only a test that checks the guard itself (what a visitor who`,
255
+ `kit.openStep(page, step, input) with a step name the TESTKIT lists for it`,
256
+ `(no other name exists), and skip with its reason when it does not resolve`,
257
+ `true. A screen after the last listed step is reached from that step through`,
258
+ `the UI. Only a test that checks the guard itself (what a visitor who`,
257
259
  `skipped the earlier steps sees) opens the bare URL.`,
258
260
  ``,
261
+ `kit.find RETURNS A PROMISE. Always \`const pay = await kit.find(page, 'button',`,
262
+ `'Pay')\` first, then use pay. Never chain on kit.find(...) (.first(), .click()),`,
263
+ `never keep it in a variable without await, and never pass it to expect()`,
264
+ `without await.`,
265
+ ``,
266
+ `kit.login PREPARES ONE PAGE. It sets the session for the page it is given and`,
267
+ `restores it when that page loads again. So:`,
268
+ ` - log out through the app's own Log Out control, never by clearing storage or`,
269
+ ` cookies (the restored session comes straight back);`,
270
+ ` - a second tab or page is not logged in by it: call kit.login on that page`,
271
+ ` too. "Logged in in another tab" — one tab's login reaching another — is not`,
272
+ ` something the testkit can set up: such a rule gets a cannot-test line.`,
273
+ ``,
259
274
  `DATA A TEST CREATES. Other runs use the same accounts, so what they left`,
260
275
  `behind is already there when this test starts.`,
261
276
  ` - A test that creates a record (a booking, an order, a vehicle, an`,
@@ -268,6 +283,10 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
268
283
  ` or a list being empty. "Exactly one booking for this request" is checked`,
269
284
  ` among the records that carry this test's unique value.`,
270
285
  ` - A test that only reads data uses the fixture role and the seeded ids.`,
286
+ ` - Two DIFFERENT people at once (two customers paying for one slot): log one`,
287
+ ` in as the fixture role and one as "run", and skip at the top unless`,
288
+ ` kit.distinctRunAccount() is true (the environment has no second account).`,
289
+ ` A TESTKIT without distinctRunAccount gives that rule a cannot-test line.`,
271
290
  ``,
272
291
  `AT MOST ${maxPerRule()} TESTS FOR ONE CRITERION. Two tests whose oracle quotes the same`,
273
292
  `criterion must reach different branches; more inputs for that criterion go`,
@@ -294,11 +313,36 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
294
313
  ` declarations. A precondition the environment lacks is a run-time skip`,
295
314
  ` inside a real test, stating what was observed.`,
296
315
  ``,
316
+ `EVERY RULE GETS A TEST THAT CAN FAIL. Each rule id this file is given gets`,
317
+ `at least one test, and that test fails when the product breaks the rule.`,
318
+ ` - A rule whose STATE this area cannot set up — data that lives in another`,
319
+ ` system, a backend-only effect the screen never shows, an external service`,
320
+ ` it cannot drive — gets ONE line instead of a test:`,
321
+ ` // cannot-test: R7 — <the state this area cannot set up>`,
322
+ ` An element, locator or route you were not shown is NEVER that reason:`,
323
+ ` find the element with kit.find(page, kind, <the name the doc gives it>),`,
324
+ ` which needs nothing from the CODE FACTS. Two different people at once`,
325
+ ` is not that reason either: see kit.distinctRunAccount() below.`,
326
+ ` Never leave a rule out silently, and never write a test that passes`,
327
+ ` whatever the product does in its place.`,
328
+ ` - Assert the ONE outcome the rule expects. Never a regex or a condition`,
329
+ ` that accepts the good and the bad outcome alike (/confirm|not available/),`,
330
+ ` never the page's whole text (body or main innerText) compared with`,
331
+ ` itself, and never an expect() only inside an if, else or catch.`,
332
+ ` - A rule about a specific thing (the operator, the total, the time, the`,
333
+ ` link's address) reads THAT element and compares its value: "the same`,
334
+ ` operator" means the operator's name read before and after.`,
335
+ ` - "No 5xx", "the page loads", "body is not empty", "a £ sign is shown",`,
336
+ ` "the count of a text that never appears is 0" prove only a rule that says`,
337
+ ` exactly that.`,
338
+ ` - A stub's own words are never the assertion: assert what the app does`,
339
+ ` with the stubbed state.`,
340
+ ``,
297
341
  `SELF-CHECK BEFORE YOU OUTPUT. Go through every test you wrote and name, for`,
298
- `each line, its evidence under E1–E5, and the distinct outcome each test`,
299
- `adds. A line with no evidence is fixed, or its test is removed; a test that`,
300
- `adds no new outcome is removed. When in doubt, report LESS: a missing test`,
301
- `costs one line of coverage; a false failure costs the team a release.`,
342
+ `each line, its evidence under E1–E5, and the outcome that makes it fail. A`,
343
+ `line with no evidence is fixed; a test that cannot fail is rewritten, or`,
344
+ `its rule gets a cannot-test line. Then check every rule id you were given`,
345
+ `has a test or a cannot-test line.`,
302
346
  ``,
303
347
  ...(memory
304
348
  ? [
@@ -307,9 +351,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
307
351
  `once for the whole journey, the TESTKIT list is the harness's real`,
308
352
  `exports, and the JOURNEY block holds exactly the rules this file tests`,
309
353
  `with their context. Those are the MECHANICS: HOW to reach and drive`,
310
- `the behaviour. Never invent a selector, route, field or helper that`,
311
- `appears nowhere below; a case you cannot reach with what is here is`,
312
- `left out. But the EXPECTED OUTCOME you`,
354
+ `the behaviour. Never invent a test id, route, request field or helper`,
355
+ `that appears nowhere below. An element is found by the name the doc`,
356
+ `gives it, with kit.find. But the EXPECTED OUTCOME you`,
313
357
  ]
314
358
  : grounding
315
359
  ? [
@@ -411,7 +455,11 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
411
455
  `FIND EVERY ELEMENT WITH kit.find(page, kind, name): kind is what the doc`,
412
456
  `says it is (button, link, input, dropdown, checkbox, radio, toggle, tab,`,
413
457
  `menuitem, dialog, heading, alert, toast, image, list, table …), name is`,
414
- `the text the doc gives it. It tries the proper role first and falls back`,
458
+ `the text the SCREEN shows for it — read it from the CODE FACTS (the`,
459
+ `component's text, label or aria-label); the doc's word only when the facts`,
460
+ `do not show that element ("help link" in the doc may read "FAQs" on the`,
461
+ `screen). Written out, never a value the test made up (kit.unique, a stub's`,
462
+ `id). It tries the proper role first and falls back`,
415
463
  `to the label, placeholder, aria-label, alt text and visible text, adding`,
416
464
  `a finding when the element is not built as what it is — so a test written`,
417
465
  `before the feature still finds it once it ships. Use page.getBy* only for`,
@@ -430,8 +478,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
430
478
  `ground on role/label/text/name instead of inventing a test-id.`,
431
479
  `NEVER invent a data-testid/data-cy: an unverified test-id is`,
432
480
  `REJECTED by the selector-grounding gate and wastes the whole file.`,
433
- `Only when an element cannot be identified by ANY real handle, leave`,
434
- `that single case out and name the control in a comment — never guess.`,
481
+ `An element the doc names is found with kit.find(page, kind, <that name>)`,
482
+ `even when the source shows no handle for it; never guess a test id.`,
435
483
  `NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
436
484
  `never a description/comment, and never a route-file placeholder`,
437
485
  `like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
@@ -546,12 +594,12 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
546
594
  ` ${focus.title}`,
547
595
  ` ${focus.covers}`,
548
596
  `Do NOT test the journey's other concerns here — separate files`,
549
- `cover those. Go DEEP on this one concern (happy path + its own`,
550
- `edge/negative/security cases) rather than broad-but-shallow.`,
597
+ `cover those. ${memory?.rules.length ? 'Test each rule below; add no case the rules do not state.' : 'Go DEEP on this one concern (happy path + its own edge/negative/security cases) rather than broad-but-shallow.'}`,
551
598
  `Use this exact describe() title: "${focus.title}".`,
552
599
  ...(memory?.rules.length
553
600
  ? [
554
- `Test exactly these rules: ${memory.rules.join(', ')}. Name the rule id AND its`,
601
+ `Test exactly these rules: ${memory.rules.join(', ')}. EVERY one gets a test or a`,
602
+ `"// cannot-test: <id> — <why>" line. Name the rule id AND its`,
555
603
  `tag in each test's oracle comment, e.g. "// oracle: ${memory.rules[0]} [spec] …":`,
556
604
  `the tag is how a failure is later graded as a defect or a finding.`,
557
605
  ]
@@ -575,11 +623,15 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
575
623
  `- calls page.waitForTimeout() or sleeps for a fixed time;`,
576
624
  `- skips a test because an element is missing (test.skip(true, …), or a`,
577
625
  ` condition that counts or looks for an element) — that is a failure to assert;`,
578
- `- has a test with no expect(), or more than ${maxPerRule()} tests for one rule;`,
579
- `- has no test at all: every rule gets one; stub a state you cannot create;`,
626
+ `- has a test with no expect(), an expect() only inside an if/else/catch, or more`,
627
+ ` than ${maxPerRule()} tests for one rule;`,
628
+ `- leaves a rule with neither a test nor a "// cannot-test: <id> — <why>" line;`,
629
+ `- asserts a regex that accepts the good and the bad outcome, compares the page's`,
630
+ ` whole text, or asserts the words its own stub returns;`,
631
+ `- uses kit.find(...) without await, or a kit.openStep step the TESTKIT does not list;`,
580
632
  `- names a button, link, field or label that is in neither the CODE FACTS nor the rules;`,
581
633
  `- defines its own helper functions (use the TESTKIT) or comments anything but`,
582
- ` the one "// oracle: <rule id> [tag] …" line above each test.`,
634
+ ` the one "// oracle: <rule id> [tag] …" line above each test and cannot-test lines.`,
583
635
  `Keep each test short: the steps, then the expect()s. No logging.`,
584
636
  ``,
585
637
  ]
@@ -0,0 +1,77 @@
1
+ /**
2
+ * testforge — finish a written file: every rule tested, every test able to fail.
3
+ *
4
+ * A file can pass every file-level gate and still leave rules out, or hold a
5
+ * test that passes whatever the app does. Those tests are taken out, and the
6
+ * rules left without a test are written again in ONE small call that sees only
7
+ * those rules, the file's top and why the earlier tests were refused. What
8
+ * that call still gets wrong is removed and reported, never sent again: a
9
+ * missing test is honest, a test that cannot fail is not.
10
+ */
11
+ import { factsSlice, journeySlice } from './journey-memory.js';
12
+ import { appendTests, findTestBlocks, spliceBlocks } from './test-blocks.js';
13
+ import { acceptedReason, describeProblems, ruleCoverage, testProblems } from '../gates/test-quality.js';
14
+ const review = (content, rules, steps, knownTexts) => ({
15
+ cov: ruleCoverage(content, rules),
16
+ problems: testProblems(content, { steps, knownTexts }),
17
+ });
18
+ /** The file without the tests that have a problem. */
19
+ function withoutBlocks(content, problems) {
20
+ const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
21
+ return blocks.length ? spliceBlocks(content, blocks.map((block) => ({ block, text: '' }))) : content;
22
+ }
23
+ /** Is the file complete as it is? */
24
+ export function fileIsComplete(content, rules, steps, knownTexts) {
25
+ const { cov, problems } = review(content, rules, steps, knownTexts);
26
+ return !cov.missing.length && !problems.length;
27
+ }
28
+ export async function completeFile(content, ctx) {
29
+ const notes = [];
30
+ const first = review(content, ctx.rules, ctx.steps, ctx.knownTexts);
31
+ let calls = 0;
32
+ let out = content;
33
+ const dropped = new Set();
34
+ if (first.cov.missing.length || first.problems.length) {
35
+ out = withoutBlocks(content, first.problems);
36
+ for (const p of first.problems)
37
+ dropped.add(p.title);
38
+ const left = ruleCoverage(out, ctx.rules).missing;
39
+ if (left.length) {
40
+ const blocks = findTestBlocks(out);
41
+ const head = out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000);
42
+ const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(out)?.[2] ?? ctx.journey.name;
43
+ const slice = journeySlice(ctx.memory, left);
44
+ calls = 1;
45
+ try {
46
+ const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(first.problems, first.cov));
47
+ const candidate = appendTests(out, tests);
48
+ const refused = ctx.check(candidate);
49
+ if (refused)
50
+ notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
51
+ else
52
+ out = candidate;
53
+ }
54
+ catch (e) {
55
+ notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
56
+ }
57
+ // What the second answer still gets wrong is removed, not sent again.
58
+ const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts });
59
+ if (second.length) {
60
+ out = withoutBlocks(out, second);
61
+ for (const p of second)
62
+ dropped.add(p.title);
63
+ }
64
+ }
65
+ }
66
+ // A cannot-test line that is no longer true (its rule has a test now) or never
67
+ // was a reason ("the locator is unknown") does not stay in the file.
68
+ const has = new Set(findTestBlocks(out).flatMap((b) => b.rules));
69
+ out = out.replace(/^[ \t]*\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)\n/gm, (line, ids, why) => ids.split(/\s*,\s*/).every((id) => has.has(id)) || !acceptedReason(why) ? '' : line);
70
+ const cov = ruleCoverage(out, ctx.rules);
71
+ const tested = [...has].filter((id) => ctx.rules.includes(id));
72
+ if (dropped.size)
73
+ notes.push(`took out ${dropped.size} test(s) that could not check their rule`);
74
+ if (cov.missing.length)
75
+ notes.push(`no test for ${cov.missing.join(', ')}`);
76
+ return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped], calls, notes };
77
+ }
@@ -49,7 +49,7 @@ function similarity(a, b) {
49
49
  * strings is a rename; a close match is a change; the rest is added or
50
50
  * removed. Pure: no AI, no I/O.
51
51
  */
52
- export function alignRules(prev, next) {
52
+ export function alignRules(prev, next, retired = []) {
53
53
  const diff = { same: [], changed: [], renamed: [], added: [], removed: [] };
54
54
  const free = new Set(prev.map((r) => r.id));
55
55
  const byText = new Map();
@@ -117,7 +117,8 @@ export function alignRules(prev, next) {
117
117
  diff.changed.push(best.id);
118
118
  }
119
119
  // 4. Everything else is new, numbered after the highest id ever used.
120
- let top = Math.max(0, ...prev.map((r) => Number(r.id.slice(1)) || 0));
120
+ // A removed rule's id is never given to a new one: old results still name it.
121
+ let top = Math.max(0, ...prev.map((r) => Number(r.id.slice(1)) || 0), ...retired.map((id) => Number(id.slice(1)) || 0));
121
122
  for (const r of next.rules) {
122
123
  if (id.has(r))
123
124
  continue;
@@ -43,9 +43,13 @@ export const HARNESS_CONTRACT = [
43
43
  ` - created(kind, id, cleanup?, role?): void record what the run made (H8).`,
44
44
  ` - the "run" role: token('run'), ensureToken('run'), login(page, 'run'),`,
45
45
  ` apiContext(request, 'run') — the account tests that CREATE data use (H8).`,
46
+ ` - distinctRunAccount(): boolean true only when the "run" role is a`,
47
+ ` different account from the fixture role (H8), so a test that needs two`,
48
+ ` different people at once can tell whether it has them.`,
46
49
  `Plus, for Playwright suites:`,
47
50
  ` - openStep(page, step, input?): Promise<true | string> open a later step`,
48
51
  ` of a multi-step flow with what the earlier steps would have left (H9).`,
52
+ ` - STEPS: object the step names openStep accepts (H9).`,
49
53
  ` - settledClick, fillFrameInputs, watchDialogs: re-exported unchanged from`,
50
54
  ` require('@activate-core/test-forge/reporters/ui-helpers.cjs') — never`,
51
55
  ` rewritten here.`,
@@ -170,7 +174,9 @@ export const HARNESS_CONTRACT = [
170
174
  ` signup — create a fresh account through the product's own sign-up, only`,
171
175
  ` when this environment can complete it (no code it cannot read and`,
172
176
  ` cannot bypass under H7); otherwise fall back to fixture and say so once.`,
173
- ` Mint and cache its token like any role.`,
177
+ ` Mint and cache its token like any role. distinctRunAccount() returns true`,
178
+ ` only in pool or signup mode, when the run account really differs from`,
179
+ ` the fixture one.`,
174
180
  `H9. STEPS OF A FLOW. A later step of a multi-step flow (onboarding, a`,
175
181
  ` booking or checkout wizard, a sign-up in stages) depends on what the`,
176
182
  ` earlier steps stored — a choice, an address, a cart, a plan — in URL`,
@@ -185,8 +191,14 @@ export const HARNESS_CONTRACT = [
185
191
  ` - It waits until the step is really shown and resolves true; when the`,
186
192
  ` product still sends the visitor elsewhere it resolves a string naming`,
187
193
  ` the step it landed on, so the test can skip with that reason.`,
188
- ` - Offer only the steps the flow has; when the project has no such flow,`,
189
- ` export openStep as a function resolving 'this product has no multi-step flow'.`,
194
+ ` - Offer EVERY step the flow has, up to and including its last page before`,
195
+ ` the result — the summary, review or checkout page where the visitor`,
196
+ ` confirms and pays — reaching a page that follows a choice (an operator,`,
197
+ ` a plan, a slot) by making that choice through the UI, as a visitor would.`,
198
+ ` Keep the names in one exported object STEPS (name → what identifies the`,
199
+ ` page), and look the step up in it: an unknown name resolves`,
200
+ ` 'unknown step "<name>" (steps: …)'. When the project has no such flow,`,
201
+ ` export STEPS = {} and openStep resolving 'this product has no multi-step flow'.`,
190
202
  ].join('\n');
191
203
  /**
192
204
  * Stamp written into every generated testkit: the contract it was built
@@ -9,6 +9,7 @@
9
9
  * repair was most of 9.6M tokens; the code below needs no AI at all.
10
10
  */
11
11
  import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
12
+ import { harnessSteps } from '../gates/test-quality.js';
12
13
  import { dirname, join } from 'node:path';
13
14
  import { journeyDir } from '../paths.js';
14
15
  /** Sections that are indexes or open questions, never something to test. */
@@ -73,9 +74,10 @@ export function readRules(markdown) {
73
74
  continue;
74
75
  }
75
76
  if (mode === 'controls') {
76
- const row = /^\|\s*(C\d+[a-z]?)\s*\|(.*)\|\s*$/.exec(line);
77
+ // "| C3 |" or a plain "| 3 |": both are control C3.
78
+ const row = /^\|\s*C?(\d+[a-z]?)\s*\|(.*)\|\s*$/.exec(line);
77
79
  if (row)
78
- controls[row[1]] = row[2].split('|').map((c) => c.trim()).filter(Boolean).join(' · ');
80
+ controls[`C${row[1]}`] = row[2].split('|').map((c) => c.trim()).filter(Boolean).join(' · ');
79
81
  continue;
80
82
  }
81
83
  if (mode !== 'rules')
@@ -147,6 +149,13 @@ function sliceText(memory, own, context, controlIds) {
147
149
  * the file, not asked of the AI, so every writer gets the same true list
148
150
  * without opening the testkit itself.
149
151
  */
152
+ /** TestForge's own runtime helpers, as a testkit re-exports them (find: uiHelpers.find). */
153
+ const RUNTIME_HELPERS = {
154
+ find: "async find(page, kind, name, { exact, timeout, within }) → Promise<Locator> — ALWAYS await it before use: `const b = await kit.find(page, 'button', 'Pay')`; never kit.find(...).first() and never expect(kit.find(...)). kind: button, link, input, dropdown, checkbox, radio, toggle, tab, menuitem, dialog, heading, alert, toast, image …; name: the text the doc gives it (or a RegExp).",
155
+ settledClick: 'async settledClick(locator, { settleMs, tries }) — clicks once the element has stopped moving (pop-ups, growing forms).',
156
+ fillFrameInputs: 'async fillFrameInputs(page, framePrefix, { name: value }) — fills inputs inside an embedded iframe (card fields).',
157
+ watchDialogs: 'watchDialogs(page) → string[] — accepts browser alerts and records their messages.',
158
+ };
150
159
  export function helperApi(harnessSource) {
151
160
  const names = new Set();
152
161
  const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(harnessSource);
@@ -166,10 +175,16 @@ export function helperApi(harnessSource) {
166
175
  }
167
176
  const lines = harnessSource.split('\n');
168
177
  const out = [];
178
+ const steps = harnessSteps(harnessSource);
169
179
  for (const name of names) {
170
180
  const at = lines.findIndex((l) => new RegExp(`^\\s*(?:export\\s+)?(?:async\\s+)?(?:function\\*?\\s+${name}\\s*\\(|(?:const|let|var)\\s+${name}\\s*=)`).test(l));
171
181
  if (at === -1) {
172
- out.push(`- ${name}`);
182
+ // Re-exported from TestForge's runtime (find: uiHelpers.find): its contract, not just its name.
183
+ out.push(RUNTIME_HELPERS[name] ? `- ${RUNTIME_HELPERS[name]}` : `- ${name}`);
184
+ continue;
185
+ }
186
+ if (name === 'openStep' && steps.length) {
187
+ out.push(`- ${lines[at].trim().replace(/^export\s+/, '').replace(/\s*\{\s*\}?\s*$/, '')} — step is EXACTLY one of: ${steps.join(', ')}. Any other name does not exist. Resolves true when the step is open, else a string saying why.`);
173
188
  continue;
174
189
  }
175
190
  let sig = lines[at].trim().replace(/^export\s+/, '').replace(/\s*\{\s*\}?\s*$/, '');
@@ -278,6 +293,60 @@ export function saveMemory(journey, docHash, memory, helpers, extra) {
278
293
  }
279
294
  return path;
280
295
  }
296
+ /**
297
+ * Put what the files really test into the journey's memory and its plan
298
+ * check: which file tests each rule, which rules cannot be tested here and
299
+ * why, and which have no test. Returns the one-line summary.
300
+ */
301
+ export function recordWritten(journey, files) {
302
+ const dir = journeyDir(journey);
303
+ const path = join(dir, 'rules.json');
304
+ const testedBy = new Map();
305
+ const why = new Map();
306
+ const missing = new Set();
307
+ for (const f of files) {
308
+ for (const id of f.tested)
309
+ testedBy.set(id, [...(testedBy.get(id) ?? []), f.slug]);
310
+ for (const c of f.cannotTest)
311
+ why.set(c.id, `cannot be tested here: ${c.why}`);
312
+ for (const id of f.missing)
313
+ missing.add(id);
314
+ }
315
+ const planned = new Set(files.flatMap((f) => [...f.tested, ...f.cannotTest.map((c) => c.id), ...f.missing]));
316
+ try {
317
+ const record = JSON.parse(readFileSync(path, 'utf-8'));
318
+ for (const r of record.rules) {
319
+ if (!planned.has(r.id))
320
+ continue;
321
+ r.testedBy = testedBy.get(r.id) ?? [];
322
+ r.skipped = why.get(r.id) ?? (missing.has(r.id) ? 'no test was written for it' : undefined);
323
+ }
324
+ writeFileSync(path, JSON.stringify(record, null, 2), 'utf-8');
325
+ }
326
+ catch {
327
+ // no memory file: the summary below still reports it
328
+ }
329
+ const dropped = files.reduce((n, f) => n + f.dropped.length, 0);
330
+ const lines = [
331
+ `${testedBy.size}/${planned.size} planned rules have a test`,
332
+ why.size ? `${why.size} cannot be tested here (${[...why.keys()].join(', ')})` : '',
333
+ missing.size ? `NO TEST for ${[...missing].join(', ')}` : '',
334
+ dropped ? `${dropped} test(s) taken out because they could not check their rule` : '',
335
+ ].filter(Boolean);
336
+ const check = join(dir, 'plan-check.md');
337
+ try {
338
+ const before = existsSync(check) ? readFileSync(check, 'utf-8').replace(/\n## Written\n[\s\S]*$/, '') : '';
339
+ const detail = [
340
+ ...[...why].map(([id, w]) => `- ${id}: ${w}`),
341
+ ...[...missing].map((id) => `- ${id}: no test was written for it`),
342
+ ];
343
+ writeFileSync(check, `${before.trimEnd()}\n\n## Written\n\n${lines.join('; ')}.\n${detail.length ? `\n${detail.join('\n')}\n` : ''}`, 'utf-8');
344
+ }
345
+ catch {
346
+ // best-effort
347
+ }
348
+ return lines.join('; ');
349
+ }
281
350
  /**
282
351
  * Rules of this journey that another journey already tests, found without the
283
352
  * AI: the same tag and nearly the same words (≥ 85 % of the words shared).
@@ -26,15 +26,18 @@ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, is
26
26
  import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
- import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, } from './ai-generator.js';
29
+ import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
30
+ import { completeFile, fileIsComplete } from './complete-file.js';
31
+ import { harnessSteps } from '../gates/test-quality.js';
30
32
  import { normalisePending } from './pending.js';
31
33
  import { packageRoot } from '../run/run-tests.js';
34
+ import { pendingReason, withPendingStamp } from '../run/readiness.js';
32
35
  import { groundJourney } from '../docs/grounding.js';
33
36
  import { generateSeed } from './seed-generator.js';
34
37
  import { dataAccess, writeRequiredData } from './required-data.js';
35
38
  import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
36
39
  import { testedTags } from './suite-planner.js';
37
- import { factsSlice, helperApi, journeySlice, readRules, saveMemory, testedElsewhere } from './journey-memory.js';
40
+ import { factsSlice, helperApi, journeySlice, readRules, recordWritten, saveMemory, testedElsewhere } from './journey-memory.js';
38
41
  import { loadSourceChoice } from '../docs/source-of-truth.js';
39
42
  import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
40
43
  import { updateJourney } from './update-journey.js';
@@ -424,19 +427,36 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
424
427
  // What the app can show: its source and the doc. A fresh test's names and
425
428
  // labels are checked against it before the test is run.
426
429
  let knownTexts;
430
+ const appTexts = () => framework === 'playwright'
431
+ ? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
432
+ : undefined;
427
433
  const genGate = (content, filePath) => testGate(content) ??
428
434
  checkTestGates(content, ext, {
429
435
  framework, forbidPlaceholders: true, filePath,
430
436
  maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 2,
431
- knownTexts: framework === 'playwright'
432
- ? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
433
- : undefined,
437
+ knownTexts: appTexts(),
434
438
  });
435
439
  // Skip if unchanged — but NOT if the journey still has deferred stubs
436
440
  // to retry (a resume pass fills those in without touching real files).
441
+ // A suite whose tests leave rules out or cannot fail is not done either:
442
+ // it goes on, and only those tests are written again.
443
+ const suiteComplete = () => {
444
+ const plan = readPlan(journey);
445
+ if (!plan?.some((f) => f.rules?.length))
446
+ return true;
447
+ const kit = harnessInfo(areaDir, framework, ext).path;
448
+ const steps = existsSync(kit) ? harnessSteps(readFileSync(kit, 'utf-8')) : [];
449
+ return plan.every((f) => {
450
+ const p = join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`);
451
+ if (!f.rules?.length || !existsSync(p))
452
+ return true;
453
+ return fileIsComplete(readFileSync(p, 'utf-8'), f.rules, steps, appTexts());
454
+ });
455
+ };
437
456
  if (!force &&
438
457
  cache[file] === hash &&
439
- !hasDeferredStubs(testDir, ext)) {
458
+ !hasDeferredStubs(testDir, ext) &&
459
+ suiteComplete()) {
440
460
  results.push({
441
461
  journeyName: journey.name,
442
462
  testDir,
@@ -452,8 +472,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
452
472
  // A feature written up before it is built: its tests come from the doc
453
473
  // alone. There is no code to read for facts and no app to run them on.
454
474
  const notBuilt = /^status:\s*not built/im.test(markdown);
455
- if (notBuilt)
456
- warnings.push('written before the feature exists (Status: not built yet) — run these tests once it ships');
475
+ // Not live yet (not built, or behind a flag): every file is stamped, and
476
+ // `testforge run` holds it back until the doc's Status line goes.
477
+ const pending = pendingReason(markdown);
478
+ if (pending)
479
+ warnings.push(`${pending}: its files are held back by \`testforge run\` until the doc's Status line is removed`);
457
480
  const collision = collisionWarning(areaDir, journey.id, ext);
458
481
  if (collision)
459
482
  warnings.push(collision);
@@ -493,17 +516,42 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
493
516
  const tested = testedTags();
494
517
  let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
495
518
  const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
519
+ const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
520
+ // What each file really tests once written: the memory and the report come from this, not the plan.
521
+ const written = [];
522
+ const finish = async (content, pf, filePath) => {
523
+ if (!memory || !pf.rules?.length)
524
+ return content;
525
+ const { mem, facts } = await fileMemory(pf);
526
+ const done = await completeFile(content, {
527
+ journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
528
+ harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
529
+ check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
530
+ });
531
+ written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
532
+ for (const n of done.notes)
533
+ warnings.push(`${pf.slug}: ${n}`);
534
+ return done.content;
535
+ };
496
536
  // A doc that changed since its suite was written: only what changed is
497
537
  // touched. Renames and removals are edits without AI; reworded and added
498
538
  // rules get one small call each file; new sections become new files below.
499
539
  const prevRecord = memory ? loadMemory(journey) : undefined;
500
- const prevMemory = !force && !opts.planOnly ? prevRecord : undefined;
501
- const prevPlan = prevMemory && prevMemory.hash !== hash ? readPlan(journey) : undefined;
540
+ // Rule ids always follow the last written version, even for --force and
541
+ // --plan-only: an id that moved would point the tests' oracle comments at
542
+ // another rule. Only the in-place update of the tests is skipped then.
502
543
  let changes;
503
- if (memory && prevMemory && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
504
- const { memory: aligned, diff } = alignRules(prevMemory.rules, memory);
544
+ let docDiff;
545
+ if (memory && prevRecord && prevRecord.hash !== hash) {
546
+ const { memory: aligned, diff } = alignRules(prevRecord.rules, memory, (prevRecord.removed ?? []).map((r) => r.id));
505
547
  memory = aligned;
548
+ docDiff = diff;
506
549
  changes = { renamed: diff.renamed.map((r) => r.id), changed: diff.changed, removed: diff.removed };
550
+ }
551
+ const prevPlan = docDiff && !force && !opts.planOnly ? readPlan(journey) : undefined;
552
+ if (memory && docDiff && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
553
+ const aligned = memory;
554
+ const diff = docDiff;
507
555
  console.log(` ${journey.id}: doc changed — ${describeDiff(diff)}`);
508
556
  const facts = notBuilt ? undefined : (await groundJourney(journey, hash, 'src', markdown, true)) ?? undefined;
509
557
  const up = await updateJourney({
@@ -519,7 +567,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
519
567
  const planned = useAI
520
568
  ? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
521
569
  : [{ slug: journey.id, title: journey.name, covers: '' }];
522
- if (memory) {
570
+ // A plan-only run leaves the memory as the tests were last written, so the
571
+ // next generate still sees what changed in the doc and updates those tests.
572
+ if (memory && !opts.planOnly) {
523
573
  const choice = loadSourceChoice();
524
574
  saveMemory(journey, hash, memory, helpers, {
525
575
  doc: file, reference: choice?.journeys?.[file] ?? choice?.reference,
@@ -593,8 +643,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
593
643
  if (useAI && memory && perCall > 1) {
594
644
  const toWrite = planned.filter((pf) => !keeps(pathOf(pf)));
595
645
  const batches = [];
596
- for (let k = 0; k < toWrite.length; k += perCall)
597
- batches.push(toWrite.slice(k, k + perCall));
646
+ // A few files to one call, but never so many rules that the writer drops
647
+ // some: 21 rules in one answer left 14 of them without a test.
648
+ const maxRules = 12;
649
+ for (const pf of toWrite) {
650
+ const last = batches[batches.length - 1];
651
+ const rules = (b) => b.reduce((n, f) => n + (f.rules?.length ?? 0), 0);
652
+ if (last && last.length < perCall && rules(last) + (pf.rules?.length ?? 0) <= maxRules)
653
+ last.push(pf);
654
+ else
655
+ batches.push([pf]);
656
+ }
598
657
  const writeBatch = async (batch) => {
599
658
  const files = await Promise.all(batch.map(async (pf) => {
600
659
  const { mem, facts } = await fileMemory(pf);
@@ -637,8 +696,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
637
696
  // A suite written before pending tests carried their reason gets
638
697
  // it now, deterministically — no regeneration, no AI.
639
698
  const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
640
- if (normalised !== existing)
641
- writeFileSync(filePath, normalised, 'utf-8');
699
+ // A kept file that leaves rules out or holds tests that cannot fail:
700
+ // only those tests are written again.
701
+ const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath) : normalised, pending);
702
+ if (finished !== existing)
703
+ writeFileSync(filePath, finished, 'utf-8');
642
704
  return { slug: pf.slug, path: filePath, status: 'ai' };
643
705
  }
644
706
  }
@@ -654,7 +716,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
654
716
  const { mem, facts } = await fileMemory(pf);
655
717
  const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
656
718
  if (ai !== null) {
657
- content = framework === 'playwright' ? normalisePending(ai) : ai;
719
+ content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
658
720
  status = 'ai';
659
721
  }
660
722
  else {
@@ -673,6 +735,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
673
735
  if (!content.includes(GEN_MARKER)) {
674
736
  content = `// ${GEN_MARKER}\n${content}`;
675
737
  }
738
+ content = withPendingStamp(content, pending);
676
739
  // Write IMMEDIATELY (not batched at the end of the journey), so an
677
740
  // interruption preserves every file generated so far.
678
741
  // keepDeferredStubs:false (config) — do NOT write a DEFERRED stub file;
@@ -685,6 +748,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
685
748
  }
686
749
  return { slug: pf.slug, path: filePath, status };
687
750
  });
751
+ if (memory && written.length) {
752
+ const sum = recordWritten(journey, written);
753
+ console.log(` ${journey.id}: ${sum}`);
754
+ }
688
755
  const testFiles = generated.map((g) => ({
689
756
  slug: g.slug,
690
757
  path: g.path,
@@ -74,6 +74,8 @@ export async function updateJourney(ctx) {
74
74
  const base = old.length ? spliceBlocks(content, old.map((block) => ({ block, text: '' }))) : content;
75
75
  let refusal;
76
76
  let next = null;
77
+ // Only the new tests are judged: a problem the file already had is not theirs.
78
+ const already = ctx.check(base, path);
77
79
  for (let attempt = 0; attempt < 2 && next === null; attempt++) {
78
80
  let tests = '';
79
81
  try {
@@ -85,7 +87,8 @@ export async function updateJourney(ctx) {
85
87
  break;
86
88
  }
87
89
  const candidate = appendTests(base, tests);
88
- refusal = findTestBlocks(tests).length ? ctx.check(candidate, path) ?? undefined : 'it contained no test(...) block';
90
+ const why = findTestBlocks(tests).length ? ctx.check(candidate, path) : 'it contained no test(...) block';
91
+ refusal = why && why !== already ? why : undefined;
89
92
  if (!refusal)
90
93
  next = candidate;
91
94
  }
@@ -77,7 +77,12 @@ function hasPage(path, haystack, appRoots) {
77
77
  export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
78
78
  let texts;
79
79
  return files.map((file) => {
80
- const { names, pages } = namesInTest(readFileSync(file, 'utf-8'));
80
+ const content = readFileSync(file, 'utf-8');
81
+ // The doc says the feature is not live: held back whatever the code has.
82
+ const pending = PENDING_STAMP.exec(content)?.[1];
83
+ if (pending)
84
+ return { file, state: 'not-built', missing: [pending.trim()], renamed: [] };
85
+ const { names, pages } = namesInTest(content);
81
86
  const missing = [];
82
87
  const renamed = [];
83
88
  for (const n of names) {
@@ -100,6 +105,28 @@ export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
100
105
  return { file, state, missing, renamed };
101
106
  });
102
107
  }
108
+ /** The line that holds back a file whose feature is not live yet. */
109
+ export const PENDING_STAMP = /^\/\/\s*testforge-pending:\s*(.+)$/m;
110
+ /**
111
+ * The status a journey doc gives its feature: "Status: not built", or
112
+ * "Status: behind flag <name>" for code that ships switched off. Undefined
113
+ * when the feature is live.
114
+ */
115
+ export function pendingReason(markdown) {
116
+ const m = /^status:\s*(not built(?: yet)?|behind (?:a )?flag\b[^\n]*)$/im.exec(markdown);
117
+ if (!m)
118
+ return undefined;
119
+ return /^not built/i.test(m[1]) ? 'the feature is not built yet' : `the feature is ${m[1].trim().replace(/^behind (?:a )?flag\s*/i, 'behind the flag ').trim()} — run it once that is switched on`;
120
+ }
121
+ /** The file with the pending stamp set to `reason`, or removed when there is none. */
122
+ export function withPendingStamp(content, reason) {
123
+ const without = content.replace(/^\/\/\s*testforge-pending:.*\n/m, '');
124
+ if (!reason)
125
+ return without;
126
+ const marker = /^\/\/ @generated by testforge.*\n/m.exec(without);
127
+ const at = marker ? marker.index + marker[0].length : 0;
128
+ return `${without.slice(0, at)}// testforge-pending: ${reason}\n${without.slice(at)}`;
129
+ }
103
130
  /** The test files under a directory. */
104
131
  export function testFilesIn(dir) {
105
132
  const out = [];
@@ -140,7 +140,7 @@ export function runTests(opts = {}) {
140
140
  const states = checkReadiness(files, sourceText(roots), () => [], [root]);
141
141
  const held = states.filter((s) => s.state === 'not-built');
142
142
  if (held.length) {
143
- console.log(`\n${held.length} file(s) pending — the code has none of the elements or pages they use yet (\`testforge bind\` shows which; --all runs them anyway):`);
143
+ console.log(`\n${held.length} file(s) pending — their feature is not live yet (\`testforge bind\` shows why; --all runs them anyway):`);
144
144
  for (const h of held.slice(0, 15))
145
145
  console.log(` not built ${relative(root, h.file)}: ${h.missing.slice(0, 3).map((m) => `"${m}"`).join(', ')}`);
146
146
  targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
@@ -164,6 +164,8 @@ export function runTests(opts = {}) {
164
164
  console.log('\nNothing to run — generate tests first (`testforge generate`).');
165
165
  return 1;
166
166
  }
167
- console.log(anyFail ? '\nSome tests failed.' : '\nAll test runs passed.');
167
+ // The runner's own summary above says how many ran and how many skipped; a run where
168
+ // everything skipped has no failure, but it is not a pass.
169
+ console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
168
170
  return anyFail ? 1 : 0;
169
171
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.0.0",
3
+ "version": "2.1.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {