@activate-core/test-forge 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md ADDED
@@ -0,0 +1,31 @@
1
+ # Changelog
2
+
3
+ ## 2.7.0
4
+
5
+ - The names new tests still wait on are looked up once, at generate time, against the journey's code facts; a value is kept only if the code has it.
6
+ - The writer never sees a constant or a "names already asked" entry that is still blank, so rewritten tests do not ask for the same blank names again.
7
+ - A kept file whose tests were taken out to be written again stays as it was when the new tests do not come back (a call that fails or is refused).
8
+ - The facts follow the methods a route handler calls: the class it is given (what `create()` returns) and its response helpers, through the helpers they call (the `{ error: { message } }` shape).
9
+ - `fillPath(URL, …values)` from `reporters/interface.cjs` fills a path's parameters whatever the product calls them.
10
+ - A waiting test says why its name is blank: not a kind TestForge looks up, nothing like it in the code yet, looked for and not found, or the lookup could not run.
11
+ - A rewritten test keeps its title (dashboard history continues).
12
+ - A testkit rebuild that fails, or drops an export the tests use, is not taken: the existing testkit stays.
13
+ - Rules asked for and refused are not asked again until the doc or the journey's code changes.
14
+ - "route", "controller" and "endpoint" no longer match every route line.
15
+ - `env --diff` and `report` read an unfilled `KEY= # hint` as unset; `run --journey` accepts only a journey id.
16
+ - `setup` stops when the config it wrote has problems, before any doc is drafted.
17
+
18
+ ## 2.6.0
19
+
20
+ - The reporters record passed tests in `testforge/passed.json`; generate never removes or rewrites them (only `--force` does).
21
+ - A kept file's tests are reported by newer checks, never removed; when code arrives only the tests using its blank names are written again; a failed rewrite keeps the earlier file.
22
+ - A journey's facts move only when its own code changes; tests written with no code are written again once it is there, with the seed.
23
+ - Route handlers and their rule classes come first in the facts; the doc's Test setup reaches the facts; the testkit's `insert` contract always reaches the writer.
24
+ - Name scan: rule, logic, request and enum files; whole-word matching; the enclosing route prefix; methods, roles, nested fields; values the code no longer has are blanked; blanked names are asked again.
25
+ - `run`: files of a feature that is not live are held back; prints how many tests ran of how many; exits 0 when nothing failed for a product reason.
26
+ - A saved review verdict applies to the same failure only; `report --results` files waiting tests as waiting.
27
+ - Globals-style Jest files parse; an unfilled env line is empty; `coverage.json` keeps other journeys; other AI CLIs get the writer rules; a ticket key counts as a source; a missing `--env-file` stops the run.
28
+
29
+ ### Upgrading
30
+ - New state files under `testforge/`: `passed.json` (commit it with the tests), `last-run.json`, `interface-filled.json`, `interface-status.json`, and `journeys/<area>/<id>/refused.json`.
31
+ - Facts are read once again after upgrading (one grounding call per journey).
@@ -17,7 +17,7 @@ export function sourceRoot(base = '.') {
17
17
  const FILE_REF = /((?:[\w.-]+\/)+[\w.-]+\.(?:tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go))(?::(\d+)(?:[-–](\d+))?)?/g;
18
18
  const QUOTED = /"([^"\n]{4,60})"/g;
19
19
  /** Words in a control's name that name no component in particular. */
20
- const GENERIC = new Set(['button', 'buttons', 'field', 'input', 'inputs', 'header', 'footer', 'items', 'entry', 'entries', 'shown', 'guest', 'logged', 'customer', 'computer', 'phone', 'sized', 'screen', 'page', 'pages', 'where', 'opens', 'close', 'link', 'links', 'text', 'label', 'click', 'state', 'result', 'neighbour', 'neighbouring', 'other', 'while', 'their', 'there', 'which', 'every', 'itself', 'changes', 'layout']);
20
+ const GENERIC = new Set(['route', 'routes', 'controller', 'endpoint', 'button', 'buttons', 'field', 'input', 'inputs', 'header', 'footer', 'items', 'entry', 'entries', 'shown', 'guest', 'logged', 'customer', 'computer', 'phone', 'sized', 'screen', 'page', 'pages', 'where', 'opens', 'close', 'link', 'links', 'text', 'label', 'click', 'state', 'result', 'neighbour', 'neighbouring', 'other', 'while', 'their', 'there', 'which', 'every', 'itself', 'changes', 'layout']);
21
21
  function walk(dir, out) {
22
22
  let entries;
23
23
  try {
@@ -302,6 +302,7 @@ export function sourceExcerpts(markdown, roots, budget = 70_000) {
302
302
  const handlers = new Set();
303
303
  const handlerFiles = [];
304
304
  const seenRules = new Set();
305
+ const seenMethods = new Set();
305
306
  // Only the routes the doc names by their path: theirs are the requests its tests send.
306
307
  for (const c of candidates.filter((x) => x.score >= 10).slice(0, 12)) {
307
308
  let line = '';
@@ -335,8 +336,53 @@ export function sourceExcerpts(markdown, roots, budget = 70_000) {
335
336
  // A handler that hands the request to a class it is given
336
337
  // (bookingOrderV2(Request $r, OrderLogicV2 $logic)) keeps its rules there.
337
338
  const sig = text[at].match(/\(([^)]*)\)/)?.[1] ?? '';
338
- for (const dep of [...sig.matchAll(/\b([A-Z]\w+)\s+\$\w+/g)].map((d) => d[1]).filter((d) => d !== 'Request'))
339
- rulesOf(dep, 2);
339
+ const deps = [...sig.matchAll(/\b([A-Z]\w+)\s+\$(\w+)/g)].filter((d) => d[1] !== 'Request');
340
+ for (const d of deps)
341
+ rulesOf(d[1], 2);
342
+ // What the response looks like is in the methods the handler calls: the
343
+ // class it is given ($order_logic->create() → what create() returns) and
344
+ // its own response helpers ($this->respondWithSuccess, often in a trait).
345
+ const body = text.slice(at, at + 45).join('\n');
346
+ for (const d of deps) {
347
+ for (const call of new Set([...body.matchAll(new RegExp(`\\$${d[2]}->(\\w+)\\(`, 'g'))].map((x) => x[1]))) {
348
+ if (/^(?:setInput|getErrors|fails|validate)$/.test(call))
349
+ continue;
350
+ methodWindow(d[1], call, 60);
351
+ }
352
+ }
353
+ for (const call of new Set([...body.matchAll(/\$this->(respond\w*|\w*[Rr]esponse\w*|json\w*)\(/g)].map((x) => x[1])))
354
+ methodWindow('', call, 15, 3);
355
+ }
356
+ }
357
+ // The body of `method` in class `cls` (any class when cls is ''), where it is defined.
358
+ function methodWindow(cls, method, lines, helperDepth = 0) {
359
+ if (seenMethods.has(`${cls}::${method}`))
360
+ return;
361
+ seenMethods.add(`${cls}::${method}`);
362
+ const re = new RegExp(`function\\s+${method}\\s*\\(`);
363
+ for (const f of files.filter((x) => x.endsWith('.php') && (!cls || x.endsWith(`/${cls}.php`))).slice(0, 4000)) {
364
+ let text;
365
+ try {
366
+ text = readFileSync(f, 'utf-8').split('\n');
367
+ }
368
+ catch {
369
+ continue;
370
+ }
371
+ const i = text.findIndex((l) => re.test(l));
372
+ if (i === -1)
373
+ continue;
374
+ add(f, i + 1, i + lines);
375
+ if (!handlerFiles.includes(f))
376
+ handlerFiles.push(f);
377
+ // A helper that builds its answer through another (respondInvalidParameters → respondWithError).
378
+ if (helperDepth > 1) {
379
+ for (const next of new Set([...text.slice(i, i + lines).join('\n').matchAll(/(?:->|::)(respond\w*|\w*[Rr]esponse\w*|json\w*)\(/g)].map((x) => x[1]))) {
380
+ if (next !== method)
381
+ methodWindow('', next, lines, helperDepth - 1);
382
+ }
383
+ }
384
+ if (!cls)
385
+ return; // a helper is defined once (a trait or base controller)
340
386
  }
341
387
  }
342
388
  // The rule methods of a class (validate, rules, getXRules), and of the
@@ -124,6 +124,10 @@ function journeyCode(journey, srcDir) {
124
124
  catch { /* unreadable tree: no code */ }
125
125
  return { excerpts, hash: createHash('sha1').update(excerpts).digest('hex').slice(0, 16) };
126
126
  }
127
+ /** A short fingerprint of the code this journey reads (no AI). */
128
+ export function journeyCodeHash(journey, srcDir = 'src') {
129
+ return journeyCode(journey, srcDir).hash;
130
+ }
127
131
  /** True when the journey's facts were read from code that has since changed. */
128
132
  export function factsCodeMoved(journey, srcDir = 'src') {
129
133
  const path = groundingPath(journey);
@@ -51,10 +51,11 @@ export const DEFAULT_EXTERNAL_MODULES = [
51
51
  * clean. Runs AFTER syntax validation.
52
52
  */
53
53
  export function checkTestGates(content, _ext, ctx) {
54
+ const reasons = [];
54
55
  // F3 — runner correctness: a Jest/backend area must never pull in
55
56
  // Playwright (not installed there; a top-level import breaks the suite).
56
57
  if (ctx.framework === 'jest' && /@playwright\/test/.test(content)) {
57
- return "jest area must not import '@playwright/test'";
58
+ reasons.push("jest area must not import '@playwright/test'");
58
59
  }
59
60
  // F4 — goto guard: `page.goto(...)` must take a real route/URL, not a
60
61
  // pasted description and not an unfilled route-file placeholder. Literal
@@ -67,10 +68,10 @@ export function checkTestGates(content, _ext, ctx) {
67
68
  while ((m = gotoRe.exec(content)) !== null) {
68
69
  const arg = m[2];
69
70
  if (/\s/.test(arg)) {
70
- return `goto() argument looks like prose, not a URL/path: "${arg.slice(0, 40)}…"`;
71
+ reasons.push(`goto() argument looks like prose, not a URL/path: "${arg.slice(0, 40)}…"`);
71
72
  }
72
73
  if (/\[[A-Za-z_]|\/:[A-Za-z_]|\{[A-Za-z_]|<[A-Za-z_]/.test(arg)) {
73
- return `goto() has an unresolved route placeholder — fill it with a real value: "${arg.slice(0, 40)}"`;
74
+ reasons.push(`goto() has an unresolved route placeholder — fill it with a real value: "${arg.slice(0, 40)}"`);
74
75
  }
75
76
  }
76
77
  // F13 — a swallowed action must be bounded. `.click({ force: true })`
@@ -85,7 +86,7 @@ export function checkTestGates(content, _ext, ctx) {
85
86
  let sw;
86
87
  while ((sw = swallowed.exec(content)) !== null) {
87
88
  if (!/timeout\s*:/.test(sw[0])) {
88
- return ('an action whose failure is swallowed with .catch() has no timeout — ' +
89
+ reasons.push('an action whose failure is swallowed with .catch() has no timeout — ' +
89
90
  'it will wait until the whole test times out and blame a later line');
90
91
  }
91
92
  }
@@ -96,7 +97,7 @@ export function checkTestGates(content, _ext, ctx) {
96
97
  // that never comes. Hold the promise first, then act, then await it.
97
98
  const lateWait = /\b(?:await\s+)?page\.goto\([^\n]*\);?\s*\n(?:\s*\/\/[^\n]*\n)*\s*await\s+page\.waitFor(?:Response|Request|Event)\s*\(/;
98
99
  if (lateWait.test(content)) {
99
- return ('page.waitForResponse/Request is armed AFTER the navigation that triggers ' +
100
+ reasons.push('page.waitForResponse/Request is armed AFTER the navigation that triggers ' +
100
101
  'it — hold the promise first, then navigate, then await it');
101
102
  }
102
103
  // F2 — no silent false-green: a file with RUNNABLE tests must assert
@@ -108,7 +109,7 @@ export function checkTestGates(content, _ext, ctx) {
108
109
  const assertions = (content.match(/\bexpect\s*\(/g) || []).length +
109
110
  (content.match(/\bassert\b/g) || []).length;
110
111
  if (activeTests > 0 && assertions === 0) {
111
- return 'has runnable test(s) but no assertions — would pass without testing anything';
112
+ reasons.push('has runnable test(s) but no assertions — would pass without testing anything');
112
113
  }
113
114
  // F1 — dependency awareness: reject imports/requires of a module the
114
115
  // project can't resolve (bare specifier not in allowedModules).
@@ -116,7 +117,7 @@ export function checkTestGates(content, _ext, ctx) {
116
117
  for (const mod of importedModules(content)) {
117
118
  const top = topLevelModule(mod);
118
119
  if (!ctx.allowedModules.has(top)) {
119
- return `imports '${mod}' which the project does not have installed`;
120
+ reasons.push(`imports '${mod}' which the project does not have installed`);
120
121
  }
121
122
  }
122
123
  }
@@ -139,7 +140,7 @@ export function checkTestGates(content, _ext, ctx) {
139
140
  content.includes(`jest.mock('${top}'`) ||
140
141
  content.includes(`jest.mock("${top}"`);
141
142
  if (!mocked) {
142
- return `imports external service '${top}' without a jest.mock — the test would hit the live service; mock it at the seam`;
143
+ reasons.push(`imports external service '${top}' without a jest.mock — the test would hit the live service; mock it at the seam`);
143
144
  }
144
145
  }
145
146
  }
@@ -160,7 +161,7 @@ export function checkTestGates(content, _ext, ctx) {
160
161
  const showedWork = /\b(?:port|origin|path\s*prefix|prefix|resource\s*type|header|predicate)\b/i;
161
162
  const m = content.match(sameHost);
162
163
  if (m && !showedWork.test(m[0])) {
163
- return ('a test is skipped because the dependency "shares a host" with the app, ' +
164
+ reasons.push('a test is skipped because the dependency "shares a host" with the app, ' +
164
165
  'without saying what else was checked — a different port is a different ' +
165
166
  'origin, and a path prefix or a request predicate usually separates them ' +
166
167
  'too; scope the stub to one of those, or name the ones you ruled out');
@@ -169,20 +170,20 @@ export function checkTestGates(content, _ext, ctx) {
169
170
  // F16 — no placeholders in a freshly written file: a placeholder never runs,
170
171
  // and a suite full of them reads as coverage it does not have.
171
172
  if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
172
- return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
173
+ reasons.push('placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file');
173
174
  }
174
175
  // F23 — need() asks for the name of a thing, anywhere in the file (a constant at
175
176
  // the top too), never for data the test should make itself.
176
177
  if (ctx.forbidPlaceholders) {
177
178
  const data = [...content.matchAll(/\bneed\(\s*(['"`])([^'"`\n]+)\1/g)].map((m) => m[2]).filter((n) => !isThingName(n));
178
179
  if (data.length) {
179
- return `asks need() for ${[...new Set(data)].slice(0, 3).map((n) => `"${n}"`).join(', ')}, which is data, not the name of a thing: the test makes its own ids, payloads and records; need() is only for "… URL", "… command", "… table", "… field", "… label"`;
180
+ reasons.push(`asks need() for ${[...new Set(data)].slice(0, 3).map((n) => `"${n}"`).join(', ')}, which is data, not the name of a thing: the test makes its own ids, payloads and records; need() is only for "… URL", "… command", "… table", "… field", "… label"`);
180
181
  }
181
182
  }
182
183
  // F22 — a fresh file exists to test its rules: one with no test at all says
183
184
  // nothing, and passes. It goes back once, told to stub what it cannot create.
184
185
  if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
185
- return 'the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out';
186
+ reasons.push('the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out');
186
187
  }
187
188
  // F24 — one comment policy: the header, an oracle line above each test,
188
189
  // cannot-test lines, a timeout reason, and a short note on a parametrised
@@ -190,7 +191,7 @@ export function checkTestGates(content, _ext, ctx) {
190
191
  if (ctx.forbidPlaceholders) {
191
192
  const stray = strayComment(content);
192
193
  if (stray) {
193
- return `comment not allowed (line ${stray.line}): "${stray.text.slice(0, 60)}" — the only comments are the // @generated header, one // oracle: line above each test, // cannot-test: lines, a // timeout: reason, and a one-line note on a parametrised row`;
194
+ reasons.push(`comment not allowed (line ${stray.line}): "${stray.text.slice(0, 60)}" — the only comments are the // @generated header, one // oracle: line above each test, // cannot-test: lines, a // timeout: reason, and a one-line note on a parametrised row`);
194
195
  }
195
196
  }
196
197
  // F21 — a run-time skip is for what the ENVIRONMENT lacks (an account, a
@@ -200,27 +201,27 @@ export function checkTestGates(content, _ext, ctx) {
200
201
  if (ctx.forbidPlaceholders) {
201
202
  const why = elementSkips(content);
202
203
  if (why.length) {
203
- return `a test skips itself because an element is not on the page (${why[0]}): a missing element the doc names is a failure — assert it instead. Skip only for an account, a variable or data the environment lacks`;
204
+ reasons.push(`a test skips itself because an element is not on the page (${why[0]}): a missing element the doc names is a failure — assert it instead. Skip only for an account, a variable or data the environment lacks`);
204
205
  }
205
206
  }
206
207
  // F17 — relative imports resolve from where the file is written.
207
208
  if (ctx.filePath) {
208
209
  const missing = unresolvedImports(content, ctx.filePath);
209
210
  if (missing.length) {
210
- return `relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`;
211
+ reasons.push(`relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`);
211
212
  }
212
213
  }
213
214
  // F18 — no fixed sleeps. A wait for a set time either wastes it or is too
214
215
  // short on a slower machine; wait for the event or the state instead.
215
216
  if (ctx.framework === 'playwright' && ctx.forbidPlaceholders && /\bwaitForTimeout\s*\(/.test(content)) {
216
- return 'page.waitForTimeout() is not used: wait for the element, response or URL the test needs (expect(...).toBeVisible(), page.waitForResponse(), expect(page).toHaveURL())';
217
+ reasons.push('page.waitForTimeout() is not used: wait for the element, response or URL the test needs (expect(...).toBeVisible(), page.waitForResponse(), expect(page).toHaveURL())');
217
218
  }
218
219
  // F19 — a long test timeout needs a reason. A 240 s budget hides a test that
219
220
  // is stuck; the cap makes it fail early with its real error.
220
221
  if (ctx.maxTestTimeoutMs) {
221
222
  const long = longTimeoutsWithoutReason(content, ctx.maxTestTimeoutMs);
222
223
  if (long.length) {
223
- return `test.setTimeout(${long[0]}) is above ${ctx.maxTestTimeoutMs} ms with no "// timeout: <why>" comment: wait for what the test needs instead, or state why it takes that long`;
224
+ reasons.push(`test.setTimeout(${long[0]}) is above ${ctx.maxTestTimeoutMs} ms with no "// timeout: <why>" comment: wait for what the test needs instead, or state why it takes that long`);
224
225
  }
225
226
  }
226
227
  // F20 — one criterion, a few tests. Thirty inputs against a rule that only
@@ -229,7 +230,7 @@ export function checkTestGates(content, _ext, ctx) {
229
230
  if (ctx.maxTestsPerRule && ctx.forbidPlaceholders) {
230
231
  const over = testsPerCriterion(content).find((c) => c.tests > ctx.maxTestsPerRule);
231
232
  if (over) {
232
- return `${over.tests} tests check the same criterion (${over.criterion.slice(0, 80)}…): keep at most ${ctx.maxTestsPerRule}, one per branch, and put further inputs in one parametrised test`;
233
+ reasons.push(`${over.tests} tests check the same criterion (${over.criterion.slice(0, 80)}…): keep at most ${ctx.maxTestsPerRule}, one per branch, and put further inputs in one parametrised test`);
233
234
  }
234
235
  }
235
236
  // F15 — a pending test says why. A reasonless fixme/skip reaches the report
@@ -238,7 +239,7 @@ export function checkTestGates(content, _ext, ctx) {
238
239
  if (ctx.framework === 'playwright') {
239
240
  const bare = reasonlessPending(content);
240
241
  if (bare.length) {
241
- return (`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
242
+ reasons.push(`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
242
243
  `test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
243
244
  }
244
245
  }
@@ -257,14 +258,14 @@ export function checkTestGates(content, _ext, ctx) {
257
258
  const sel = String.raw `(?:getByTestId\s*\(|\[\s*data-(?:testid|cy|test)\s*=)`;
258
259
  const selNearMarker = new RegExp(`(?:${sel}[^\\n]*?\\b${marker}\\b)|(?:\\b${marker}\\b[^\\n]*?${sel})`, 'i');
259
260
  if (selNearMarker.test(active)) {
260
- return 'a runnable test uses a guessed/placeholder selector — ground it in a real component attribute, or defer the test (test.fixme/skip)';
261
+ reasons.push('a runnable test uses a guessed/placeholder selector — ground it in a real component attribute, or defer the test (test.fixme/skip)');
261
262
  }
262
263
  // (b) When the project's real test-hooks are known, any test-id a
263
264
  // runnable test uses that isn't one of them is an invented selector.
264
265
  if (ctx.knownSelectors && ctx.knownSelectors.size > 0) {
265
266
  for (const id of usedTestIds(active)) {
266
267
  if (!ctx.knownSelectors.has(id)) {
267
- return `references selector '${id}' not found in the project's components — ground it or defer the test`;
268
+ reasons.push(`references selector '${id}' not found in the project's components — ground it or defer the test`);
268
269
  }
269
270
  }
270
271
  }
@@ -272,11 +273,13 @@ export function checkTestGates(content, _ext, ctx) {
272
273
  if (ctx.knownTexts) {
273
274
  const missing = usedLabels(active).filter((l) => !ctx.knownTexts.includes(l.toLowerCase()));
274
275
  if (missing.length) {
275
- return `uses ${missing.map((m) => `"${m}"`).slice(0, 3).join(', ')} as an accessible name, label or placeholder, but that text appears nowhere in the app's source or the journey doc — take every name from the code facts or the rules`;
276
+ reasons.push(`uses ${missing.map((m) => `"${m}"`).slice(0, 3).join(', ')} as an accessible name, label or placeholder, but that text appears nowhere in the app's source or the journey doc — take every name from the code facts or the rules`);
276
277
  }
277
278
  }
278
279
  }
279
- return null;
280
+ // Every reason at once: a retry told one problem fixes it and trips on the next.
281
+ const unique = [...new Set(reasons)];
282
+ return unique.length ? (unique.length === 1 ? unique[0] : `${unique[0]}\nAlso:\n${unique.slice(1, 5).map((r) => `- ${r}`).join('\n')}`) : null;
280
283
  }
281
284
  /**
282
285
  * The run-time skips whose condition looks at the page: test.skip(true, …),
@@ -676,6 +679,29 @@ function testCriteria(content) {
676
679
  return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
677
680
  });
678
681
  }
682
+ /**
683
+ * The file with at most `max` tests per criterion: the first ones are kept,
684
+ * the extras removed. A mechanical fix — no AI retry for "too many tests".
685
+ */
686
+ export function trimExtraTests(content, max) {
687
+ const blocks = findTestBlocks(content);
688
+ const crit = testCriteria(content);
689
+ const seen = new Map();
690
+ const extra = blocks.filter((_, i) => {
691
+ const c = crit[i]?.criterion;
692
+ if (!c)
693
+ return false;
694
+ const n = (seen.get(c) ?? 0) + 1;
695
+ seen.set(c, n);
696
+ return n > max;
697
+ });
698
+ if (!extra.length)
699
+ return content;
700
+ let out = content;
701
+ for (const b of [...extra].sort((x, y) => y.start - x.start))
702
+ out = out.slice(0, b.start) + out.slice(b.end).replace(/^;?[ \t]*\n?/, '');
703
+ return out;
704
+ }
679
705
  /** How many tests of a file check each criterion (tests with no oracle are not counted). */
680
706
  export function testsPerCriterion(content) {
681
707
  const counts = new Map();
@@ -27,6 +27,12 @@ export function shortReason(why) {
27
27
  export function withoutNames(content) {
28
28
  return content.replace(/^[ \t]*(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])[^'"`\n]*\2\s*\);?[ \t]*\n/gm, (line, name) => new RegExp(`\\b${name}\\b`).test(content.replace(line, '')) ? line : '');
29
29
  }
30
+ /** A file top without the `const X = need('…')` lines whose name is still blank. */
31
+ export function withoutBlankNames(head, isBlank) {
32
+ if (!isBlank)
33
+ return head;
34
+ return head.replace(/^[ \t]*(?:const|let)\s+[A-Za-z_$][\w$]*\s*=\s*need\(\s*(['"`])([^'"`\n]*)\1\s*\);?[ \t]*\n/gm, (line, _q, name) => (isBlank(name) ? '' : line));
35
+ }
30
36
  /** The file without the tests that have a problem. */
31
37
  function withoutBlocks(content, problems) {
32
38
  const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
@@ -71,8 +77,10 @@ export async function completeFile(content, ctx) {
71
77
  : [];
72
78
  const redo = redoBlocks
73
79
  .filter((b) => !first.problems.some((p) => p.block.start === b.start))
74
- .map((b) => ({ title: b.title, rules: b.rules, reason: 'it waits on a name the code now has: written again from the code', block: b }));
80
+ .map((b) => ({ title: b.title, rules: b.rules, reason: `it waits on a name the code now has: write it again from the code with the same title, "${b.title}"`, block: b }));
75
81
  const firstDrop = [...first.problems.filter(removable), ...redo];
82
+ let failedRedo = false;
83
+ const refusedRules = [];
76
84
  const reported = first.problems.filter((p) => !removable(p));
77
85
  if (reported.length) {
78
86
  notes.push(`left ${reported.length} earlier test(s) in place that a check flags (${[...new Set(reported.map((p) => shortReason(p.reason)))].slice(0, 3).join('; ')}) — not removed, they were already in the file`);
@@ -82,24 +90,31 @@ export async function completeFile(content, ctx) {
82
90
  drop(firstDrop);
83
91
  // Only rules the doc has are written again: an id with no text makes the writer invent one.
84
92
  const known = new Set(ctx.memory.rules.filter((r) => r.tag).map((r) => r.id));
85
- const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])].filter((id) => known.has(id) && ctx.rules.includes(id));
93
+ const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])]
94
+ .filter((id) => known.has(id) && ctx.rules.includes(id) && !(ctx.skip?.has(id) && !redo.some((p) => p.rules.includes(id))));
86
95
  if (left.length) {
87
96
  const blocks = findTestBlocks(out);
88
- const head = out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000);
97
+ // The file's top as the writer sees it: a constant that still waits on a
98
+ // blank name is left out, or the new tests reuse it and wait again.
99
+ const head = withoutBlankNames(out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000), ctx.isBlank);
89
100
  const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(out)?.[2] ?? ctx.journey.name;
90
101
  const slice = journeySlice(ctx.memory, left);
91
102
  calls = 1;
92
103
  try {
93
104
  const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(firstDrop, first.cov));
94
- const candidate = appendTests(out, onlyNewTests(tests, out, left));
105
+ const candidate = (ctx.fix ?? ((c) => c))(appendTests(out, onlyNewTests(tests, out, left)));
95
106
  const refused = ctx.check(candidate);
96
- if (refused)
107
+ if (refused) {
97
108
  notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
109
+ failedRedo = true;
110
+ refusedRules.push(...left);
111
+ }
98
112
  else
99
113
  out = candidate;
100
114
  }
101
115
  catch (e) {
102
116
  notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
117
+ failedRedo = true;
103
118
  }
104
119
  // What the second answer still gets wrong is removed, not sent again.
105
120
  const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts }).filter(removable);
@@ -109,6 +124,13 @@ export async function completeFile(content, ctx) {
109
124
  }
110
125
  }
111
126
  }
127
+ // A kept file whose tests were taken out to be written again, and the new
128
+ // ones did not come: it stays as it was. A waiting test beats no test.
129
+ if (ctx.kept && failedRedo && redo.length + firstDrop.length > 0) {
130
+ notes.push('kept the file as it was: the tests to write again did not come back');
131
+ out = content;
132
+ dropped.clear();
133
+ }
112
134
  // The last word, checked mechanically: every test that passed is still here.
113
135
  const lost = [...locked].filter((t) => before.has(t) && !findTestBlocks(out).some((b) => b.title === t));
114
136
  if (lost.length) {
@@ -133,5 +155,5 @@ export async function completeFile(content, ctx) {
133
155
  }
134
156
  if (cov.missing.length)
135
157
  notes.push(`no test for ${cov.missing.join(', ')}`);
136
- return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes };
158
+ return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes, refusedRules: [...new Set([...refusedRules, ...(calls ? ruleCoverage(out, ctx.rules).missing.filter((id) => !refusedRules.includes(id) && !failedRedo) : [])])] };
137
159
  }
@@ -30,6 +30,7 @@ import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, wri
30
30
  import { completeFile } from './complete-file.js';
31
31
  import { tuckStrayTests } from './test-blocks.js';
32
32
  import { passedTitles } from './passed.js';
33
+ import { scanNames } from '../run/name-scan.js';
33
34
  import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
34
35
  /** Names the file asks for that testforge/interface.json still has blank. */
35
36
  /** Does this test use a name the product has not given yet (directly, or through the constant it was put in)? */
@@ -43,6 +44,20 @@ function usesBlankName(content, block) {
43
44
  return !!v && new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(text);
44
45
  });
45
46
  }
47
+ /** The names a testkit exports (module.exports = { … } or export …). */
48
+ export function exportNames(src) {
49
+ const names = new Set();
50
+ const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(src);
51
+ if (cjs)
52
+ for (const n of cjs[1].split(',')) {
53
+ const m = /^\s*(\w+)/.exec(n);
54
+ if (m)
55
+ names.add(m[1]);
56
+ }
57
+ for (const m of src.matchAll(/^export\s+(?:async\s+)?(?:function\*?|const|let|class)\s+(\w+)/gm))
58
+ names.add(m[1]);
59
+ return [...names];
60
+ }
46
61
  function blankNames(content) {
47
62
  const iface = readInterface();
48
63
  return [...content.matchAll(/\bneed\(\s*(['"`])([^'"`]+)\1\s*\)/g)].map((m) => m[2]).filter((n) => {
@@ -54,7 +69,7 @@ import { harnessSteps, ruleCoverage } from '../gates/test-quality.js';
54
69
  import { normalisePending } from './pending.js';
55
70
  import { packageRoot } from '../run/run-tests.js';
56
71
  import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
57
- import { factsCodeMoved, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
72
+ import { factsCodeMoved, journeyCodeHash, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
58
73
  import { generateSeed, seedProblem } from './seed-generator.js';
59
74
  import { dataAccess, writeRequiredData } from './required-data.js';
60
75
  import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
@@ -64,7 +79,7 @@ import { loadSourceChoice } from '../docs/source-of-truth.js';
64
79
  import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
65
80
  import { updateJourney } from './update-journey.js';
66
81
  import { sourceRoot, sourceText } from '../docs/excerpts.js';
67
- import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
82
+ import { isSyntaxValid, checkTestGates, trimExtraTests, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
68
83
  import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
69
84
  import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
70
85
  import { hashContent, loadCache, saveCache } from '../docs/cache.js';
@@ -113,7 +128,9 @@ export async function pool(items, limit, fn) {
113
128
  /** Async version of generateValidated (for concurrent per-file gen). */
114
129
  async function generateValidatedAsync(generate, ext, gate,
115
130
  /** Why an earlier draft of the same file (from a batch) was refused, with that draft. */
116
- firstRefusal) {
131
+ firstRefusal,
132
+ /** Mechanical fixes applied before the gate: what code can fix costs no retry. */
133
+ fix = (c) => c) {
117
134
  let content = '';
118
135
  let refusal = firstRefusal;
119
136
  for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
@@ -121,7 +138,7 @@ firstRefusal) {
121
138
  return null; // retrying a dead CLI only burns time
122
139
  try {
123
140
  // A second try sees what it wrote and why that was refused; asked blind, it made the same file again.
124
- content = await generate(refusal);
141
+ content = fix(await generate(refusal));
125
142
  }
126
143
  catch (e) {
127
144
  noteAiFailure(e);
@@ -294,7 +311,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
294
311
  // command. Test files are untouched.
295
312
  console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
296
313
  }
314
+ const old = existsSync(info.path) ? readFileSync(info.path, 'utf-8') : '';
297
315
  const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
316
+ // The tests already written call the old testkit: a rebuild that fails, or
317
+ // drops something they use, is not taken — the old one stays.
318
+ const lost = old && kit !== null ? exportNames(old).filter((n) => !exportNames(kit).includes(n)) : [];
319
+ if (old && (kit === null || lost.length)) {
320
+ console.log(` kept the existing ${area} test harness: the rebuild ${kit === null ? 'failed' : `dropped ${lost.join(', ')}`}`);
321
+ exportFind(info.path);
322
+ harnessByArea.set(area, info.path);
323
+ return info.path;
324
+ }
298
325
  if (kit === null) {
299
326
  harnessByArea.set(area, null);
300
327
  return undefined;
@@ -453,6 +480,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
453
480
  const appTexts = () => framework === 'playwright'
454
481
  ? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
455
482
  : undefined;
483
+ const fixTests = (content) => trimExtraTests(content, config.maxTestsPerRule ?? 2);
456
484
  const genGate = (content, filePath) => testGate(content) ??
457
485
  checkTestGates(content, ext, {
458
486
  framework, forbidPlaceholders: true, filePath,
@@ -463,7 +491,21 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
463
491
  // to retry (a resume pass fills those in without touching real files).
464
492
  // A suite whose tests leave rules out or cannot fail is not done either:
465
493
  // it goes on, and only those tests are written again.
494
+ // Rules asked for and refused before, with this doc and this code: not asked again until one moves.
495
+ const refusedPath = join(journeyDir(journey), 'refused.json');
496
+ let refusedKeyMemo;
497
+ const refusedKey = () => (refusedKeyMemo ??= `${hash}:${journeyCodeHash(journey, sourceRoot())}`);
498
+ const readRefused = () => {
499
+ try {
500
+ const r = JSON.parse(readFileSync(refusedPath, 'utf-8'));
501
+ return r.key === refusedKey() ? (r.rules ?? {}) : {};
502
+ }
503
+ catch {
504
+ return {};
505
+ }
506
+ };
466
507
  const suiteComplete = () => {
508
+ const refused = readRefused();
467
509
  const plan = readPlan(journey);
468
510
  if (!plan?.some((f) => f.rules?.length))
469
511
  return true;
@@ -475,7 +517,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
475
517
  return true;
476
518
  // Rules left without a test make a suite unfinished. A check that flags an
477
519
  // existing test does not: kept tests are reported, never rewritten.
478
- return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.length;
520
+ return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.filter((id) => !(refused[f.slug] ?? []).includes(id)).length;
479
521
  });
480
522
  };
481
523
  // Tests an earlier version appended after the outer describe go back inside
@@ -574,7 +616,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
574
616
  // What each file really tests once written: the memory and the report come from this, not the plan.
575
617
  const written = [];
576
618
  // need() names asked so far (the project's and this run's): handed to every later file.
577
- const askedNames = new Set(Object.keys(readInterface()).filter(isThingName));
619
+ // Only names the product has given are handed on for reuse: a blank one in
620
+ // that list reads as "use this name", and the new test waits on it again.
621
+ const filledNow = readInterface();
622
+ const askedNames = new Set(Object.keys(filledNow).filter((n) => isThingName(n) && String(filledNow[n] ?? '').trim()));
578
623
  const finish = async (content, pf, filePath, kept = false) => {
579
624
  if (!memory || !pf.rules?.length)
580
625
  return content;
@@ -582,13 +627,27 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
582
627
  const done = await completeFile(content, {
583
628
  kept, locked: passedTitles(filePath),
584
629
  redo: kept && stale.has(filePath) ? (redoAll ? () => true : (b) => usesBlankName(content, b)) : undefined,
630
+ isBlank: (n) => blankNames(`need('${n}')`).length > 0,
631
+ skip: new Set(readRefused()[pf.slug] ?? []),
632
+ fix: fixTests,
585
633
  journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
586
634
  harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
587
635
  check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
588
636
  });
589
637
  written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
638
+ // What was asked and still has no test is not asked again for this doc and code.
639
+ if (done.refusedRules.length) {
640
+ try {
641
+ const all = readRefused();
642
+ all[pf.slug] = [...new Set([...(all[pf.slug] ?? []), ...done.refusedRules])];
643
+ mkdirSync(dirname(refusedPath), { recursive: true });
644
+ writeFileSync(refusedPath, JSON.stringify({ key: refusedKey(), rules: all }, null, 2), 'utf-8');
645
+ }
646
+ catch { /* best effort */ }
647
+ }
590
648
  for (const n of neededNames(done.content))
591
- askedNames.add(n);
649
+ if (!blankNames(`need('${n}')`).length)
650
+ askedNames.add(n);
592
651
  for (const n of done.notes)
593
652
  warnings.push(`${pf.slug}: ${n}`);
594
653
  return done.content;
@@ -774,7 +833,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
774
833
  const got = await writeFilesWithAI(files);
775
834
  noteAiSuccess();
776
835
  for (const [p, c] of got)
777
- drafts.set(p, c);
836
+ drafts.set(p, fixTests(c));
778
837
  }
779
838
  catch (e) {
780
839
  noteAiFailure(e);
@@ -835,7 +894,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
835
894
  }
836
895
  }
837
896
  const { mem, facts } = await fileMemory(pf);
838
- const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
897
+ const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined, fixTests);
839
898
  if (ai !== null) {
840
899
  content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
841
900
  status = 'ai';
@@ -876,6 +935,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
876
935
  }
877
936
  return { slug: pf.slug, path: filePath, status };
878
937
  });
938
+ // The names the new tests still wait on, looked up once against this
939
+ // journey's code facts: a field the facts show is filled now, not left
940
+ // for the run. Kept only if the code itself has the value.
941
+ if (useAI && !notBuilt && memory) {
942
+ const facts = await grounding();
943
+ const files = planned.map(pathOf).filter((p) => existsSync(p));
944
+ const blank = [...new Set(files.flatMap((p) => { try {
945
+ return blankNames(readFileSync(p, 'utf-8'));
946
+ }
947
+ catch {
948
+ return [];
949
+ } }))];
950
+ if (facts && blank.length) {
951
+ const res = scanNames(blank, sourceRoot(), facts);
952
+ if (res.filled.length)
953
+ warnings.push(`filled ${res.filled.length} name(s) from the code facts: ${res.filled.slice(0, 6).map((f) => `${f.name} = ${f.value}`).join(', ')}${res.filled.length > 6 ? ', …' : ''}`);
954
+ }
955
+ }
879
956
  if (memory && written.length) {
880
957
  const sum = recordWritten(journey, written);
881
958
  console.log(` ${journey.id}: ${sum}`);
@@ -17,7 +17,9 @@ export function envNames(text) {
17
17
  const out = new Set();
18
18
  for (const line of text.split('\n')) {
19
19
  const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
20
- if (m && m[2].trim().replace(/^(['"])\1$/, '') !== '')
20
+ // An unquoted value ends at a comment: `KEY= # hint` (an unfilled template line) is unset.
21
+ const v = /^\s*(['"]).*\1\s*$/.test(m?.[2] ?? '') ? (m?.[2] ?? '').trim() : (m?.[2] ?? '').replace(/(^|\s)#.*$/, '').trim();
22
+ if (m && v.replace(/^(['"])\1$/, '') !== '')
21
23
  out.add(m[1]);
22
24
  }
23
25
  return out;
@@ -36,10 +36,18 @@ export async function runSetup(srcDir, opts = {}) {
36
36
  runSetupCall(loadConfig().relatedDirs ?? []);
37
37
  if (existsSync('testforge.config.json')) {
38
38
  const problems = checkSetupConfig();
39
- console.log(problems.length ? ` testforge.config.json written, with problems to fix:\n ${problems.join('\n ')}\n` : ' done — testforge.config.json');
39
+ if (problems.length) {
40
+ // Drafting docs on a config with problems spends AI calls on a project
41
+ // that cannot run yet: stop, and say what to fix.
42
+ console.log(` testforge.config.json written, with problems to fix first:\n ${problems.join('\n ')}\n`);
43
+ console.log('Fix these, then run `testforge setup` again (or `testforge discover`).');
44
+ return;
45
+ }
46
+ console.log(' done — testforge.config.json');
40
47
  }
41
48
  else {
42
- console.log(' warning: testforge.config.json was not created');
49
+ console.log(' testforge.config.json was not created — nothing else was done. Run `testforge setup` again.');
50
+ return;
43
51
  }
44
52
  console.log(existsSync('testforge.md') ? ' done — testforge.md\n' : ' warning: testforge.md was not created\n');
45
53
  // Step 3 — discover. This is where the source-of-truth barrier lives:
@@ -183,6 +183,9 @@ E2. What you touch comes from <code_facts> and <testkit>.
183
183
  - Look in ROUTES, SHAPES and DATA before need(): a name any of them shows is
184
184
  written as it is there. need() is only for a name none of them has (see
185
185
  NAMES). Never guess one.
186
+ - A path from need() may hold parameters ({booking}, :id): fill them with
187
+ fillPath(URL, value, …) from the same module as need(), never with
188
+ .replace() on a guessed placeholder name.
186
189
  - Never need() what the testkit already gives: kit.insert returns the new
187
190
  row's id; a second person of a role (another operator, another customer)
188
191
  is a row made with kit.insert; the roles are the ones <testkit> lists.
@@ -199,9 +202,11 @@ of making it.
199
202
  - Owner-scoped data ("my orders") follows the token owner, not a query
200
203
  parameter. A non-owner check needs a second token; without one, skip and
201
204
  name the missing identity.
202
- - If a precondition really cannot be met, skip at run time and state what was
203
- observed ("TEST_X is not set", "GET /x answered 404"), never a guessed
204
- cause.
205
+ - If a precondition really cannot be met (an account, an env variable or data
206
+ the environment lacks — never an element missing from the page), skip at
207
+ run time inside the test and state what was observed ("TEST_X is not set",
208
+ "GET /x answered 404"), never a guessed cause. That is the only skip: no
209
+ skipped test declarations, describe.skip or test.todo.
205
210
 
206
211
  E4. What you wait for is the event that actually happens. Wait on a
207
212
  condition, never for a fixed time.
@@ -290,8 +295,11 @@ export const WEB = `WEB (Playwright)
290
295
  title. A stubbed item carries every field the screen reads to show it,
291
296
  including the value the list filters on (its status), and the test finds it
292
297
  on screen by a field the screen renders as text, never by one it does not
293
- show. Match the stub by exact host, port, path prefix or a request
294
- predicate, never by a substring that also matches the app's own pages.
298
+ show. Finding it is not the check: assert what the product does with that
299
+ state (the badge, the message, the button it disables or hides), never only
300
+ that the text you stubbed appears. Match the stub by exact host, port, path
301
+ prefix or a request predicate, never by a substring that also matches the
302
+ app's own pages.
295
303
  - Waiting: prefer retrying assertions. Register waitForResponse or
296
304
  waitForRequest before the action that triggers it. After a login, sign-out
297
305
  or navigation, wait for the control you will use, then assert the URL the
@@ -18,6 +18,8 @@ import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
18
18
  import { runAiSync } from '../generate/ai-usage.js';
19
19
  import { clearInterface, readInterface, recordInterface } from '../generate/interface-file.js';
20
20
  const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
21
+ /** Why each still-blank name is blank, for the waiting message (name → reason). */
22
+ const STATUS_PATH = join(STATE_DIR, 'interface-status.json');
21
23
  /** Values the scan filled (name → value), so each run can check the code still has them. */
22
24
  const FILLED_PATH = join(STATE_DIR, 'interface-filled.json');
23
25
  const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
@@ -215,7 +217,27 @@ export function namesToAsk(blank, root, memo) {
215
217
  * Look for the blank names in the code and fill what is found. `blank` are
216
218
  * the names the tests to run still wait for. Returns what was filled.
217
219
  */
218
- export function scanNames(blank, root = '.') {
220
+ /** Is `value` in the project's code, as a key, string, route path or verb (the facts are read from it, so this checks them)? */
221
+ function inProjectCode(value, root) {
222
+ const files = codeFiles(root);
223
+ const route = files.filter((f) => fileMatches('route', relative(root, f)));
224
+ const read = (fs_) => fs_.flatMap((f) => { try {
225
+ return readFileSync(f, 'utf-8').split('\n');
226
+ }
227
+ catch {
228
+ return [];
229
+ } });
230
+ if (/^\/|^[\w-]+\//.test(value.trim()))
231
+ return foundInCode(value, read(route));
232
+ return foundInCode(value, read(files));
233
+ }
234
+ /**
235
+ * Look for the blank names in the code and fill what is found. `blank` are
236
+ * the names the tests to run still wait for. With `facts` (at generate time)
237
+ * every blank name is asked once against the journey's code facts too; a
238
+ * value is kept only if the code itself has it. Returns what was filled.
239
+ */
240
+ export function scanNames(blank, root = '.', facts) {
219
241
  const memo = (() => {
220
242
  try {
221
243
  return JSON.parse(readFileSync(SCAN_PATH, 'utf-8'));
@@ -225,6 +247,10 @@ export function scanNames(blank, root = '.') {
225
247
  }
226
248
  })();
227
249
  const all = namesToAsk(blank, root, memo);
250
+ if (facts)
251
+ for (const n of blank)
252
+ if (!all.some((a) => a.name === n))
253
+ all.push({ name: n, lines: [] });
228
254
  // One call stays small: names past the budget are asked on the next run.
229
255
  const ask = [];
230
256
  let size = 0;
@@ -239,11 +265,14 @@ export function scanNames(blank, root = '.') {
239
265
  }
240
266
  const filled = [];
241
267
  let calls = 0;
268
+ let callFailed = false;
242
269
  if (ask.length) {
243
270
  const known = Object.entries(readInterface()).filter(([, v]) => String(v).trim()).slice(0, 8);
244
271
  const prompt = [
245
272
  'You match names a test needs to values in code. For each name in <names>,',
246
- "pick its value from that name's own lines only, or null.",
273
+ facts
274
+ ? "pick its value from that name's own lines or from <code_facts> (the routes, request fields, response shapes and tables of this feature), or null."
275
+ : "pick its value from that name's own lines only, or null.",
247
276
  '',
248
277
  'A value is copied from the lines, never invented.',
249
278
  '- A URL is the path a client calls: it starts with "/" and includes every',
@@ -255,6 +284,7 @@ export function scanNames(blank, root = '.') {
255
284
  'If no line clearly is that thing, the value is null.',
256
285
  ...(known.length ? ['', '<filled_examples>', ...known.map(([k, v]) => `- ${k}: ${v}`), '</filled_examples>', 'These values are already filled; follow their format.'] : []),
257
286
  '',
287
+ ...(facts ? ['<code_facts>', facts.slice(0, 20_000), '</code_facts>', ''] : []),
258
288
  '<names>',
259
289
  ...ask.flatMap((a) => [`NAME: ${a.name}`, ...a.lines.map((l) => ` ${l}`)]),
260
290
  '</names>',
@@ -272,7 +302,7 @@ export function scanNames(blank, root = '.') {
272
302
  const keep = {};
273
303
  for (const a of ask) {
274
304
  const v = json[a.name];
275
- if (typeof v === 'string' && foundInCode(v, a.lines))
305
+ if (typeof v === 'string' && v.trim() && (foundInCode(v, a.lines) || (facts && inProjectCode(v, root))))
276
306
  keep[a.name] = v.trim();
277
307
  }
278
308
  recordInterface(keep);
@@ -297,6 +327,7 @@ export function scanNames(blank, root = '.') {
297
327
  console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
298
328
  for (const a of ask)
299
329
  delete memo[a.name]; // look again next time
330
+ callFailed = true;
300
331
  }
301
332
  }
302
333
  try {
@@ -307,6 +338,30 @@ export function scanNames(blank, root = '.') {
307
338
  // best-effort memo
308
339
  }
309
340
  const done = new Set(filled.map((f) => f.name));
341
+ // Why each name is still blank, so a waiting test says whether to look at
342
+ // the product or at TestForge.
343
+ try {
344
+ const status = (() => { try {
345
+ return JSON.parse(readFileSync(STATUS_PATH, 'utf-8'));
346
+ }
347
+ catch {
348
+ return {};
349
+ } })();
350
+ for (const n of blank) {
351
+ if (done.has(n)) {
352
+ delete status[n];
353
+ continue;
354
+ }
355
+ const asked = ask.find((a) => a.name === n);
356
+ status[n] = !kindOf(n) && !facts ? 'TestForge does not look this kind of name up in the code; fill it by hand'
357
+ : asked && callFailed ? 'the lookup could not run this time; it is tried again on the next run'
358
+ : asked ? 'looked for in the code and not found'
359
+ : 'nothing in the code looks like it yet';
360
+ }
361
+ mkdirSync(dirname(STATUS_PATH), { recursive: true });
362
+ writeFileSync(STATUS_PATH, `${JSON.stringify(status, null, 2)}\n`, 'utf-8');
363
+ }
364
+ catch { /* the message just has no reason */ }
310
365
  return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
311
366
  }
312
367
  /**
@@ -43,6 +43,8 @@ export function loadDotenv(cwd = process.cwd()) {
43
43
  let v = s.slice(i + 1).trim();
44
44
  if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))
45
45
  v = v.slice(1, -1);
46
+ else
47
+ v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
46
48
  if (k && process.env[k] === undefined)
47
49
  process.env[k] = v;
48
50
  }
@@ -110,6 +110,11 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
110
110
  }
111
111
  export function runTests(opts = {}) {
112
112
  const config = loadConfig();
113
+ // The journey goes into a shell command: only a journey id's characters.
114
+ if (opts.journey && !/^[\w.-]+$/.test(opts.journey)) {
115
+ console.log(`"${opts.journey}" is not a journey id (letters, digits, - _ . only) — nothing was run.`);
116
+ return 1;
117
+ }
113
118
  const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
114
119
  // An env file named on the command line that is not there is a mistake, not a default.
115
120
  if (opts.envFile && !existsSync(opts.envFile)) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.6.0",
3
+ "version": "2.7.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -10,7 +10,8 @@
10
10
  "files": [
11
11
  "dist",
12
12
  "reporters",
13
- "README.md"
13
+ "README.md",
14
+ "CHANGELOG.md"
14
15
  ],
15
16
  "scripts": {
16
17
  "build": "rm -rf dist && tsc",
@@ -103,7 +103,9 @@ function usersOf(file, name) {
103
103
  const end = close < 0 ? lines.length : lineOf(close) + 1;
104
104
  return { start: i, end, text: lines.slice(i, end).join('\n'), title: (/\(\s*(['"`])((?:(?!\1).)*)\1/.exec(lines[i]) || [])[2] };
105
105
  });
106
- const uses = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
106
+ const word = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
107
+ // A name inside a string ('redeem points URL') is not a use of the constant URL.
108
+ const uses = { test: (t) => word.test(String(t).replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, '""')) };
107
109
  const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
108
110
  if (lines.some((l, i) => i !== declLine && !inTest(i) && uses.test(l))) return undefined;
109
111
  return tests.filter((t) => uses.test(t.text));
@@ -159,14 +161,35 @@ function quietHooks() {
159
161
  }
160
162
  }
161
163
 
164
+ /** Why the last name scan left `name` blank, if it recorded one. */
165
+ function scanReason(name) {
166
+ try {
167
+ const p = path.join(path.dirname(interfacePath()), 'interface-status.json');
168
+ return JSON.parse(fs.readFileSync(p, 'utf-8'))[name];
169
+ } catch {
170
+ return undefined;
171
+ }
172
+ }
173
+
162
174
  /** The product's value for `name`, or an error naming what to fill. */
163
175
  function need(name) {
164
176
  const v = values()[name];
165
177
  if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
166
178
  // The prefix lets the reporters file this as "waiting for the product", not as a defect.
167
- const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json): this test runs for real once it is there.`;
179
+ const why = scanReason(name);
180
+ const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json)${why ? ` — ${why}` : ''}: this test runs for real once it is there.`;
168
181
  if (blockTestsThatUse(message, name)) return `<waiting: ${name}>`;
169
182
  throw new Error(message);
170
183
  }
171
184
 
172
- module.exports = { need, _usersOf: usersOf };
185
+ /**
186
+ * A path from need() with its parameters filled in order, whatever the
187
+ * product calls them: /bookings/{booking}/cancel, /orders/:id or {id}.
188
+ * fillPath(need('booking cancel URL'), bookingId)
189
+ */
190
+ function fillPath(template, ...values) {
191
+ let i = 0;
192
+ return String(template).replace(/\{[^}/]+\}|(?<=\/):[A-Za-z_]\w*/g, (m) => (i < values.length ? encodeURIComponent(String(values[i++])) : m));
193
+ }
194
+
195
+ module.exports = { need, fillPath, _usersOf: usersOf };