@activate-core/test-forge 2.5.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,8 +16,12 @@ import { dirname, join, relative } from 'node:path';
16
16
  import { STATE_DIR } from '../paths.js';
17
17
  import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
18
18
  import { runAiSync } from '../generate/ai-usage.js';
19
- import { readInterface, recordInterface } from '../generate/interface-file.js';
19
+ import { clearInterface, readInterface, recordInterface } from '../generate/interface-file.js';
20
20
  const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
21
+ /** Why each still-blank name is blank, for the waiting message (name → reason). */
22
+ const STATUS_PATH = join(STATE_DIR, 'interface-status.json');
23
+ /** Values the scan filled (name → value), so each run can check the code still has them. */
24
+ const FILLED_PATH = join(STATE_DIR, 'interface-filled.json');
21
25
  const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
22
26
  const CODE = /\.(php|ts|tsx|js|jsx|mjs|cjs|py|rb|go|vue|svelte|prisma|sql)$/;
23
27
  const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls', 'item', 'items', 'menu', 'link', 'entry', 'icon', 'text', 'title', 'page']);
@@ -26,11 +30,14 @@ export function kindOf(name) {
26
30
  const n = name.trim().toLowerCase();
27
31
  if (/\b(?:url|urls|path|prefix|endpoint|route)$/.test(n))
28
32
  return 'route';
33
+ // The HTTP method of a request lives on the route line (Route::post, router.post, @Post).
34
+ if (/\bmethod$/.test(n))
35
+ return 'method';
29
36
  if (/\bcommand$/.test(n))
30
37
  return 'command';
31
- if (/\b(?:table|column|model)$/.test(n))
38
+ if (/\b(?:table|column|model|class)$/.test(n))
32
39
  return 'schema';
33
- if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|setting)$/.test(n))
40
+ if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|value name|setting|role|permission|event|queue|job)$/.test(n))
34
41
  return 'field';
35
42
  if (/\blabel$/.test(n))
36
43
  return 'label';
@@ -39,28 +46,32 @@ export function kindOf(name) {
39
46
  /** Files of a kind: a route file, a console command, a migration or model, a resource or controller, a component. */
40
47
  function fileMatches(kind, rel) {
41
48
  switch (kind) {
42
- case 'route': return /(^|\/)(routes?|router|urls)(\/|\.[a-z]+$)|\.routes?\.[a-z]+$|\/routes?\.[a-z]+$|(^|\/)app\/.*\/(page|route)\.[jt]sx?$/i.test(rel);
49
+ case 'route':
50
+ case 'method': return /(^|\/)(routes?|router|urls)(\/|\.[a-z]+$)|\.routes?\.[a-z]+$|\/routes?\.[a-z]+$|(^|\/)app\/.*\/(page|route)\.[jt]sx?$|\.controller\.[jt]s$/i.test(rel);
43
51
  case 'command': return /(^|\/)(console|commands?|cli|bin)(\/|$)|command/i.test(rel);
44
52
  case 'schema': return /(^|\/)(migrations?|models?|entities|schema)(\/|\.)|\.prisma$|\.sql$/i.test(rel);
45
- case 'field': return /(^|\/)(resources?|transformers?|serializers?|controllers?|dto|types?|api)(\/|\.)|resource|serializer|controller/i.test(rel);
53
+ // A request's fields are where it is validated (FormRequests, rule and
54
+ // logic classes), a status value where its enum or model constant is.
55
+ case 'field': return /(^|\/)(resources?|transformers?|serializers?|controllers?|dto|types?|api|requests?|validators?|validation|rules?|logic|services?|models?|enums?|constants?|schemas?)(\/|\.)|resource|serializer|controller|request|validat|enum/i.test(rel);
46
56
  case 'label': return /\.(tsx|jsx|vue|svelte)$/.test(rel);
47
57
  }
48
58
  }
49
59
  /** What a line that DEFINES a name of each kind looks like. */
50
60
  const SHAPE = {
51
61
  // …or a URL the app keeps as a constant (faq: 'https://…').
62
+ method: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|export\s+(?:async\s+)?function\s+(?:GET|POST|PUT|PATCH|DELETE)\b/,
52
63
  route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]|\b\w+\s*:\s*['"`]https?:\/\//,
53
64
  command: /\$signature\s*=|\$name\s*=\s*['"]|->command\(\s*['"]|\.command\(\s*['"]|add_parser\(|@click\.command/,
54
65
  schema: /Schema::(?:create|table)\(|\$table->\w+\(\s*['"]|create_table|\bclass\s+\w+\s+extends\s+\w*Model\b|protected\s+\$(?:table|fillable)|\bmodel\s+\w+\s*\{|@(?:Entity|Column|Table)\b/,
55
- field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]/,
66
+ field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]|^\s*case\s+\w+\s*=|\bconst\s+[A-Z_]+\s*=/,
56
67
  // …or a menu entry's label (label: 'Orders').
57
68
  // …or a JSX text child on a line of its own (<Button>\n Escalate\n</Button>).
58
69
  label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]|\blabel\s*:\s*['"`]|^\s*[A-Z][\w'’ ]{1,40}\s*$|\/>\s*[A-Z][\w'’ ]{1,40}\s*$/,
59
70
  };
60
71
  /** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
61
72
  function candidateLines(name, kind, files, root) {
62
- const words = [...new Set(name.toLowerCase().match(/[a-z]{3,}/g) ?? [])]
63
- .filter((w) => !GENERIC.has(w))
73
+ const words = [...new Set(name.toLowerCase().match(/[a-z]{2,}/g) ?? [])]
74
+ .filter((w) => !GENERIC.has(w) && (w.length >= 3 || w === 'id'))
64
75
  .map((w) => (w.length > 5 ? w.replace(/(?:ings?|ed|es|s)$/, '') : w));
65
76
  if (!words.length)
66
77
  return [];
@@ -83,19 +94,46 @@ function candidateLines(name, kind, files, root) {
83
94
  lines.forEach((l, i) => {
84
95
  if (/^\s*(?:\/\/|#|\*)/.test(l) || l.length > 300)
85
96
  return;
97
+ // Log and console lines name things in prose, not as the product's keys.
98
+ if (/\b(?:Log::|logger\.|console\.|->(?:info|error|warning|debug)\()/.test(l))
99
+ return;
86
100
  const low = l.toLowerCase();
87
- if (kind === 'route' && /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>/.test(l))
101
+ // Whole identifier parts, not substrings: "order" is not in "orderBy",
102
+ // "out" is not in "Route"; address_id and addressId both give "address" and "id".
103
+ const parts = new Set(l.replace(/([a-z0-9])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
104
+ const has = (w) => parts.has(w) || [...parts].some((p) => p.length > 3 && (p.startsWith(w) || w.startsWith(p)) && Math.abs(p.length - w.length) <= 3);
105
+ if ((kind === 'route' || kind === 'method') && /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>/.test(l))
88
106
  prefixes.push(`${rel}:${i + 1}: ${l.trim()}`);
89
107
  if (!SHAPE[kind].test(l))
90
108
  return;
91
- const score = words.filter((w) => low.includes(w)).length;
109
+ const score = words.filter((w) => has(w)).length;
92
110
  if (score >= need)
93
- hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score });
111
+ hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score, file: rel, at: i, lines });
94
112
  });
95
113
  }
96
114
  if (!hits.length)
97
115
  return [];
98
- const best = hits.sort((a, b) => b.score - a.score).slice(0, 8).map((h) => h.line);
116
+ const top = hits.sort((a, b) => b.score - a.score).slice(0, 8);
117
+ const best = top.map((h) => h.line);
118
+ if (kind === 'route' || kind === 'method') {
119
+ // The prefixes of the groups each route sits in (by indentation), not
120
+ // whichever prefixes come first in its file.
121
+ const PREFIX = /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>|Route::group\(/;
122
+ const indent = (l) => l.length - l.trimStart().length;
123
+ const enclosing = [];
124
+ for (const h of top) {
125
+ let cur = indent(h.lines[h.at]);
126
+ for (let j = h.at - 1; j >= 0 && cur > 0; j--) {
127
+ const l = h.lines[j];
128
+ if (!l.trim() || indent(l) >= cur)
129
+ continue;
130
+ cur = indent(l);
131
+ if (PREFIX.test(l))
132
+ enclosing.push(`${h.file}:${j + 1}: ${l.trim()}`);
133
+ }
134
+ }
135
+ return [...best, ...new Set(enclosing)].slice(0, 16);
136
+ }
99
137
  // A route's full path needs the prefix of the group it sits in: those of its own files ride along.
100
138
  const files_ = new Set(best.map((l) => l.split(':')[0]));
101
139
  return [...best, ...prefixes.filter((p) => files_.has(p.split(':')[0])).slice(0, 6)];
@@ -140,6 +178,17 @@ export function foundInCode(value, lines) {
140
178
  const parts = v.split('/').filter((p) => p && p !== 'api' && !/^[{:<]/.test(p));
141
179
  return parts.length > 0 && parts.every((p) => text.includes(p));
142
180
  }
181
+ // An HTTP method: Route::post / router.post / @Post / export function POST.
182
+ if (/^(?:get|post|put|patch|delete)$/i.test(v))
183
+ return new RegExp(`\\b${v}\\b`, 'i').test(text);
184
+ // A nested response field (error.message): each part is a key in the code.
185
+ if (/^[\w-]+(?:\.[\w-]+)+$/.test(v) && !/^\d/.test(v))
186
+ return v.split('.').every((p) => text.includes(p));
187
+ // A short or numeric value ("80", "on") must stand as a value, not inside another word.
188
+ if (v.length < 4 || /^\d+(?:\.\d+)?$/.test(v)) {
189
+ const esc = v.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
190
+ return new RegExp(`(['"\`])${esc}\\1|(?:=>|=|:)\\s*${esc}\\b`).test(text);
191
+ }
143
192
  return text.includes(v);
144
193
  }
145
194
  /**
@@ -168,7 +217,27 @@ export function namesToAsk(blank, root, memo) {
168
217
  * Look for the blank names in the code and fill what is found. `blank` are
169
218
  * the names the tests to run still wait for. Returns what was filled.
170
219
  */
171
- export function scanNames(blank, root = '.') {
220
+ /** Is `value` in the project's code, as a key, string, route path or verb (the facts are read from it, so this checks them)? */
221
+ function inProjectCode(value, root) {
222
+ const files = codeFiles(root);
223
+ const route = files.filter((f) => fileMatches('route', relative(root, f)));
224
+ const read = (fs_) => fs_.flatMap((f) => { try {
225
+ return readFileSync(f, 'utf-8').split('\n');
226
+ }
227
+ catch {
228
+ return [];
229
+ } });
230
+ if (/^\/|^[\w-]+\//.test(value.trim()))
231
+ return foundInCode(value, read(route));
232
+ return foundInCode(value, read(files));
233
+ }
234
+ /**
235
+ * Look for the blank names in the code and fill what is found. `blank` are
236
+ * the names the tests to run still wait for. With `facts` (at generate time)
237
+ * every blank name is asked once against the journey's code facts too; a
238
+ * value is kept only if the code itself has it. Returns what was filled.
239
+ */
240
+ export function scanNames(blank, root = '.', facts) {
172
241
  const memo = (() => {
173
242
  try {
174
243
  return JSON.parse(readFileSync(SCAN_PATH, 'utf-8'));
@@ -178,6 +247,10 @@ export function scanNames(blank, root = '.') {
178
247
  }
179
248
  })();
180
249
  const all = namesToAsk(blank, root, memo);
250
+ if (facts)
251
+ for (const n of blank)
252
+ if (!all.some((a) => a.name === n))
253
+ all.push({ name: n, lines: [] });
181
254
  // One call stays small: names past the budget are asked on the next run.
182
255
  const ask = [];
183
256
  let size = 0;
@@ -192,20 +265,26 @@ export function scanNames(blank, root = '.') {
192
265
  }
193
266
  const filled = [];
194
267
  let calls = 0;
268
+ let callFailed = false;
195
269
  if (ask.length) {
196
270
  const known = Object.entries(readInterface()).filter(([, v]) => String(v).trim()).slice(0, 8);
197
271
  const prompt = [
198
272
  'You match names a test needs to values in code. For each name in <names>,',
199
- "pick its value from that name's own lines only, or null.",
273
+ facts
274
+ ? "pick its value from that name's own lines or from <code_facts> (the routes, request fields, response shapes and tables of this feature), or null."
275
+ : "pick its value from that name's own lines only, or null.",
200
276
  '',
201
277
  'A value is copied from the lines, never invented.',
202
278
  '- A URL is the path a client calls: it starts with "/" and includes every',
203
279
  ' route prefix the lines show (a route inside a prefix group gets the prefix).',
204
280
  '- A command is the command name a person types.',
205
- '- A field, key or column is the exact key.',
281
+ '- A field, key or column is the exact key. A field inside an object is its path with dots (error.message).',
282
+ '- A request method is the HTTP verb in capitals (POST).',
283
+ '- A status or enum value is the stored value, not the constant\'s name.',
206
284
  'If no line clearly is that thing, the value is null.',
207
285
  ...(known.length ? ['', '<filled_examples>', ...known.map(([k, v]) => `- ${k}: ${v}`), '</filled_examples>', 'These values are already filled; follow their format.'] : []),
208
286
  '',
287
+ ...(facts ? ['<code_facts>', facts.slice(0, 20_000), '</code_facts>', ''] : []),
209
288
  '<names>',
210
289
  ...ask.flatMap((a) => [`NAME: ${a.name}`, ...a.lines.map((l) => ` ${l}`)]),
211
290
  '</names>',
@@ -223,17 +302,32 @@ export function scanNames(blank, root = '.') {
223
302
  const keep = {};
224
303
  for (const a of ask) {
225
304
  const v = json[a.name];
226
- if (typeof v === 'string' && foundInCode(v, a.lines))
305
+ if (typeof v === 'string' && v.trim() && (foundInCode(v, a.lines) || (facts && inProjectCode(v, root))))
227
306
  keep[a.name] = v.trim();
228
307
  }
229
308
  recordInterface(keep);
230
- for (const [name, value] of Object.entries(keep))
309
+ for (const [name, value] of Object.entries(keep)) {
231
310
  filled.push({ name, value });
311
+ // Asked again if someone blanks it later, even with the same lines.
312
+ memo[name] = `filled:${memo[name]}`;
313
+ }
314
+ try {
315
+ const prev = JSON.parse(readFileSync(FILLED_PATH, 'utf-8'));
316
+ writeFileSync(FILLED_PATH, `${JSON.stringify({ ...prev, ...keep }, null, 2)}\n`, 'utf-8');
317
+ }
318
+ catch {
319
+ try {
320
+ mkdirSync(dirname(FILLED_PATH), { recursive: true });
321
+ writeFileSync(FILLED_PATH, `${JSON.stringify(keep, null, 2)}\n`, 'utf-8');
322
+ }
323
+ catch { /* best effort */ }
324
+ }
232
325
  }
233
326
  catch (e) {
234
327
  console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
235
328
  for (const a of ask)
236
329
  delete memo[a.name]; // look again next time
330
+ callFailed = true;
237
331
  }
238
332
  }
239
333
  try {
@@ -244,5 +338,72 @@ export function scanNames(blank, root = '.') {
244
338
  // best-effort memo
245
339
  }
246
340
  const done = new Set(filled.map((f) => f.name));
341
+ // Why each name is still blank, so a waiting test says whether to look at
342
+ // the product or at TestForge.
343
+ try {
344
+ const status = (() => { try {
345
+ return JSON.parse(readFileSync(STATUS_PATH, 'utf-8'));
346
+ }
347
+ catch {
348
+ return {};
349
+ } })();
350
+ for (const n of blank) {
351
+ if (done.has(n)) {
352
+ delete status[n];
353
+ continue;
354
+ }
355
+ const asked = ask.find((a) => a.name === n);
356
+ status[n] = !kindOf(n) && !facts ? 'TestForge does not look this kind of name up in the code; fill it by hand'
357
+ : asked && callFailed ? 'the lookup could not run this time; it is tried again on the next run'
358
+ : asked ? 'looked for in the code and not found'
359
+ : 'nothing in the code looks like it yet';
360
+ }
361
+ mkdirSync(dirname(STATUS_PATH), { recursive: true });
362
+ writeFileSync(STATUS_PATH, `${JSON.stringify(status, null, 2)}\n`, 'utf-8');
363
+ }
364
+ catch { /* the message just has no reason */ }
247
365
  return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
248
366
  }
367
+ /**
368
+ * Values the scan filled earlier that the code no longer has (a route or field
369
+ * renamed): blanked again, so their tests report "waiting" rather than failing
370
+ * as if the product were wrong, and the next scan looks for the new name. No AI.
371
+ */
372
+ export function recheckFilled(root = '.') {
373
+ let filled;
374
+ try {
375
+ filled = JSON.parse(readFileSync(FILLED_PATH, 'utf-8'));
376
+ }
377
+ catch {
378
+ return [];
379
+ }
380
+ const current = readInterface();
381
+ const files = codeFiles(root);
382
+ const gone = [];
383
+ for (const [name, value] of Object.entries(filled)) {
384
+ if (String(current[name] ?? '').trim() !== value)
385
+ continue; // changed or set by hand: not ours to judge
386
+ const kind = kindOf(name);
387
+ if (!kind)
388
+ continue;
389
+ const lines = files.filter((f) => fileMatches(kind, relative(root, f))).flatMap((f) => {
390
+ try {
391
+ return readFileSync(f, 'utf-8').split('\n');
392
+ }
393
+ catch {
394
+ return [];
395
+ }
396
+ });
397
+ if (!foundInCode(value, lines))
398
+ gone.push(name);
399
+ }
400
+ if (gone.length) {
401
+ clearInterface(gone);
402
+ const left = Object.fromEntries(Object.entries(filled).filter(([n]) => !gone.includes(n)));
403
+ try {
404
+ writeFileSync(FILLED_PATH, `${JSON.stringify(left, null, 2)}\n`, 'utf-8');
405
+ }
406
+ catch { /* best effort */ }
407
+ }
408
+ return gone;
409
+ }
@@ -43,6 +43,8 @@ export function loadDotenv(cwd = process.cwd()) {
43
43
  let v = s.slice(i + 1).trim();
44
44
  if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))
45
45
  v = v.slice(1, -1);
46
+ else
47
+ v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
46
48
  if (k && process.env[k] === undefined)
47
49
  process.env[k] = v;
48
50
  }
@@ -75,8 +77,9 @@ export function mapPlaywright(json, category = 'web') {
75
77
  for (const t of spec.tests || []) {
76
78
  const r = (t.results || [])[(t.results || []).length - 1] || {};
77
79
  const st = r.status || t.status;
78
- const status = st === 'passed' ? 'passed' : st === 'skipped' ? 'skipped' : 'failed';
79
80
  const raw = r.error && r.error.message;
81
+ const notProduct = st !== 'passed' && st !== 'skipped' ? shared.notTheProduct(raw) : undefined;
82
+ const status = st === 'passed' ? 'passed' : st === 'skipped' || notProduct ? 'skipped' : 'failed';
80
83
  const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
81
84
  const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
82
85
  const err = r.error || {};
@@ -95,8 +98,8 @@ export function mapPlaywright(json, category = 'web') {
95
98
  ...(detail ? { detail } : {}),
96
99
  error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
97
100
  sig: status === 'failed' ? shared.sig(raw) : undefined,
98
- note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
99
- ...(status === 'failed' ? verdictOf(loc, spec.title) : {}),
101
+ note: status === 'skipped' ? notProduct || (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
102
+ ...(status === 'failed' ? verdictOf(loc, spec.title, raw) : {}),
100
103
  });
101
104
  }
102
105
  }
@@ -114,14 +117,16 @@ export function mapJest(json, category = 'api', testDir) {
114
117
  const name = shared.suiteName(f.name || f.testFilePath || 'suite', testDir);
115
118
  const su = (suites[name] ||= { name, category, tests: [] });
116
119
  for (const a of f.assertionResults || []) {
117
- const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
118
120
  const raw = a.failureMessages && a.failureMessages[0];
121
+ const notProduct = a.status === 'failed' ? shared.notTheProduct(raw) : undefined;
122
+ const status = a.status === 'passed' ? 'passed' : a.status === 'failed' && !notProduct ? 'failed' : 'skipped';
119
123
  const err = strip(raw || '').split('\n').find((l) => l.trim()) || '';
120
124
  su.tests.push({
121
125
  name: a.title,
122
126
  status,
123
127
  error: status === 'failed' ? err.slice(0, 200) : undefined,
124
128
  sig: status === 'failed' ? shared.sig(raw) : undefined,
129
+ ...(notProduct ? { note: notProduct } : {}),
125
130
  });
126
131
  }
127
132
  }
@@ -12,8 +12,8 @@
12
12
  * It runs from the area's nearest package root (where package.json /
13
13
  * playwright.config live), with config.envFile merged into the env.
14
14
  */
15
- import { testFilesIn } from './readiness.js';
16
- import { scanNames } from './name-scan.js';
15
+ import { PENDING_STAMP, testFilesIn } from './readiness.js';
16
+ import { recheckFilled, scanNames } from './name-scan.js';
17
17
  import { INTERFACE_PATH, blankNames, neededNames } from '../generate/interface-file.js';
18
18
  import { harnessInfo } from '../generate/harness.js';
19
19
  import { missingPackages } from '../generate/verify.js';
@@ -110,7 +110,17 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
110
110
  }
111
111
  export function runTests(opts = {}) {
112
112
  const config = loadConfig();
113
+ // The journey goes into a shell command: only a journey id's characters.
114
+ if (opts.journey && !/^[\w.-]+$/.test(opts.journey)) {
115
+ console.log(`"${opts.journey}" is not a journey id (letters, digits, - _ . only) — nothing was run.`);
116
+ return 1;
117
+ }
113
118
  const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
119
+ // An env file named on the command line that is not there is a mistake, not a default.
120
+ if (opts.envFile && !existsSync(opts.envFile)) {
121
+ console.log(`The env file "${opts.envFile}" does not exist — nothing was run.`);
122
+ return 1;
123
+ }
114
124
  const envVars = loadEnvFile(envFile);
115
125
  if (Object.keys(envVars).length > 0) {
116
126
  console.log(`Loaded env from ${envFile} (${Object.keys(envVars).length} vars).`);
@@ -156,8 +166,31 @@ export function runTests(opts = {}) {
156
166
  // says so. Before the run, the names those tests wait for are looked for
157
167
  // in the code — only where that kind of name lives, only when those lines
158
168
  // changed — and filled; the tests themselves are never touched.
159
- const targets = undefined;
169
+ let targets = undefined;
170
+ // A feature the doc says is not live yet: its files are held back, not run
171
+ // to fail. They run once the doc's Status line is removed and they are generated again.
172
+ {
173
+ const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
174
+ const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
175
+ const held = files.filter((f) => { try {
176
+ return PENDING_STAMP.test(readFileSync(f, 'utf-8'));
177
+ }
178
+ catch {
179
+ return false;
180
+ } });
181
+ if (held.length) {
182
+ const run = files.filter((f) => !held.includes(f));
183
+ console.log(` held back (feature not live yet): ${held.length} file(s)`);
184
+ if (!run.length) {
185
+ console.log(`[skip] ${area}: every test file waits for a feature that is not live yet`);
186
+ continue;
187
+ }
188
+ targets = run.map((f) => resolve(f));
189
+ }
190
+ }
160
191
  if (!opts.noScan) {
192
+ for (const n of recheckFilled(root))
193
+ console.log(` no longer in the code: ${n} — blanked; its tests wait until the new name is found`);
161
194
  const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
162
195
  const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
163
196
  const blank = blankNames([...new Set(files.flatMap((f) => neededNames(readFileSync(f, 'utf-8'))))]);
@@ -176,8 +209,16 @@ export function runTests(opts = {}) {
176
209
  ranAnything = true;
177
210
  console.log(`\n=== ${area} (${framework}) ===`);
178
211
  console.log(`$ ${cmd} [cwd: ${root}]`);
212
+ const started = Date.now();
179
213
  const res = spawnSync(cmd, { shell: true, cwd: root, env, stdio: 'inherit' });
180
- if (res.status !== 0)
214
+ // The reporters' counts: a run whose only "failures" are tests waiting for
215
+ // the product or stopped by the environment has no product failure.
216
+ const sum = runSummary(root, started);
217
+ if (sum) {
218
+ const ran = sum.passed + sum.failed;
219
+ console.log(`\n${area}: ${ran} of ${sum.total} test(s) ran — ${sum.passed} passed, ${sum.failed} failed; ${sum.skipped} did not run (waiting for the product, environment, or skipped)`);
220
+ }
221
+ if (res.status !== 0 && !(sum && sum.failed === 0 && sum.total > 0))
181
222
  anyFail = true;
182
223
  }
183
224
  if (!ranAnything) {
@@ -190,3 +231,16 @@ export function runTests(opts = {}) {
190
231
  console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
191
232
  return anyFail ? 1 : 0;
192
233
  }
234
+ /** Counts the reporters wrote for runs that started at or after `since`, summed. */
235
+ function runSummary(root, since) {
236
+ try {
237
+ const all = JSON.parse(readFileSync(join(root, 'testforge', 'last-run.json'), 'utf-8'));
238
+ const fresh = Object.values(all).filter((r) => Date.parse(r.at) >= since - 1000);
239
+ if (!fresh.length)
240
+ return undefined;
241
+ return fresh.reduce((a, r) => ({ total: a.total + r.total, passed: a.passed + r.passed, failed: a.failed + r.failed, skipped: a.skipped + r.skipped }), { total: 0, passed: 0, failed: 0, skipped: 0 });
242
+ }
243
+ catch {
244
+ return undefined;
245
+ }
246
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.5.0",
3
+ "version": "2.7.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -10,12 +10,14 @@
10
10
  "files": [
11
11
  "dist",
12
12
  "reporters",
13
- "README.md"
13
+ "README.md",
14
+ "CHANGELOG.md"
14
15
  ],
15
16
  "scripts": {
16
17
  "build": "rm -rf dist && tsc",
17
18
  "dev": "tsc --watch",
18
- "test": "tsx --test test/*.test.ts"
19
+ "test": "tsx --test test/*.test.ts",
20
+ "prepublishOnly": "npm run build && npm test"
19
21
  },
20
22
  "license": "MIT",
21
23
  "publishConfig": {
@@ -31,6 +31,7 @@ function loadEnv(root) {
31
31
  const k = s.slice(0, i).trim();
32
32
  let v = s.slice(i + 1).trim();
33
33
  if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
34
+ else v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
34
35
  if (k && process.env[k] === undefined) process.env[k] = v;
35
36
  }
36
37
  }
@@ -395,12 +396,19 @@ function reviewVerdicts(root) {
395
396
  try {
396
397
  for (const r of JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) || []) {
397
398
  for (const t of r.tests || []) {
398
- map.set(`${path.resolve(String(t.file))}|${t.title}`, { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) });
399
+ map.set(`${path.resolve(String(t.file))}|${t.title}`, { at: sig(t.detail), v: { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) } });
399
400
  }
400
401
  }
401
402
  } catch { /* a half-written review is ignored */ }
402
403
  }
403
- return (file, title) => (file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined);
404
+ // A verdict is about one failure: the same test failing differently later is a
405
+ // new failure, and an old "test fault" must not hide a real regression.
406
+ return (file, title, raw) => {
407
+ const hit = file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined;
408
+ if (!hit) return undefined;
409
+ if (raw !== undefined && hit.at && sig(raw) !== hit.at) return undefined;
410
+ return hit.v;
411
+ };
404
412
  }
405
413
 
406
414
  /** The name a test is still waiting for (need() threw for a blank interface name), else undefined. */
@@ -431,4 +439,49 @@ function notTheProduct(raw) {
431
439
  return environmentFor(raw);
432
440
  }
433
441
 
434
- module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
442
+ /**
443
+ * Remember the tests that passed (testforge/passed.json): generate never removes
444
+ * or rewrites a test that has passed. Add-only — a later failure is a regression
445
+ * to report, not a reason to forget that the test once checked the product.
446
+ */
447
+ function recordPassed(root, passed) {
448
+ if (!passed.length) return;
449
+ const file = path.join(root, 'testforge', 'passed.json');
450
+ let all = {};
451
+ try { all = JSON.parse(fs.readFileSync(file, 'utf-8')) || {}; } catch { all = {}; }
452
+ let changed = false;
453
+ for (const t of passed) {
454
+ if (!t.file || !t.title) continue;
455
+ const rel = path.relative(root, path.resolve(t.file)).split(path.sep).join('/');
456
+ const list = Array.isArray(all[rel]) ? all[rel] : (all[rel] = []);
457
+ if (!list.includes(t.title)) { list.push(t.title); changed = true; }
458
+ }
459
+ if (!changed) return;
460
+ try {
461
+ fs.mkdirSync(path.dirname(file), { recursive: true });
462
+ fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n');
463
+ } catch (e) {
464
+ console.log(`[testforge] could not record passed tests: ${e.message}`);
465
+ }
466
+ }
467
+
468
+ /**
469
+ * The run's counts, for `testforge run` to read: how many ran of how many, and
470
+ * whether anything failed for a product reason. Waiting and environment stops
471
+ * are "skipped" here, as on the dashboard — never a failure of the run.
472
+ */
473
+ function recordSummary(root, category, suites) {
474
+ const tests = suites.flatMap((s) => s.tests || []);
475
+ const n = (st) => tests.filter((t) => t.status === st).length;
476
+ const out = { category, at: new Date().toISOString(), total: tests.length, passed: n('passed'), failed: n('failed'), skipped: n('skipped') };
477
+ try {
478
+ const file = path.join(root, 'testforge', 'last-run.json');
479
+ let all = {};
480
+ try { all = JSON.parse(fs.readFileSync(file, 'utf-8')) || {}; } catch { all = {}; }
481
+ all[category] = out;
482
+ fs.mkdirSync(path.dirname(file), { recursive: true });
483
+ fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n');
484
+ } catch { /* the run's own output still has the counts */ }
485
+ }
486
+
487
+ module.exports = { recordSummary, recordPassed, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
@@ -103,7 +103,9 @@ function usersOf(file, name) {
103
103
  const end = close < 0 ? lines.length : lineOf(close) + 1;
104
104
  return { start: i, end, text: lines.slice(i, end).join('\n'), title: (/\(\s*(['"`])((?:(?!\1).)*)\1/.exec(lines[i]) || [])[2] };
105
105
  });
106
- const uses = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
106
+ const word = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
107
+ // A name inside a string ('redeem points URL') is not a use of the constant URL.
108
+ const uses = { test: (t) => word.test(String(t).replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, '""')) };
107
109
  const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
108
110
  if (lines.some((l, i) => i !== declLine && !inTest(i) && uses.test(l))) return undefined;
109
111
  return tests.filter((t) => uses.test(t.text));
@@ -134,6 +136,10 @@ function blockTestsThatUse(message, name) {
134
136
  try { fullName = globalThis.expect.getState().currentTestName; } catch { /* unknown */ }
135
137
  if (needs(undefined, fullName)) throw new Error(message);
136
138
  });
139
+ // Used outside the tests (a hook or a helper): every test waits, so the
140
+ // file's own hooks must not run either, or a beforeAll/afterEach would
141
+ // fail on the placeholder and hide the "waiting" reason.
142
+ if (!users) quietHooks();
137
143
  return true;
138
144
  }
139
145
  require('@playwright/test').test.beforeEach(async ({}, testInfo) => { // Playwright
@@ -145,14 +151,45 @@ function blockTestsThatUse(message, name) {
145
151
  }
146
152
  }
147
153
 
154
+ /**
155
+ * Jest: the hooks this test file registers from here on do nothing. Each test
156
+ * file has its own globals, so no other file is touched.
157
+ */
158
+ function quietHooks() {
159
+ for (const h of ['beforeAll', 'afterAll', 'beforeEach', 'afterEach']) {
160
+ if (typeof globalThis[h] === 'function') globalThis[h] = () => {};
161
+ }
162
+ }
163
+
164
+ /** Why the last name scan left `name` blank, if it recorded one. */
165
+ function scanReason(name) {
166
+ try {
167
+ const p = path.join(path.dirname(interfacePath()), 'interface-status.json');
168
+ return JSON.parse(fs.readFileSync(p, 'utf-8'))[name];
169
+ } catch {
170
+ return undefined;
171
+ }
172
+ }
173
+
148
174
  /** The product's value for `name`, or an error naming what to fill. */
149
175
  function need(name) {
150
176
  const v = values()[name];
151
177
  if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
152
178
  // The prefix lets the reporters file this as "waiting for the product", not as a defect.
153
- const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json): this test runs for real once it is there.`;
179
+ const why = scanReason(name);
180
+ const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json)${why ? ` — ${why}` : ''}: this test runs for real once it is there.`;
154
181
  if (blockTestsThatUse(message, name)) return `<waiting: ${name}>`;
155
182
  throw new Error(message);
156
183
  }
157
184
 
158
- module.exports = { need, _usersOf: usersOf };
185
+ /**
186
+ * A path from need() with its parameters filled in order, whatever the
187
+ * product calls them: /bookings/{booking}/cancel, /orders/:id or {id}.
188
+ * fillPath(need('booking cancel URL'), bookingId)
189
+ */
190
+ function fillPath(template, ...values) {
191
+ let i = 0;
192
+ return String(template).replace(/\{[^}/]+\}|(?<=\/):[A-Za-z_]\w*/g, (m) => (i < values.length ? encodeURIComponent(String(values[i++])) : m));
193
+ }
194
+
195
+ module.exports = { need, fillPath, _usersOf: usersOf };