@activate-core/test-forge 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +24 -3
- package/dist/cli/index.js +2 -1
- package/dist/core/docs/excerpts.js +186 -6
- package/dist/core/docs/grounding.js +82 -16
- package/dist/core/docs/reference-text.js +2 -0
- package/dist/core/gates/coverage.js +9 -0
- package/dist/core/gates/validate.js +50 -23
- package/dist/core/generate/ai-generator.js +9 -3
- package/dist/core/generate/ai-usage.js +5 -2
- package/dist/core/generate/complete-file.js +70 -10
- package/dist/core/generate/defect-review.js +2 -1
- package/dist/core/generate/harness.js +34 -0
- package/dist/core/generate/interface-file.js +12 -0
- package/dist/core/generate/journey-memory.js +7 -1
- package/dist/core/generate/passed.js +24 -0
- package/dist/core/generate/run-generate.js +201 -20
- package/dist/core/generate/test-blocks.js +34 -10
- package/dist/core/project/config.js +5 -0
- package/dist/core/project/env-diff.js +3 -1
- package/dist/core/project/setup.js +10 -2
- package/dist/core/prompts/blocks.js +37 -8
- package/dist/core/run/name-scan.js +178 -17
- package/dist/core/run/report.js +9 -4
- package/dist/core/run/run-tests.js +58 -4
- package/package.json +5 -3
- package/reporters/_shared.cjs +56 -3
- package/reporters/interface.cjs +40 -3
- package/reporters/jest-reporter.cjs +7 -3
- package/reporters/playwright-reporter.cjs +6 -2
- package/reporters/run-data.cjs +6 -1
|
@@ -16,8 +16,12 @@ import { dirname, join, relative } from 'node:path';
|
|
|
16
16
|
import { STATE_DIR } from '../paths.js';
|
|
17
17
|
import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
18
18
|
import { runAiSync } from '../generate/ai-usage.js';
|
|
19
|
-
import { readInterface, recordInterface } from '../generate/interface-file.js';
|
|
19
|
+
import { clearInterface, readInterface, recordInterface } from '../generate/interface-file.js';
|
|
20
20
|
const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
|
|
21
|
+
/** Why each still-blank name is blank, for the waiting message (name → reason). */
|
|
22
|
+
const STATUS_PATH = join(STATE_DIR, 'interface-status.json');
|
|
23
|
+
/** Values the scan filled (name → value), so each run can check the code still has them. */
|
|
24
|
+
const FILLED_PATH = join(STATE_DIR, 'interface-filled.json');
|
|
21
25
|
const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
|
|
22
26
|
const CODE = /\.(php|ts|tsx|js|jsx|mjs|cjs|py|rb|go|vue|svelte|prisma|sql)$/;
|
|
23
27
|
const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls', 'item', 'items', 'menu', 'link', 'entry', 'icon', 'text', 'title', 'page']);
|
|
@@ -26,11 +30,14 @@ export function kindOf(name) {
|
|
|
26
30
|
const n = name.trim().toLowerCase();
|
|
27
31
|
if (/\b(?:url|urls|path|prefix|endpoint|route)$/.test(n))
|
|
28
32
|
return 'route';
|
|
33
|
+
// The HTTP method of a request lives on the route line (Route::post, router.post, @Post).
|
|
34
|
+
if (/\bmethod$/.test(n))
|
|
35
|
+
return 'method';
|
|
29
36
|
if (/\bcommand$/.test(n))
|
|
30
37
|
return 'command';
|
|
31
|
-
if (/\b(?:table|column|model)$/.test(n))
|
|
38
|
+
if (/\b(?:table|column|model|class)$/.test(n))
|
|
32
39
|
return 'schema';
|
|
33
|
-
if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|setting)$/.test(n))
|
|
40
|
+
if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|value name|setting|role|permission|event|queue|job)$/.test(n))
|
|
34
41
|
return 'field';
|
|
35
42
|
if (/\blabel$/.test(n))
|
|
36
43
|
return 'label';
|
|
@@ -39,28 +46,32 @@ export function kindOf(name) {
|
|
|
39
46
|
/** Files of a kind: a route file, a console command, a migration or model, a resource or controller, a component. */
|
|
40
47
|
function fileMatches(kind, rel) {
|
|
41
48
|
switch (kind) {
|
|
42
|
-
case 'route':
|
|
49
|
+
case 'route':
|
|
50
|
+
case 'method': return /(^|\/)(routes?|router|urls)(\/|\.[a-z]+$)|\.routes?\.[a-z]+$|\/routes?\.[a-z]+$|(^|\/)app\/.*\/(page|route)\.[jt]sx?$|\.controller\.[jt]s$/i.test(rel);
|
|
43
51
|
case 'command': return /(^|\/)(console|commands?|cli|bin)(\/|$)|command/i.test(rel);
|
|
44
52
|
case 'schema': return /(^|\/)(migrations?|models?|entities|schema)(\/|\.)|\.prisma$|\.sql$/i.test(rel);
|
|
45
|
-
|
|
53
|
+
// A request's fields are where it is validated (FormRequests, rule and
|
|
54
|
+
// logic classes), a status value where its enum or model constant is.
|
|
55
|
+
case 'field': return /(^|\/)(resources?|transformers?|serializers?|controllers?|dto|types?|api|requests?|validators?|validation|rules?|logic|services?|models?|enums?|constants?|schemas?)(\/|\.)|resource|serializer|controller|request|validat|enum/i.test(rel);
|
|
46
56
|
case 'label': return /\.(tsx|jsx|vue|svelte)$/.test(rel);
|
|
47
57
|
}
|
|
48
58
|
}
|
|
49
59
|
/** What a line that DEFINES a name of each kind looks like. */
|
|
50
60
|
const SHAPE = {
|
|
51
61
|
// …or a URL the app keeps as a constant (faq: 'https://…').
|
|
62
|
+
method: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|export\s+(?:async\s+)?function\s+(?:GET|POST|PUT|PATCH|DELETE)\b/,
|
|
52
63
|
route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]|\b\w+\s*:\s*['"`]https?:\/\//,
|
|
53
64
|
command: /\$signature\s*=|\$name\s*=\s*['"]|->command\(\s*['"]|\.command\(\s*['"]|add_parser\(|@click\.command/,
|
|
54
65
|
schema: /Schema::(?:create|table)\(|\$table->\w+\(\s*['"]|create_table|\bclass\s+\w+\s+extends\s+\w*Model\b|protected\s+\$(?:table|fillable)|\bmodel\s+\w+\s*\{|@(?:Entity|Column|Table)\b/,
|
|
55
|
-
field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]
|
|
66
|
+
field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]|^\s*case\s+\w+\s*=|\bconst\s+[A-Z_]+\s*=/,
|
|
56
67
|
// …or a menu entry's label (label: 'Orders').
|
|
57
68
|
// …or a JSX text child on a line of its own (<Button>\n Escalate\n</Button>).
|
|
58
69
|
label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]|\blabel\s*:\s*['"`]|^\s*[A-Z][\w'’ ]{1,40}\s*$|\/>\s*[A-Z][\w'’ ]{1,40}\s*$/,
|
|
59
70
|
};
|
|
60
71
|
/** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
|
|
61
72
|
function candidateLines(name, kind, files, root) {
|
|
62
|
-
const words = [...new Set(name.toLowerCase().match(/[a-z]{
|
|
63
|
-
.filter((w) => !GENERIC.has(w))
|
|
73
|
+
const words = [...new Set(name.toLowerCase().match(/[a-z]{2,}/g) ?? [])]
|
|
74
|
+
.filter((w) => !GENERIC.has(w) && (w.length >= 3 || w === 'id'))
|
|
64
75
|
.map((w) => (w.length > 5 ? w.replace(/(?:ings?|ed|es|s)$/, '') : w));
|
|
65
76
|
if (!words.length)
|
|
66
77
|
return [];
|
|
@@ -83,19 +94,46 @@ function candidateLines(name, kind, files, root) {
|
|
|
83
94
|
lines.forEach((l, i) => {
|
|
84
95
|
if (/^\s*(?:\/\/|#|\*)/.test(l) || l.length > 300)
|
|
85
96
|
return;
|
|
97
|
+
// Log and console lines name things in prose, not as the product's keys.
|
|
98
|
+
if (/\b(?:Log::|logger\.|console\.|->(?:info|error|warning|debug)\()/.test(l))
|
|
99
|
+
return;
|
|
86
100
|
const low = l.toLowerCase();
|
|
87
|
-
|
|
101
|
+
// Whole identifier parts, not substrings: "order" is not in "orderBy",
|
|
102
|
+
// "out" is not in "Route"; address_id and addressId both give "address" and "id".
|
|
103
|
+
const parts = new Set(l.replace(/([a-z0-9])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
|
|
104
|
+
const has = (w) => parts.has(w) || [...parts].some((p) => p.length > 3 && (p.startsWith(w) || w.startsWith(p)) && Math.abs(p.length - w.length) <= 3);
|
|
105
|
+
if ((kind === 'route' || kind === 'method') && /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>/.test(l))
|
|
88
106
|
prefixes.push(`${rel}:${i + 1}: ${l.trim()}`);
|
|
89
107
|
if (!SHAPE[kind].test(l))
|
|
90
108
|
return;
|
|
91
|
-
const score = words.filter((w) =>
|
|
109
|
+
const score = words.filter((w) => has(w)).length;
|
|
92
110
|
if (score >= need)
|
|
93
|
-
hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score });
|
|
111
|
+
hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score, file: rel, at: i, lines });
|
|
94
112
|
});
|
|
95
113
|
}
|
|
96
114
|
if (!hits.length)
|
|
97
115
|
return [];
|
|
98
|
-
const
|
|
116
|
+
const top = hits.sort((a, b) => b.score - a.score).slice(0, 8);
|
|
117
|
+
const best = top.map((h) => h.line);
|
|
118
|
+
if (kind === 'route' || kind === 'method') {
|
|
119
|
+
// The prefixes of the groups each route sits in (by indentation), not
|
|
120
|
+
// whichever prefixes come first in its file.
|
|
121
|
+
const PREFIX = /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>|Route::group\(/;
|
|
122
|
+
const indent = (l) => l.length - l.trimStart().length;
|
|
123
|
+
const enclosing = [];
|
|
124
|
+
for (const h of top) {
|
|
125
|
+
let cur = indent(h.lines[h.at]);
|
|
126
|
+
for (let j = h.at - 1; j >= 0 && cur > 0; j--) {
|
|
127
|
+
const l = h.lines[j];
|
|
128
|
+
if (!l.trim() || indent(l) >= cur)
|
|
129
|
+
continue;
|
|
130
|
+
cur = indent(l);
|
|
131
|
+
if (PREFIX.test(l))
|
|
132
|
+
enclosing.push(`${h.file}:${j + 1}: ${l.trim()}`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return [...best, ...new Set(enclosing)].slice(0, 16);
|
|
136
|
+
}
|
|
99
137
|
// A route's full path needs the prefix of the group it sits in: those of its own files ride along.
|
|
100
138
|
const files_ = new Set(best.map((l) => l.split(':')[0]));
|
|
101
139
|
return [...best, ...prefixes.filter((p) => files_.has(p.split(':')[0])).slice(0, 6)];
|
|
@@ -140,6 +178,17 @@ export function foundInCode(value, lines) {
|
|
|
140
178
|
const parts = v.split('/').filter((p) => p && p !== 'api' && !/^[{:<]/.test(p));
|
|
141
179
|
return parts.length > 0 && parts.every((p) => text.includes(p));
|
|
142
180
|
}
|
|
181
|
+
// An HTTP method: Route::post / router.post / @Post / export function POST.
|
|
182
|
+
if (/^(?:get|post|put|patch|delete)$/i.test(v))
|
|
183
|
+
return new RegExp(`\\b${v}\\b`, 'i').test(text);
|
|
184
|
+
// A nested response field (error.message): each part is a key in the code.
|
|
185
|
+
if (/^[\w-]+(?:\.[\w-]+)+$/.test(v) && !/^\d/.test(v))
|
|
186
|
+
return v.split('.').every((p) => text.includes(p));
|
|
187
|
+
// A short or numeric value ("80", "on") must stand as a value, not inside another word.
|
|
188
|
+
if (v.length < 4 || /^\d+(?:\.\d+)?$/.test(v)) {
|
|
189
|
+
const esc = v.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
190
|
+
return new RegExp(`(['"\`])${esc}\\1|(?:=>|=|:)\\s*${esc}\\b`).test(text);
|
|
191
|
+
}
|
|
143
192
|
return text.includes(v);
|
|
144
193
|
}
|
|
145
194
|
/**
|
|
@@ -168,7 +217,27 @@ export function namesToAsk(blank, root, memo) {
|
|
|
168
217
|
* Look for the blank names in the code and fill what is found. `blank` are
|
|
169
218
|
* the names the tests to run still wait for. Returns what was filled.
|
|
170
219
|
*/
|
|
171
|
-
|
|
220
|
+
/** Is `value` in the project's code, as a key, string, route path or verb (the facts are read from it, so this checks them)? */
|
|
221
|
+
function inProjectCode(value, root) {
|
|
222
|
+
const files = codeFiles(root);
|
|
223
|
+
const route = files.filter((f) => fileMatches('route', relative(root, f)));
|
|
224
|
+
const read = (fs_) => fs_.flatMap((f) => { try {
|
|
225
|
+
return readFileSync(f, 'utf-8').split('\n');
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
return [];
|
|
229
|
+
} });
|
|
230
|
+
if (/^\/|^[\w-]+\//.test(value.trim()))
|
|
231
|
+
return foundInCode(value, read(route));
|
|
232
|
+
return foundInCode(value, read(files));
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Look for the blank names in the code and fill what is found. `blank` are
|
|
236
|
+
* the names the tests to run still wait for. With `facts` (at generate time)
|
|
237
|
+
* every blank name is asked once against the journey's code facts too; a
|
|
238
|
+
* value is kept only if the code itself has it. Returns what was filled.
|
|
239
|
+
*/
|
|
240
|
+
export function scanNames(blank, root = '.', facts) {
|
|
172
241
|
const memo = (() => {
|
|
173
242
|
try {
|
|
174
243
|
return JSON.parse(readFileSync(SCAN_PATH, 'utf-8'));
|
|
@@ -178,6 +247,10 @@ export function scanNames(blank, root = '.') {
|
|
|
178
247
|
}
|
|
179
248
|
})();
|
|
180
249
|
const all = namesToAsk(blank, root, memo);
|
|
250
|
+
if (facts)
|
|
251
|
+
for (const n of blank)
|
|
252
|
+
if (!all.some((a) => a.name === n))
|
|
253
|
+
all.push({ name: n, lines: [] });
|
|
181
254
|
// One call stays small: names past the budget are asked on the next run.
|
|
182
255
|
const ask = [];
|
|
183
256
|
let size = 0;
|
|
@@ -192,20 +265,26 @@ export function scanNames(blank, root = '.') {
|
|
|
192
265
|
}
|
|
193
266
|
const filled = [];
|
|
194
267
|
let calls = 0;
|
|
268
|
+
let callFailed = false;
|
|
195
269
|
if (ask.length) {
|
|
196
270
|
const known = Object.entries(readInterface()).filter(([, v]) => String(v).trim()).slice(0, 8);
|
|
197
271
|
const prompt = [
|
|
198
272
|
'You match names a test needs to values in code. For each name in <names>,',
|
|
199
|
-
|
|
273
|
+
facts
|
|
274
|
+
? "pick its value from that name's own lines or from <code_facts> (the routes, request fields, response shapes and tables of this feature), or null."
|
|
275
|
+
: "pick its value from that name's own lines only, or null.",
|
|
200
276
|
'',
|
|
201
277
|
'A value is copied from the lines, never invented.',
|
|
202
278
|
'- A URL is the path a client calls: it starts with "/" and includes every',
|
|
203
279
|
' route prefix the lines show (a route inside a prefix group gets the prefix).',
|
|
204
280
|
'- A command is the command name a person types.',
|
|
205
|
-
'- A field, key or column is the exact key.',
|
|
281
|
+
'- A field, key or column is the exact key. A field inside an object is its path with dots (error.message).',
|
|
282
|
+
'- A request method is the HTTP verb in capitals (POST).',
|
|
283
|
+
'- A status or enum value is the stored value, not the constant\'s name.',
|
|
206
284
|
'If no line clearly is that thing, the value is null.',
|
|
207
285
|
...(known.length ? ['', '<filled_examples>', ...known.map(([k, v]) => `- ${k}: ${v}`), '</filled_examples>', 'These values are already filled; follow their format.'] : []),
|
|
208
286
|
'',
|
|
287
|
+
...(facts ? ['<code_facts>', facts.slice(0, 20_000), '</code_facts>', ''] : []),
|
|
209
288
|
'<names>',
|
|
210
289
|
...ask.flatMap((a) => [`NAME: ${a.name}`, ...a.lines.map((l) => ` ${l}`)]),
|
|
211
290
|
'</names>',
|
|
@@ -223,17 +302,32 @@ export function scanNames(blank, root = '.') {
|
|
|
223
302
|
const keep = {};
|
|
224
303
|
for (const a of ask) {
|
|
225
304
|
const v = json[a.name];
|
|
226
|
-
if (typeof v === 'string' && foundInCode(v, a.lines))
|
|
305
|
+
if (typeof v === 'string' && v.trim() && (foundInCode(v, a.lines) || (facts && inProjectCode(v, root))))
|
|
227
306
|
keep[a.name] = v.trim();
|
|
228
307
|
}
|
|
229
308
|
recordInterface(keep);
|
|
230
|
-
for (const [name, value] of Object.entries(keep))
|
|
309
|
+
for (const [name, value] of Object.entries(keep)) {
|
|
231
310
|
filled.push({ name, value });
|
|
311
|
+
// Asked again if someone blanks it later, even with the same lines.
|
|
312
|
+
memo[name] = `filled:${memo[name]}`;
|
|
313
|
+
}
|
|
314
|
+
try {
|
|
315
|
+
const prev = JSON.parse(readFileSync(FILLED_PATH, 'utf-8'));
|
|
316
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify({ ...prev, ...keep }, null, 2)}\n`, 'utf-8');
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
try {
|
|
320
|
+
mkdirSync(dirname(FILLED_PATH), { recursive: true });
|
|
321
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify(keep, null, 2)}\n`, 'utf-8');
|
|
322
|
+
}
|
|
323
|
+
catch { /* best effort */ }
|
|
324
|
+
}
|
|
232
325
|
}
|
|
233
326
|
catch (e) {
|
|
234
327
|
console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
|
|
235
328
|
for (const a of ask)
|
|
236
329
|
delete memo[a.name]; // look again next time
|
|
330
|
+
callFailed = true;
|
|
237
331
|
}
|
|
238
332
|
}
|
|
239
333
|
try {
|
|
@@ -244,5 +338,72 @@ export function scanNames(blank, root = '.') {
|
|
|
244
338
|
// best-effort memo
|
|
245
339
|
}
|
|
246
340
|
const done = new Set(filled.map((f) => f.name));
|
|
341
|
+
// Why each name is still blank, so a waiting test says whether to look at
|
|
342
|
+
// the product or at TestForge.
|
|
343
|
+
try {
|
|
344
|
+
const status = (() => { try {
|
|
345
|
+
return JSON.parse(readFileSync(STATUS_PATH, 'utf-8'));
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
return {};
|
|
349
|
+
} })();
|
|
350
|
+
for (const n of blank) {
|
|
351
|
+
if (done.has(n)) {
|
|
352
|
+
delete status[n];
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
const asked = ask.find((a) => a.name === n);
|
|
356
|
+
status[n] = !kindOf(n) && !facts ? 'TestForge does not look this kind of name up in the code; fill it by hand'
|
|
357
|
+
: asked && callFailed ? 'the lookup could not run this time; it is tried again on the next run'
|
|
358
|
+
: asked ? 'looked for in the code and not found'
|
|
359
|
+
: 'nothing in the code looks like it yet';
|
|
360
|
+
}
|
|
361
|
+
mkdirSync(dirname(STATUS_PATH), { recursive: true });
|
|
362
|
+
writeFileSync(STATUS_PATH, `${JSON.stringify(status, null, 2)}\n`, 'utf-8');
|
|
363
|
+
}
|
|
364
|
+
catch { /* the message just has no reason */ }
|
|
247
365
|
return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
|
|
248
366
|
}
|
|
367
|
+
/**
|
|
368
|
+
* Values the scan filled earlier that the code no longer has (a route or field
|
|
369
|
+
* renamed): blanked again, so their tests report "waiting" rather than failing
|
|
370
|
+
* as if the product were wrong, and the next scan looks for the new name. No AI.
|
|
371
|
+
*/
|
|
372
|
+
export function recheckFilled(root = '.') {
|
|
373
|
+
let filled;
|
|
374
|
+
try {
|
|
375
|
+
filled = JSON.parse(readFileSync(FILLED_PATH, 'utf-8'));
|
|
376
|
+
}
|
|
377
|
+
catch {
|
|
378
|
+
return [];
|
|
379
|
+
}
|
|
380
|
+
const current = readInterface();
|
|
381
|
+
const files = codeFiles(root);
|
|
382
|
+
const gone = [];
|
|
383
|
+
for (const [name, value] of Object.entries(filled)) {
|
|
384
|
+
if (String(current[name] ?? '').trim() !== value)
|
|
385
|
+
continue; // changed or set by hand: not ours to judge
|
|
386
|
+
const kind = kindOf(name);
|
|
387
|
+
if (!kind)
|
|
388
|
+
continue;
|
|
389
|
+
const lines = files.filter((f) => fileMatches(kind, relative(root, f))).flatMap((f) => {
|
|
390
|
+
try {
|
|
391
|
+
return readFileSync(f, 'utf-8').split('\n');
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
return [];
|
|
395
|
+
}
|
|
396
|
+
});
|
|
397
|
+
if (!foundInCode(value, lines))
|
|
398
|
+
gone.push(name);
|
|
399
|
+
}
|
|
400
|
+
if (gone.length) {
|
|
401
|
+
clearInterface(gone);
|
|
402
|
+
const left = Object.fromEntries(Object.entries(filled).filter(([n]) => !gone.includes(n)));
|
|
403
|
+
try {
|
|
404
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify(left, null, 2)}\n`, 'utf-8');
|
|
405
|
+
}
|
|
406
|
+
catch { /* best effort */ }
|
|
407
|
+
}
|
|
408
|
+
return gone;
|
|
409
|
+
}
|
package/dist/core/run/report.js
CHANGED
|
@@ -43,6 +43,8 @@ export function loadDotenv(cwd = process.cwd()) {
|
|
|
43
43
|
let v = s.slice(i + 1).trim();
|
|
44
44
|
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))
|
|
45
45
|
v = v.slice(1, -1);
|
|
46
|
+
else
|
|
47
|
+
v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
|
|
46
48
|
if (k && process.env[k] === undefined)
|
|
47
49
|
process.env[k] = v;
|
|
48
50
|
}
|
|
@@ -75,8 +77,9 @@ export function mapPlaywright(json, category = 'web') {
|
|
|
75
77
|
for (const t of spec.tests || []) {
|
|
76
78
|
const r = (t.results || [])[(t.results || []).length - 1] || {};
|
|
77
79
|
const st = r.status || t.status;
|
|
78
|
-
const status = st === 'passed' ? 'passed' : st === 'skipped' ? 'skipped' : 'failed';
|
|
79
80
|
const raw = r.error && r.error.message;
|
|
81
|
+
const notProduct = st !== 'passed' && st !== 'skipped' ? shared.notTheProduct(raw) : undefined;
|
|
82
|
+
const status = st === 'passed' ? 'passed' : st === 'skipped' || notProduct ? 'skipped' : 'failed';
|
|
80
83
|
const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
|
|
81
84
|
const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
|
|
82
85
|
const err = r.error || {};
|
|
@@ -95,8 +98,8 @@ export function mapPlaywright(json, category = 'web') {
|
|
|
95
98
|
...(detail ? { detail } : {}),
|
|
96
99
|
error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
|
|
97
100
|
sig: status === 'failed' ? shared.sig(raw) : undefined,
|
|
98
|
-
note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
|
|
99
|
-
...(status === 'failed' ? verdictOf(loc, spec.title) : {}),
|
|
101
|
+
note: status === 'skipped' ? notProduct || (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
|
|
102
|
+
...(status === 'failed' ? verdictOf(loc, spec.title, raw) : {}),
|
|
100
103
|
});
|
|
101
104
|
}
|
|
102
105
|
}
|
|
@@ -114,14 +117,16 @@ export function mapJest(json, category = 'api', testDir) {
|
|
|
114
117
|
const name = shared.suiteName(f.name || f.testFilePath || 'suite', testDir);
|
|
115
118
|
const su = (suites[name] ||= { name, category, tests: [] });
|
|
116
119
|
for (const a of f.assertionResults || []) {
|
|
117
|
-
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
|
|
118
120
|
const raw = a.failureMessages && a.failureMessages[0];
|
|
121
|
+
const notProduct = a.status === 'failed' ? shared.notTheProduct(raw) : undefined;
|
|
122
|
+
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' && !notProduct ? 'failed' : 'skipped';
|
|
119
123
|
const err = strip(raw || '').split('\n').find((l) => l.trim()) || '';
|
|
120
124
|
su.tests.push({
|
|
121
125
|
name: a.title,
|
|
122
126
|
status,
|
|
123
127
|
error: status === 'failed' ? err.slice(0, 200) : undefined,
|
|
124
128
|
sig: status === 'failed' ? shared.sig(raw) : undefined,
|
|
129
|
+
...(notProduct ? { note: notProduct } : {}),
|
|
125
130
|
});
|
|
126
131
|
}
|
|
127
132
|
}
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
* It runs from the area's nearest package root (where package.json /
|
|
13
13
|
* playwright.config live), with config.envFile merged into the env.
|
|
14
14
|
*/
|
|
15
|
-
import { testFilesIn } from './readiness.js';
|
|
16
|
-
import { scanNames } from './name-scan.js';
|
|
15
|
+
import { PENDING_STAMP, testFilesIn } from './readiness.js';
|
|
16
|
+
import { recheckFilled, scanNames } from './name-scan.js';
|
|
17
17
|
import { INTERFACE_PATH, blankNames, neededNames } from '../generate/interface-file.js';
|
|
18
18
|
import { harnessInfo } from '../generate/harness.js';
|
|
19
19
|
import { missingPackages } from '../generate/verify.js';
|
|
@@ -110,7 +110,17 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
|
|
|
110
110
|
}
|
|
111
111
|
export function runTests(opts = {}) {
|
|
112
112
|
const config = loadConfig();
|
|
113
|
+
// The journey goes into a shell command: only a journey id's characters.
|
|
114
|
+
if (opts.journey && !/^[\w.-]+$/.test(opts.journey)) {
|
|
115
|
+
console.log(`"${opts.journey}" is not a journey id (letters, digits, - _ . only) — nothing was run.`);
|
|
116
|
+
return 1;
|
|
117
|
+
}
|
|
113
118
|
const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
|
|
119
|
+
// An env file named on the command line that is not there is a mistake, not a default.
|
|
120
|
+
if (opts.envFile && !existsSync(opts.envFile)) {
|
|
121
|
+
console.log(`The env file "${opts.envFile}" does not exist — nothing was run.`);
|
|
122
|
+
return 1;
|
|
123
|
+
}
|
|
114
124
|
const envVars = loadEnvFile(envFile);
|
|
115
125
|
if (Object.keys(envVars).length > 0) {
|
|
116
126
|
console.log(`Loaded env from ${envFile} (${Object.keys(envVars).length} vars).`);
|
|
@@ -156,8 +166,31 @@ export function runTests(opts = {}) {
|
|
|
156
166
|
// says so. Before the run, the names those tests wait for are looked for
|
|
157
167
|
// in the code — only where that kind of name lives, only when those lines
|
|
158
168
|
// changed — and filled; the tests themselves are never touched.
|
|
159
|
-
|
|
169
|
+
let targets = undefined;
|
|
170
|
+
// A feature the doc says is not live yet: its files are held back, not run
|
|
171
|
+
// to fail. They run once the doc's Status line is removed and they are generated again.
|
|
172
|
+
{
|
|
173
|
+
const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
|
|
174
|
+
const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
|
|
175
|
+
const held = files.filter((f) => { try {
|
|
176
|
+
return PENDING_STAMP.test(readFileSync(f, 'utf-8'));
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return false;
|
|
180
|
+
} });
|
|
181
|
+
if (held.length) {
|
|
182
|
+
const run = files.filter((f) => !held.includes(f));
|
|
183
|
+
console.log(` held back (feature not live yet): ${held.length} file(s)`);
|
|
184
|
+
if (!run.length) {
|
|
185
|
+
console.log(`[skip] ${area}: every test file waits for a feature that is not live yet`);
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
targets = run.map((f) => resolve(f));
|
|
189
|
+
}
|
|
190
|
+
}
|
|
160
191
|
if (!opts.noScan) {
|
|
192
|
+
for (const n of recheckFilled(root))
|
|
193
|
+
console.log(` no longer in the code: ${n} — blanked; its tests wait until the new name is found`);
|
|
161
194
|
const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
|
|
162
195
|
const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
|
|
163
196
|
const blank = blankNames([...new Set(files.flatMap((f) => neededNames(readFileSync(f, 'utf-8'))))]);
|
|
@@ -176,8 +209,16 @@ export function runTests(opts = {}) {
|
|
|
176
209
|
ranAnything = true;
|
|
177
210
|
console.log(`\n=== ${area} (${framework}) ===`);
|
|
178
211
|
console.log(`$ ${cmd} [cwd: ${root}]`);
|
|
212
|
+
const started = Date.now();
|
|
179
213
|
const res = spawnSync(cmd, { shell: true, cwd: root, env, stdio: 'inherit' });
|
|
180
|
-
|
|
214
|
+
// The reporters' counts: a run whose only "failures" are tests waiting for
|
|
215
|
+
// the product or stopped by the environment has no product failure.
|
|
216
|
+
const sum = runSummary(root, started);
|
|
217
|
+
if (sum) {
|
|
218
|
+
const ran = sum.passed + sum.failed;
|
|
219
|
+
console.log(`\n${area}: ${ran} of ${sum.total} test(s) ran — ${sum.passed} passed, ${sum.failed} failed; ${sum.skipped} did not run (waiting for the product, environment, or skipped)`);
|
|
220
|
+
}
|
|
221
|
+
if (res.status !== 0 && !(sum && sum.failed === 0 && sum.total > 0))
|
|
181
222
|
anyFail = true;
|
|
182
223
|
}
|
|
183
224
|
if (!ranAnything) {
|
|
@@ -190,3 +231,16 @@ export function runTests(opts = {}) {
|
|
|
190
231
|
console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
|
|
191
232
|
return anyFail ? 1 : 0;
|
|
192
233
|
}
|
|
234
|
+
/** Counts the reporters wrote for runs that started at or after `since`, summed. */
|
|
235
|
+
function runSummary(root, since) {
|
|
236
|
+
try {
|
|
237
|
+
const all = JSON.parse(readFileSync(join(root, 'testforge', 'last-run.json'), 'utf-8'));
|
|
238
|
+
const fresh = Object.values(all).filter((r) => Date.parse(r.at) >= since - 1000);
|
|
239
|
+
if (!fresh.length)
|
|
240
|
+
return undefined;
|
|
241
|
+
return fresh.reduce((a, r) => ({ total: a.total + r.total, passed: a.passed + r.passed, failed: a.failed + r.failed, skipped: a.skipped + r.skipped }), { total: 0, passed: 0, failed: 0, skipped: 0 });
|
|
242
|
+
}
|
|
243
|
+
catch {
|
|
244
|
+
return undefined;
|
|
245
|
+
}
|
|
246
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -10,12 +10,14 @@
|
|
|
10
10
|
"files": [
|
|
11
11
|
"dist",
|
|
12
12
|
"reporters",
|
|
13
|
-
"README.md"
|
|
13
|
+
"README.md",
|
|
14
|
+
"CHANGELOG.md"
|
|
14
15
|
],
|
|
15
16
|
"scripts": {
|
|
16
17
|
"build": "rm -rf dist && tsc",
|
|
17
18
|
"dev": "tsc --watch",
|
|
18
|
-
"test": "tsx --test test/*.test.ts"
|
|
19
|
+
"test": "tsx --test test/*.test.ts",
|
|
20
|
+
"prepublishOnly": "npm run build && npm test"
|
|
19
21
|
},
|
|
20
22
|
"license": "MIT",
|
|
21
23
|
"publishConfig": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -31,6 +31,7 @@ function loadEnv(root) {
|
|
|
31
31
|
const k = s.slice(0, i).trim();
|
|
32
32
|
let v = s.slice(i + 1).trim();
|
|
33
33
|
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
|
|
34
|
+
else v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
|
|
34
35
|
if (k && process.env[k] === undefined) process.env[k] = v;
|
|
35
36
|
}
|
|
36
37
|
}
|
|
@@ -395,12 +396,19 @@ function reviewVerdicts(root) {
|
|
|
395
396
|
try {
|
|
396
397
|
for (const r of JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) || []) {
|
|
397
398
|
for (const t of r.tests || []) {
|
|
398
|
-
map.set(`${path.resolve(String(t.file))}|${t.title}`, { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) });
|
|
399
|
+
map.set(`${path.resolve(String(t.file))}|${t.title}`, { at: sig(t.detail), v: { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) } });
|
|
399
400
|
}
|
|
400
401
|
}
|
|
401
402
|
} catch { /* a half-written review is ignored */ }
|
|
402
403
|
}
|
|
403
|
-
|
|
404
|
+
// A verdict is about one failure: the same test failing differently later is a
|
|
405
|
+
// new failure, and an old "test fault" must not hide a real regression.
|
|
406
|
+
return (file, title, raw) => {
|
|
407
|
+
const hit = file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined;
|
|
408
|
+
if (!hit) return undefined;
|
|
409
|
+
if (raw !== undefined && hit.at && sig(raw) !== hit.at) return undefined;
|
|
410
|
+
return hit.v;
|
|
411
|
+
};
|
|
404
412
|
}
|
|
405
413
|
|
|
406
414
|
/** The name a test is still waiting for (need() threw for a blank interface name), else undefined. */
|
|
@@ -431,4 +439,49 @@ function notTheProduct(raw) {
|
|
|
431
439
|
return environmentFor(raw);
|
|
432
440
|
}
|
|
433
441
|
|
|
434
|
-
|
|
442
|
+
/**
|
|
443
|
+
* Remember the tests that passed (testforge/passed.json): generate never removes
|
|
444
|
+
* or rewrites a test that has passed. Add-only — a later failure is a regression
|
|
445
|
+
* to report, not a reason to forget that the test once checked the product.
|
|
446
|
+
*/
|
|
447
|
+
function recordPassed(root, passed) {
|
|
448
|
+
if (!passed.length) return;
|
|
449
|
+
const file = path.join(root, 'testforge', 'passed.json');
|
|
450
|
+
let all = {};
|
|
451
|
+
try { all = JSON.parse(fs.readFileSync(file, 'utf-8')) || {}; } catch { all = {}; }
|
|
452
|
+
let changed = false;
|
|
453
|
+
for (const t of passed) {
|
|
454
|
+
if (!t.file || !t.title) continue;
|
|
455
|
+
const rel = path.relative(root, path.resolve(t.file)).split(path.sep).join('/');
|
|
456
|
+
const list = Array.isArray(all[rel]) ? all[rel] : (all[rel] = []);
|
|
457
|
+
if (!list.includes(t.title)) { list.push(t.title); changed = true; }
|
|
458
|
+
}
|
|
459
|
+
if (!changed) return;
|
|
460
|
+
try {
|
|
461
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
462
|
+
fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n');
|
|
463
|
+
} catch (e) {
|
|
464
|
+
console.log(`[testforge] could not record passed tests: ${e.message}`);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* The run's counts, for `testforge run` to read: how many ran of how many, and
|
|
470
|
+
* whether anything failed for a product reason. Waiting and environment stops
|
|
471
|
+
* are "skipped" here, as on the dashboard — never a failure of the run.
|
|
472
|
+
*/
|
|
473
|
+
function recordSummary(root, category, suites) {
|
|
474
|
+
const tests = suites.flatMap((s) => s.tests || []);
|
|
475
|
+
const n = (st) => tests.filter((t) => t.status === st).length;
|
|
476
|
+
const out = { category, at: new Date().toISOString(), total: tests.length, passed: n('passed'), failed: n('failed'), skipped: n('skipped') };
|
|
477
|
+
try {
|
|
478
|
+
const file = path.join(root, 'testforge', 'last-run.json');
|
|
479
|
+
let all = {};
|
|
480
|
+
try { all = JSON.parse(fs.readFileSync(file, 'utf-8')) || {}; } catch { all = {}; }
|
|
481
|
+
all[category] = out;
|
|
482
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
483
|
+
fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n');
|
|
484
|
+
} catch { /* the run's own output still has the counts */ }
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
module.exports = { recordSummary, recordPassed, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
|
package/reporters/interface.cjs
CHANGED
|
@@ -103,7 +103,9 @@ function usersOf(file, name) {
|
|
|
103
103
|
const end = close < 0 ? lines.length : lineOf(close) + 1;
|
|
104
104
|
return { start: i, end, text: lines.slice(i, end).join('\n'), title: (/\(\s*(['"`])((?:(?!\1).)*)\1/.exec(lines[i]) || [])[2] };
|
|
105
105
|
});
|
|
106
|
-
const
|
|
106
|
+
const word = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
|
|
107
|
+
// A name inside a string ('redeem points URL') is not a use of the constant URL.
|
|
108
|
+
const uses = { test: (t) => word.test(String(t).replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, '""')) };
|
|
107
109
|
const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
|
|
108
110
|
if (lines.some((l, i) => i !== declLine && !inTest(i) && uses.test(l))) return undefined;
|
|
109
111
|
return tests.filter((t) => uses.test(t.text));
|
|
@@ -134,6 +136,10 @@ function blockTestsThatUse(message, name) {
|
|
|
134
136
|
try { fullName = globalThis.expect.getState().currentTestName; } catch { /* unknown */ }
|
|
135
137
|
if (needs(undefined, fullName)) throw new Error(message);
|
|
136
138
|
});
|
|
139
|
+
// Used outside the tests (a hook or a helper): every test waits, so the
|
|
140
|
+
// file's own hooks must not run either, or a beforeAll/afterEach would
|
|
141
|
+
// fail on the placeholder and hide the "waiting" reason.
|
|
142
|
+
if (!users) quietHooks();
|
|
137
143
|
return true;
|
|
138
144
|
}
|
|
139
145
|
require('@playwright/test').test.beforeEach(async ({}, testInfo) => { // Playwright
|
|
@@ -145,14 +151,45 @@ function blockTestsThatUse(message, name) {
|
|
|
145
151
|
}
|
|
146
152
|
}
|
|
147
153
|
|
|
154
|
+
/**
|
|
155
|
+
* Jest: the hooks this test file registers from here on do nothing. Each test
|
|
156
|
+
* file has its own globals, so no other file is touched.
|
|
157
|
+
*/
|
|
158
|
+
function quietHooks() {
|
|
159
|
+
for (const h of ['beforeAll', 'afterAll', 'beforeEach', 'afterEach']) {
|
|
160
|
+
if (typeof globalThis[h] === 'function') globalThis[h] = () => {};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Why the last name scan left `name` blank, if it recorded one. */
|
|
165
|
+
function scanReason(name) {
|
|
166
|
+
try {
|
|
167
|
+
const p = path.join(path.dirname(interfacePath()), 'interface-status.json');
|
|
168
|
+
return JSON.parse(fs.readFileSync(p, 'utf-8'))[name];
|
|
169
|
+
} catch {
|
|
170
|
+
return undefined;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
148
174
|
/** The product's value for `name`, or an error naming what to fill. */
|
|
149
175
|
function need(name) {
|
|
150
176
|
const v = values()[name];
|
|
151
177
|
if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
|
|
152
178
|
// The prefix lets the reporters file this as "waiting for the product", not as a defect.
|
|
153
|
-
const
|
|
179
|
+
const why = scanReason(name);
|
|
180
|
+
const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json)${why ? ` — ${why}` : ''}: this test runs for real once it is there.`;
|
|
154
181
|
if (blockTestsThatUse(message, name)) return `<waiting: ${name}>`;
|
|
155
182
|
throw new Error(message);
|
|
156
183
|
}
|
|
157
184
|
|
|
158
|
-
|
|
185
|
+
/**
|
|
186
|
+
* A path from need() with its parameters filled in order, whatever the
|
|
187
|
+
* product calls them: /bookings/{booking}/cancel, /orders/:id or {id}.
|
|
188
|
+
* fillPath(need('booking cancel URL'), bookingId)
|
|
189
|
+
*/
|
|
190
|
+
function fillPath(template, ...values) {
|
|
191
|
+
let i = 0;
|
|
192
|
+
return String(template).replace(/\{[^}/]+\}|(?<=\/):[A-Za-z_]\w*/g, (m) => (i < values.length ? encodeURIComponent(String(values[i++])) : m));
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
module.exports = { need, fillPath, _usersOf: usersOf };
|