@activate-core/test-forge 2.4.6 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -11
- package/dist/cli/index.js +12 -12
- package/dist/core/docs/cache.js +1 -1
- package/dist/core/docs/discover.js +5 -5
- package/dist/core/docs/excerpts.js +139 -5
- package/dist/core/docs/grounding.js +78 -16
- package/dist/core/docs/reference-text.js +310 -52
- package/dist/core/gates/coverage.js +9 -0
- package/dist/core/gates/validate.js +1 -0
- package/dist/core/generate/ai-generator.js +9 -3
- package/dist/core/generate/ai-usage.js +11 -4
- package/dist/core/generate/bind.js +3 -2
- package/dist/core/generate/complete-file.js +45 -7
- package/dist/core/generate/defect-review.js +2 -1
- package/dist/core/generate/harness.js +34 -0
- package/dist/core/generate/interface-file.js +12 -0
- package/dist/core/generate/journey-memory.js +7 -1
- package/dist/core/generate/passed.js +24 -0
- package/dist/core/generate/review-journey.js +2 -2
- package/dist/core/generate/run-generate.js +117 -13
- package/dist/core/generate/suite-planner.js +2 -2
- package/dist/core/generate/test-blocks.js +34 -10
- package/dist/core/paths.js +41 -6
- package/dist/core/project/config.js +5 -0
- package/dist/core/project/setup.js +9 -9
- package/dist/core/project/update.js +6 -6
- package/dist/core/prompts/blocks.js +24 -3
- package/dist/core/run/name-scan.js +120 -14
- package/dist/core/run/report.js +7 -4
- package/dist/core/run/run-tests.js +53 -4
- package/package.json +3 -2
- package/reporters/_shared.cjs +56 -3
- package/reporters/interface.cjs +14 -0
- package/reporters/jest-reporter.cjs +7 -3
- package/reporters/playwright-reporter.cjs +6 -2
- package/reporters/run-data.cjs +6 -1
|
@@ -110,7 +110,10 @@ FORMAT
|
|
|
110
110
|
"## Test setup", which lists what the environment must provide (accounts and
|
|
111
111
|
records to seed, flags to turn on, external services to stub with their
|
|
112
112
|
endpoint or SDK, browser dialogs), written as instructions to the test
|
|
113
|
-
generator
|
|
113
|
+
generator. Name every request the tests send by its method and path
|
|
114
|
+
(POST /api/v2/booking-order), even a booking or payment the journey only
|
|
115
|
+
passes through: only a route named here has its handler and validation
|
|
116
|
+
rules read for the tests;
|
|
114
117
|
"## Questions for product (not asserted)", which holds only what neither the
|
|
115
118
|
reference nor the code settles. Keep it short.
|
|
116
119
|
- Do not write a Coverage table. It is generated from the [C<ids>] on each
|
|
@@ -174,8 +177,15 @@ E2. What you touch comes from <code_facts> and <testkit>.
|
|
|
174
177
|
any version prefix, and use the same prefix throughout the file.
|
|
175
178
|
- A response field comes from the serializer for that endpoint in
|
|
176
179
|
<code_facts>, not from the types of another endpoint.
|
|
177
|
-
- A
|
|
178
|
-
|
|
180
|
+
- A request is sent with the fields its handler validates, as SHAPES in
|
|
181
|
+
<code_facts> lists them, nested as written (address.latitude is
|
|
182
|
+
{ address: { latitude } }). A column, table or status value comes from DATA.
|
|
183
|
+
- Look in ROUTES, SHAPES and DATA before need(): a name any of them shows is
|
|
184
|
+
written as it is there. need() is only for a name none of them has (see
|
|
185
|
+
NAMES). Never guess one.
|
|
186
|
+
- Never need() what the testkit already gives: kit.insert returns the new
|
|
187
|
+
row's id; a second person of a role (another operator, another customer)
|
|
188
|
+
is a row made with kit.insert; the roles are the ones <testkit> lists.
|
|
179
189
|
- Call testkit helpers exactly as <testkit> lists them, and never write your
|
|
180
190
|
own copy.
|
|
181
191
|
|
|
@@ -306,6 +316,17 @@ export const API = `API (Jest)
|
|
|
306
316
|
token, the wrong role, and a different owner.
|
|
307
317
|
- Check stored state only through testkit resolvers or helpers. Never open
|
|
308
318
|
your own database connection.
|
|
319
|
+
- Build the data a scenario describes (operators and their locations,
|
|
320
|
+
services, hours and earlier jobs; a setting switched on or off) with
|
|
321
|
+
kit.insert(table, row) and kit.update(table, match, values), using the
|
|
322
|
+
table and column names from <code_facts>. Never ask for a URL or command to
|
|
323
|
+
seed, rank or switch something: data is written, and the outcome is read
|
|
324
|
+
from the product's own endpoints (the operator a booking gets, a status).
|
|
325
|
+
- A value the product works out from other records (a standing, an average
|
|
326
|
+
per day) is not a column: set it with the testkit's _tf_ column from its
|
|
327
|
+
DERIVED export. Where the product picks one record among many (the nearest,
|
|
328
|
+
the best), build the scenario at kit.AREA so nothing already in the
|
|
329
|
+
database takes part.
|
|
309
330
|
- Keep tests hermetic. Every external service listed under EXTERNALS in
|
|
310
331
|
<code_facts> is stood in for:
|
|
311
332
|
- when the server runs inside the test process, with jest.mock on the exact
|
|
@@ -16,8 +16,10 @@ import { dirname, join, relative } from 'node:path';
|
|
|
16
16
|
import { STATE_DIR } from '../paths.js';
|
|
17
17
|
import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
18
18
|
import { runAiSync } from '../generate/ai-usage.js';
|
|
19
|
-
import { readInterface, recordInterface } from '../generate/interface-file.js';
|
|
19
|
+
import { clearInterface, readInterface, recordInterface } from '../generate/interface-file.js';
|
|
20
20
|
const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
|
|
21
|
+
/** Values the scan filled (name → value), so each run can check the code still has them. */
|
|
22
|
+
const FILLED_PATH = join(STATE_DIR, 'interface-filled.json');
|
|
21
23
|
const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
|
|
22
24
|
const CODE = /\.(php|ts|tsx|js|jsx|mjs|cjs|py|rb|go|vue|svelte|prisma|sql)$/;
|
|
23
25
|
const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls', 'item', 'items', 'menu', 'link', 'entry', 'icon', 'text', 'title', 'page']);
|
|
@@ -26,11 +28,14 @@ export function kindOf(name) {
|
|
|
26
28
|
const n = name.trim().toLowerCase();
|
|
27
29
|
if (/\b(?:url|urls|path|prefix|endpoint|route)$/.test(n))
|
|
28
30
|
return 'route';
|
|
31
|
+
// The HTTP method of a request lives on the route line (Route::post, router.post, @Post).
|
|
32
|
+
if (/\bmethod$/.test(n))
|
|
33
|
+
return 'method';
|
|
29
34
|
if (/\bcommand$/.test(n))
|
|
30
35
|
return 'command';
|
|
31
|
-
if (/\b(?:table|column|model)$/.test(n))
|
|
36
|
+
if (/\b(?:table|column|model|class)$/.test(n))
|
|
32
37
|
return 'schema';
|
|
33
|
-
if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|setting)$/.test(n))
|
|
38
|
+
if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|value name|setting|role|permission|event|queue|job)$/.test(n))
|
|
34
39
|
return 'field';
|
|
35
40
|
if (/\blabel$/.test(n))
|
|
36
41
|
return 'label';
|
|
@@ -39,28 +44,32 @@ export function kindOf(name) {
|
|
|
39
44
|
/** Files of a kind: a route file, a console command, a migration or model, a resource or controller, a component. */
|
|
40
45
|
function fileMatches(kind, rel) {
|
|
41
46
|
switch (kind) {
|
|
42
|
-
case 'route':
|
|
47
|
+
case 'route':
|
|
48
|
+
case 'method': return /(^|\/)(routes?|router|urls)(\/|\.[a-z]+$)|\.routes?\.[a-z]+$|\/routes?\.[a-z]+$|(^|\/)app\/.*\/(page|route)\.[jt]sx?$|\.controller\.[jt]s$/i.test(rel);
|
|
43
49
|
case 'command': return /(^|\/)(console|commands?|cli|bin)(\/|$)|command/i.test(rel);
|
|
44
50
|
case 'schema': return /(^|\/)(migrations?|models?|entities|schema)(\/|\.)|\.prisma$|\.sql$/i.test(rel);
|
|
45
|
-
|
|
51
|
+
// A request's fields are where it is validated (FormRequests, rule and
|
|
52
|
+
// logic classes), a status value where its enum or model constant is.
|
|
53
|
+
case 'field': return /(^|\/)(resources?|transformers?|serializers?|controllers?|dto|types?|api|requests?|validators?|validation|rules?|logic|services?|models?|enums?|constants?|schemas?)(\/|\.)|resource|serializer|controller|request|validat|enum/i.test(rel);
|
|
46
54
|
case 'label': return /\.(tsx|jsx|vue|svelte)$/.test(rel);
|
|
47
55
|
}
|
|
48
56
|
}
|
|
49
57
|
/** What a line that DEFINES a name of each kind looks like. */
|
|
50
58
|
const SHAPE = {
|
|
51
59
|
// …or a URL the app keeps as a constant (faq: 'https://…').
|
|
60
|
+
method: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|export\s+(?:async\s+)?function\s+(?:GET|POST|PUT|PATCH|DELETE)\b/,
|
|
52
61
|
route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]|\b\w+\s*:\s*['"`]https?:\/\//,
|
|
53
62
|
command: /\$signature\s*=|\$name\s*=\s*['"]|->command\(\s*['"]|\.command\(\s*['"]|add_parser\(|@click\.command/,
|
|
54
63
|
schema: /Schema::(?:create|table)\(|\$table->\w+\(\s*['"]|create_table|\bclass\s+\w+\s+extends\s+\w*Model\b|protected\s+\$(?:table|fillable)|\bmodel\s+\w+\s*\{|@(?:Entity|Column|Table)\b/,
|
|
55
|
-
field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]
|
|
64
|
+
field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]|^\s*case\s+\w+\s*=|\bconst\s+[A-Z_]+\s*=/,
|
|
56
65
|
// …or a menu entry's label (label: 'Orders').
|
|
57
66
|
// …or a JSX text child on a line of its own (<Button>\n Escalate\n</Button>).
|
|
58
67
|
label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]|\blabel\s*:\s*['"`]|^\s*[A-Z][\w'’ ]{1,40}\s*$|\/>\s*[A-Z][\w'’ ]{1,40}\s*$/,
|
|
59
68
|
};
|
|
60
69
|
/** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
|
|
61
70
|
function candidateLines(name, kind, files, root) {
|
|
62
|
-
const words = [...new Set(name.toLowerCase().match(/[a-z]{
|
|
63
|
-
.filter((w) => !GENERIC.has(w))
|
|
71
|
+
const words = [...new Set(name.toLowerCase().match(/[a-z]{2,}/g) ?? [])]
|
|
72
|
+
.filter((w) => !GENERIC.has(w) && (w.length >= 3 || w === 'id'))
|
|
64
73
|
.map((w) => (w.length > 5 ? w.replace(/(?:ings?|ed|es|s)$/, '') : w));
|
|
65
74
|
if (!words.length)
|
|
66
75
|
return [];
|
|
@@ -83,19 +92,46 @@ function candidateLines(name, kind, files, root) {
|
|
|
83
92
|
lines.forEach((l, i) => {
|
|
84
93
|
if (/^\s*(?:\/\/|#|\*)/.test(l) || l.length > 300)
|
|
85
94
|
return;
|
|
95
|
+
// Log and console lines name things in prose, not as the product's keys.
|
|
96
|
+
if (/\b(?:Log::|logger\.|console\.|->(?:info|error|warning|debug)\()/.test(l))
|
|
97
|
+
return;
|
|
86
98
|
const low = l.toLowerCase();
|
|
87
|
-
|
|
99
|
+
// Whole identifier parts, not substrings: "order" is not in "orderBy",
|
|
100
|
+
// "out" is not in "Route"; address_id and addressId both give "address" and "id".
|
|
101
|
+
const parts = new Set(l.replace(/([a-z0-9])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
|
|
102
|
+
const has = (w) => parts.has(w) || [...parts].some((p) => p.length > 3 && (p.startsWith(w) || w.startsWith(p)) && Math.abs(p.length - w.length) <= 3);
|
|
103
|
+
if ((kind === 'route' || kind === 'method') && /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>/.test(l))
|
|
88
104
|
prefixes.push(`${rel}:${i + 1}: ${l.trim()}`);
|
|
89
105
|
if (!SHAPE[kind].test(l))
|
|
90
106
|
return;
|
|
91
|
-
const score = words.filter((w) =>
|
|
107
|
+
const score = words.filter((w) => has(w)).length;
|
|
92
108
|
if (score >= need)
|
|
93
|
-
hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score });
|
|
109
|
+
hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score, file: rel, at: i, lines });
|
|
94
110
|
});
|
|
95
111
|
}
|
|
96
112
|
if (!hits.length)
|
|
97
113
|
return [];
|
|
98
|
-
const
|
|
114
|
+
const top = hits.sort((a, b) => b.score - a.score).slice(0, 8);
|
|
115
|
+
const best = top.map((h) => h.line);
|
|
116
|
+
if (kind === 'route' || kind === 'method') {
|
|
117
|
+
// The prefixes of the groups each route sits in (by indentation), not
|
|
118
|
+
// whichever prefixes come first in its file.
|
|
119
|
+
const PREFIX = /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>|Route::group\(/;
|
|
120
|
+
const indent = (l) => l.length - l.trimStart().length;
|
|
121
|
+
const enclosing = [];
|
|
122
|
+
for (const h of top) {
|
|
123
|
+
let cur = indent(h.lines[h.at]);
|
|
124
|
+
for (let j = h.at - 1; j >= 0 && cur > 0; j--) {
|
|
125
|
+
const l = h.lines[j];
|
|
126
|
+
if (!l.trim() || indent(l) >= cur)
|
|
127
|
+
continue;
|
|
128
|
+
cur = indent(l);
|
|
129
|
+
if (PREFIX.test(l))
|
|
130
|
+
enclosing.push(`${h.file}:${j + 1}: ${l.trim()}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return [...best, ...new Set(enclosing)].slice(0, 16);
|
|
134
|
+
}
|
|
99
135
|
// A route's full path needs the prefix of the group it sits in: those of its own files ride along.
|
|
100
136
|
const files_ = new Set(best.map((l) => l.split(':')[0]));
|
|
101
137
|
return [...best, ...prefixes.filter((p) => files_.has(p.split(':')[0])).slice(0, 6)];
|
|
@@ -140,6 +176,17 @@ export function foundInCode(value, lines) {
|
|
|
140
176
|
const parts = v.split('/').filter((p) => p && p !== 'api' && !/^[{:<]/.test(p));
|
|
141
177
|
return parts.length > 0 && parts.every((p) => text.includes(p));
|
|
142
178
|
}
|
|
179
|
+
// An HTTP method: Route::post / router.post / @Post / export function POST.
|
|
180
|
+
if (/^(?:get|post|put|patch|delete)$/i.test(v))
|
|
181
|
+
return new RegExp(`\\b${v}\\b`, 'i').test(text);
|
|
182
|
+
// A nested response field (error.message): each part is a key in the code.
|
|
183
|
+
if (/^[\w-]+(?:\.[\w-]+)+$/.test(v) && !/^\d/.test(v))
|
|
184
|
+
return v.split('.').every((p) => text.includes(p));
|
|
185
|
+
// A short or numeric value ("80", "on") must stand as a value, not inside another word.
|
|
186
|
+
if (v.length < 4 || /^\d+(?:\.\d+)?$/.test(v)) {
|
|
187
|
+
const esc = v.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
188
|
+
return new RegExp(`(['"\`])${esc}\\1|(?:=>|=|:)\\s*${esc}\\b`).test(text);
|
|
189
|
+
}
|
|
143
190
|
return text.includes(v);
|
|
144
191
|
}
|
|
145
192
|
/**
|
|
@@ -202,7 +249,9 @@ export function scanNames(blank, root = '.') {
|
|
|
202
249
|
'- A URL is the path a client calls: it starts with "/" and includes every',
|
|
203
250
|
' route prefix the lines show (a route inside a prefix group gets the prefix).',
|
|
204
251
|
'- A command is the command name a person types.',
|
|
205
|
-
'- A field, key or column is the exact key.',
|
|
252
|
+
'- A field, key or column is the exact key. A field inside an object is its path with dots (error.message).',
|
|
253
|
+
'- A request method is the HTTP verb in capitals (POST).',
|
|
254
|
+
'- A status or enum value is the stored value, not the constant\'s name.',
|
|
206
255
|
'If no line clearly is that thing, the value is null.',
|
|
207
256
|
...(known.length ? ['', '<filled_examples>', ...known.map(([k, v]) => `- ${k}: ${v}`), '</filled_examples>', 'These values are already filled; follow their format.'] : []),
|
|
208
257
|
'',
|
|
@@ -227,8 +276,22 @@ export function scanNames(blank, root = '.') {
|
|
|
227
276
|
keep[a.name] = v.trim();
|
|
228
277
|
}
|
|
229
278
|
recordInterface(keep);
|
|
230
|
-
for (const [name, value] of Object.entries(keep))
|
|
279
|
+
for (const [name, value] of Object.entries(keep)) {
|
|
231
280
|
filled.push({ name, value });
|
|
281
|
+
// Asked again if someone blanks it later, even with the same lines.
|
|
282
|
+
memo[name] = `filled:${memo[name]}`;
|
|
283
|
+
}
|
|
284
|
+
try {
|
|
285
|
+
const prev = JSON.parse(readFileSync(FILLED_PATH, 'utf-8'));
|
|
286
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify({ ...prev, ...keep }, null, 2)}\n`, 'utf-8');
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
try {
|
|
290
|
+
mkdirSync(dirname(FILLED_PATH), { recursive: true });
|
|
291
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify(keep, null, 2)}\n`, 'utf-8');
|
|
292
|
+
}
|
|
293
|
+
catch { /* best effort */ }
|
|
294
|
+
}
|
|
232
295
|
}
|
|
233
296
|
catch (e) {
|
|
234
297
|
console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
|
|
@@ -246,3 +309,46 @@ export function scanNames(blank, root = '.') {
|
|
|
246
309
|
const done = new Set(filled.map((f) => f.name));
|
|
247
310
|
return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
|
|
248
311
|
}
|
|
312
|
+
/**
|
|
313
|
+
* Values the scan filled earlier that the code no longer has (a route or field
|
|
314
|
+
* renamed): blanked again, so their tests report "waiting" rather than failing
|
|
315
|
+
* as if the product were wrong, and the next scan looks for the new name. No AI.
|
|
316
|
+
*/
|
|
317
|
+
export function recheckFilled(root = '.') {
|
|
318
|
+
let filled;
|
|
319
|
+
try {
|
|
320
|
+
filled = JSON.parse(readFileSync(FILLED_PATH, 'utf-8'));
|
|
321
|
+
}
|
|
322
|
+
catch {
|
|
323
|
+
return [];
|
|
324
|
+
}
|
|
325
|
+
const current = readInterface();
|
|
326
|
+
const files = codeFiles(root);
|
|
327
|
+
const gone = [];
|
|
328
|
+
for (const [name, value] of Object.entries(filled)) {
|
|
329
|
+
if (String(current[name] ?? '').trim() !== value)
|
|
330
|
+
continue; // changed or set by hand: not ours to judge
|
|
331
|
+
const kind = kindOf(name);
|
|
332
|
+
if (!kind)
|
|
333
|
+
continue;
|
|
334
|
+
const lines = files.filter((f) => fileMatches(kind, relative(root, f))).flatMap((f) => {
|
|
335
|
+
try {
|
|
336
|
+
return readFileSync(f, 'utf-8').split('\n');
|
|
337
|
+
}
|
|
338
|
+
catch {
|
|
339
|
+
return [];
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
if (!foundInCode(value, lines))
|
|
343
|
+
gone.push(name);
|
|
344
|
+
}
|
|
345
|
+
if (gone.length) {
|
|
346
|
+
clearInterface(gone);
|
|
347
|
+
const left = Object.fromEntries(Object.entries(filled).filter(([n]) => !gone.includes(n)));
|
|
348
|
+
try {
|
|
349
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify(left, null, 2)}\n`, 'utf-8');
|
|
350
|
+
}
|
|
351
|
+
catch { /* best effort */ }
|
|
352
|
+
}
|
|
353
|
+
return gone;
|
|
354
|
+
}
|
package/dist/core/run/report.js
CHANGED
|
@@ -75,8 +75,9 @@ export function mapPlaywright(json, category = 'web') {
|
|
|
75
75
|
for (const t of spec.tests || []) {
|
|
76
76
|
const r = (t.results || [])[(t.results || []).length - 1] || {};
|
|
77
77
|
const st = r.status || t.status;
|
|
78
|
-
const status = st === 'passed' ? 'passed' : st === 'skipped' ? 'skipped' : 'failed';
|
|
79
78
|
const raw = r.error && r.error.message;
|
|
79
|
+
const notProduct = st !== 'passed' && st !== 'skipped' ? shared.notTheProduct(raw) : undefined;
|
|
80
|
+
const status = st === 'passed' ? 'passed' : st === 'skipped' || notProduct ? 'skipped' : 'failed';
|
|
80
81
|
const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
|
|
81
82
|
const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
|
|
82
83
|
const err = r.error || {};
|
|
@@ -95,8 +96,8 @@ export function mapPlaywright(json, category = 'web') {
|
|
|
95
96
|
...(detail ? { detail } : {}),
|
|
96
97
|
error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
|
|
97
98
|
sig: status === 'failed' ? shared.sig(raw) : undefined,
|
|
98
|
-
note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
|
|
99
|
-
...(status === 'failed' ? verdictOf(loc, spec.title) : {}),
|
|
99
|
+
note: status === 'skipped' ? notProduct || (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
|
|
100
|
+
...(status === 'failed' ? verdictOf(loc, spec.title, raw) : {}),
|
|
100
101
|
});
|
|
101
102
|
}
|
|
102
103
|
}
|
|
@@ -114,14 +115,16 @@ export function mapJest(json, category = 'api', testDir) {
|
|
|
114
115
|
const name = shared.suiteName(f.name || f.testFilePath || 'suite', testDir);
|
|
115
116
|
const su = (suites[name] ||= { name, category, tests: [] });
|
|
116
117
|
for (const a of f.assertionResults || []) {
|
|
117
|
-
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
|
|
118
118
|
const raw = a.failureMessages && a.failureMessages[0];
|
|
119
|
+
const notProduct = a.status === 'failed' ? shared.notTheProduct(raw) : undefined;
|
|
120
|
+
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' && !notProduct ? 'failed' : 'skipped';
|
|
119
121
|
const err = strip(raw || '').split('\n').find((l) => l.trim()) || '';
|
|
120
122
|
su.tests.push({
|
|
121
123
|
name: a.title,
|
|
122
124
|
status,
|
|
123
125
|
error: status === 'failed' ? err.slice(0, 200) : undefined,
|
|
124
126
|
sig: status === 'failed' ? shared.sig(raw) : undefined,
|
|
127
|
+
...(notProduct ? { note: notProduct } : {}),
|
|
125
128
|
});
|
|
126
129
|
}
|
|
127
130
|
}
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
* It runs from the area's nearest package root (where package.json /
|
|
13
13
|
* playwright.config live), with config.envFile merged into the env.
|
|
14
14
|
*/
|
|
15
|
-
import { testFilesIn } from './readiness.js';
|
|
16
|
-
import { scanNames } from './name-scan.js';
|
|
15
|
+
import { PENDING_STAMP, testFilesIn } from './readiness.js';
|
|
16
|
+
import { recheckFilled, scanNames } from './name-scan.js';
|
|
17
17
|
import { INTERFACE_PATH, blankNames, neededNames } from '../generate/interface-file.js';
|
|
18
18
|
import { harnessInfo } from '../generate/harness.js';
|
|
19
19
|
import { missingPackages } from '../generate/verify.js';
|
|
@@ -111,6 +111,11 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
|
|
|
111
111
|
export function runTests(opts = {}) {
|
|
112
112
|
const config = loadConfig();
|
|
113
113
|
const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
|
|
114
|
+
// An env file named on the command line that is not there is a mistake, not a default.
|
|
115
|
+
if (opts.envFile && !existsSync(opts.envFile)) {
|
|
116
|
+
console.log(`The env file "${opts.envFile}" does not exist — nothing was run.`);
|
|
117
|
+
return 1;
|
|
118
|
+
}
|
|
114
119
|
const envVars = loadEnvFile(envFile);
|
|
115
120
|
if (Object.keys(envVars).length > 0) {
|
|
116
121
|
console.log(`Loaded env from ${envFile} (${Object.keys(envVars).length} vars).`);
|
|
@@ -156,8 +161,31 @@ export function runTests(opts = {}) {
|
|
|
156
161
|
// says so. Before the run, the names those tests wait for are looked for
|
|
157
162
|
// in the code — only where that kind of name lives, only when those lines
|
|
158
163
|
// changed — and filled; the tests themselves are never touched.
|
|
159
|
-
|
|
164
|
+
let targets = undefined;
|
|
165
|
+
// A feature the doc says is not live yet: its files are held back, not run
|
|
166
|
+
// to fail. They run once the doc's Status line is removed and they are generated again.
|
|
167
|
+
{
|
|
168
|
+
const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
|
|
169
|
+
const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
|
|
170
|
+
const held = files.filter((f) => { try {
|
|
171
|
+
return PENDING_STAMP.test(readFileSync(f, 'utf-8'));
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
return false;
|
|
175
|
+
} });
|
|
176
|
+
if (held.length) {
|
|
177
|
+
const run = files.filter((f) => !held.includes(f));
|
|
178
|
+
console.log(` held back (feature not live yet): ${held.length} file(s)`);
|
|
179
|
+
if (!run.length) {
|
|
180
|
+
console.log(`[skip] ${area}: every test file waits for a feature that is not live yet`);
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
targets = run.map((f) => resolve(f));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
160
186
|
if (!opts.noScan) {
|
|
187
|
+
for (const n of recheckFilled(root))
|
|
188
|
+
console.log(` no longer in the code: ${n} — blanked; its tests wait until the new name is found`);
|
|
161
189
|
const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
|
|
162
190
|
const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
|
|
163
191
|
const blank = blankNames([...new Set(files.flatMap((f) => neededNames(readFileSync(f, 'utf-8'))))]);
|
|
@@ -176,8 +204,16 @@ export function runTests(opts = {}) {
|
|
|
176
204
|
ranAnything = true;
|
|
177
205
|
console.log(`\n=== ${area} (${framework}) ===`);
|
|
178
206
|
console.log(`$ ${cmd} [cwd: ${root}]`);
|
|
207
|
+
const started = Date.now();
|
|
179
208
|
const res = spawnSync(cmd, { shell: true, cwd: root, env, stdio: 'inherit' });
|
|
180
|
-
|
|
209
|
+
// The reporters' counts: a run whose only "failures" are tests waiting for
|
|
210
|
+
// the product or stopped by the environment has no product failure.
|
|
211
|
+
const sum = runSummary(root, started);
|
|
212
|
+
if (sum) {
|
|
213
|
+
const ran = sum.passed + sum.failed;
|
|
214
|
+
console.log(`\n${area}: ${ran} of ${sum.total} test(s) ran — ${sum.passed} passed, ${sum.failed} failed; ${sum.skipped} did not run (waiting for the product, environment, or skipped)`);
|
|
215
|
+
}
|
|
216
|
+
if (res.status !== 0 && !(sum && sum.failed === 0 && sum.total > 0))
|
|
181
217
|
anyFail = true;
|
|
182
218
|
}
|
|
183
219
|
if (!ranAnything) {
|
|
@@ -190,3 +226,16 @@ export function runTests(opts = {}) {
|
|
|
190
226
|
console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
|
|
191
227
|
return anyFail ? 1 : 0;
|
|
192
228
|
}
|
|
229
|
+
/** Counts the reporters wrote for runs that started at or after `since`, summed. */
|
|
230
|
+
function runSummary(root, since) {
|
|
231
|
+
try {
|
|
232
|
+
const all = JSON.parse(readFileSync(join(root, 'testforge', 'last-run.json'), 'utf-8'));
|
|
233
|
+
const fresh = Object.values(all).filter((r) => Date.parse(r.at) >= since - 1000);
|
|
234
|
+
if (!fresh.length)
|
|
235
|
+
return undefined;
|
|
236
|
+
return fresh.reduce((a, r) => ({ total: a.total + r.total, passed: a.passed + r.passed, failed: a.failed + r.failed, skipped: a.skipped + r.skipped }), { total: 0, passed: 0, failed: 0, skipped: 0 });
|
|
237
|
+
}
|
|
238
|
+
catch {
|
|
239
|
+
return undefined;
|
|
240
|
+
}
|
|
241
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.6.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
"scripts": {
|
|
16
16
|
"build": "rm -rf dist && tsc",
|
|
17
17
|
"dev": "tsc --watch",
|
|
18
|
-
"test": "tsx --test test/*.test.ts"
|
|
18
|
+
"test": "tsx --test test/*.test.ts",
|
|
19
|
+
"prepublishOnly": "npm run build && npm test"
|
|
19
20
|
},
|
|
20
21
|
"license": "MIT",
|
|
21
22
|
"publishConfig": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -31,6 +31,7 @@ function loadEnv(root) {
|
|
|
31
31
|
const k = s.slice(0, i).trim();
|
|
32
32
|
let v = s.slice(i + 1).trim();
|
|
33
33
|
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
|
|
34
|
+
else v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
|
|
34
35
|
if (k && process.env[k] === undefined) process.env[k] = v;
|
|
35
36
|
}
|
|
36
37
|
}
|
|
@@ -395,12 +396,19 @@ function reviewVerdicts(root) {
|
|
|
395
396
|
try {
|
|
396
397
|
for (const r of JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) || []) {
|
|
397
398
|
for (const t of r.tests || []) {
|
|
398
|
-
map.set(`${path.resolve(String(t.file))}|${t.title}`, { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) });
|
|
399
|
+
map.set(`${path.resolve(String(t.file))}|${t.title}`, { at: sig(t.detail), v: { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) } });
|
|
399
400
|
}
|
|
400
401
|
}
|
|
401
402
|
} catch { /* a half-written review is ignored */ }
|
|
402
403
|
}
|
|
403
|
-
|
|
404
|
+
// A verdict is about one failure: the same test failing differently later is a
|
|
405
|
+
// new failure, and an old "test fault" must not hide a real regression.
|
|
406
|
+
return (file, title, raw) => {
|
|
407
|
+
const hit = file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined;
|
|
408
|
+
if (!hit) return undefined;
|
|
409
|
+
if (raw !== undefined && hit.at && sig(raw) !== hit.at) return undefined;
|
|
410
|
+
return hit.v;
|
|
411
|
+
};
|
|
404
412
|
}
|
|
405
413
|
|
|
406
414
|
/** The name a test is still waiting for (need() threw for a blank interface name), else undefined. */
|
|
@@ -431,4 +439,49 @@ function notTheProduct(raw) {
|
|
|
431
439
|
return environmentFor(raw);
|
|
432
440
|
}
|
|
433
441
|
|
|
434
|
-
|
|
442
|
+
/**
|
|
443
|
+
* Remember the tests that passed (testforge/passed.json): generate never removes
|
|
444
|
+
* or rewrites a test that has passed. Add-only — a later failure is a regression
|
|
445
|
+
* to report, not a reason to forget that the test once checked the product.
|
|
446
|
+
*/
|
|
447
|
+
function recordPassed(root, passed) {
|
|
448
|
+
if (!passed.length) return;
|
|
449
|
+
const file = path.join(root, 'testforge', 'passed.json');
|
|
450
|
+
let all = {};
|
|
451
|
+
try { all = JSON.parse(fs.readFileSync(file, 'utf-8')) || {}; } catch { all = {}; }
|
|
452
|
+
let changed = false;
|
|
453
|
+
for (const t of passed) {
|
|
454
|
+
if (!t.file || !t.title) continue;
|
|
455
|
+
const rel = path.relative(root, path.resolve(t.file)).split(path.sep).join('/');
|
|
456
|
+
const list = Array.isArray(all[rel]) ? all[rel] : (all[rel] = []);
|
|
457
|
+
if (!list.includes(t.title)) { list.push(t.title); changed = true; }
|
|
458
|
+
}
|
|
459
|
+
if (!changed) return;
|
|
460
|
+
try {
|
|
461
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
462
|
+
fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n');
|
|
463
|
+
} catch (e) {
|
|
464
|
+
console.log(`[testforge] could not record passed tests: ${e.message}`);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* The run's counts, for `testforge run` to read: how many ran of how many, and
|
|
470
|
+
* whether anything failed for a product reason. Waiting and environment stops
|
|
471
|
+
* are "skipped" here, as on the dashboard — never a failure of the run.
|
|
472
|
+
*/
|
|
473
|
+
function recordSummary(root, category, suites) {
|
|
474
|
+
const tests = suites.flatMap((s) => s.tests || []);
|
|
475
|
+
const n = (st) => tests.filter((t) => t.status === st).length;
|
|
476
|
+
const out = { category, at: new Date().toISOString(), total: tests.length, passed: n('passed'), failed: n('failed'), skipped: n('skipped') };
|
|
477
|
+
try {
|
|
478
|
+
const file = path.join(root, 'testforge', 'last-run.json');
|
|
479
|
+
let all = {};
|
|
480
|
+
try { all = JSON.parse(fs.readFileSync(file, 'utf-8')) || {}; } catch { all = {}; }
|
|
481
|
+
all[category] = out;
|
|
482
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
483
|
+
fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n');
|
|
484
|
+
} catch { /* the run's own output still has the counts */ }
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
module.exports = { recordSummary, recordPassed, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
|
package/reporters/interface.cjs
CHANGED
|
@@ -134,6 +134,10 @@ function blockTestsThatUse(message, name) {
|
|
|
134
134
|
try { fullName = globalThis.expect.getState().currentTestName; } catch { /* unknown */ }
|
|
135
135
|
if (needs(undefined, fullName)) throw new Error(message);
|
|
136
136
|
});
|
|
137
|
+
// Used outside the tests (a hook or a helper): every test waits, so the
|
|
138
|
+
// file's own hooks must not run either, or a beforeAll/afterEach would
|
|
139
|
+
// fail on the placeholder and hide the "waiting" reason.
|
|
140
|
+
if (!users) quietHooks();
|
|
137
141
|
return true;
|
|
138
142
|
}
|
|
139
143
|
require('@playwright/test').test.beforeEach(async ({}, testInfo) => { // Playwright
|
|
@@ -145,6 +149,16 @@ function blockTestsThatUse(message, name) {
|
|
|
145
149
|
}
|
|
146
150
|
}
|
|
147
151
|
|
|
152
|
+
/**
|
|
153
|
+
* Jest: the hooks this test file registers from here on do nothing. Each test
|
|
154
|
+
* file has its own globals, so no other file is touched.
|
|
155
|
+
*/
|
|
156
|
+
function quietHooks() {
|
|
157
|
+
for (const h of ['beforeAll', 'afterAll', 'beforeEach', 'afterEach']) {
|
|
158
|
+
if (typeof globalThis[h] === 'function') globalThis[h] = () => {};
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
148
162
|
/** The product's value for `name`, or an error naming what to fill. */
|
|
149
163
|
function need(name) {
|
|
150
164
|
const v = values()[name];
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to jest.config.js:
|
|
3
3
|
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle, recordPassed, recordSummary } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgeJestReporter {
|
|
7
7
|
constructor(_globalConfig, options) {
|
|
8
8
|
this.category = (options && options.category) || 'api';
|
|
9
9
|
this.testRoot = testRoot(findRoot(process.cwd()), this.category);
|
|
10
|
-
this.
|
|
10
|
+
this.root = findRoot(process.cwd());
|
|
11
|
+
this.verdictOf = reviewVerdicts(this.root);
|
|
11
12
|
}
|
|
12
13
|
async onRunComplete(_contexts, results) {
|
|
13
14
|
const suites = {};
|
|
@@ -31,12 +32,15 @@ module.exports = class TestForgeJestReporter {
|
|
|
31
32
|
checks: testChecks(f.testFilePath, a.location && a.location.line),
|
|
32
33
|
...(status === 'failed' ? expectation(raw) : {}),
|
|
33
34
|
detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
|
|
34
|
-
...(status === 'failed' ? this.verdictOf(f.testFilePath, a.fullName || a.title) : {}),
|
|
35
|
+
...(status === 'failed' ? this.verdictOf(f.testFilePath, a.fullName || a.title, raw) : {}),
|
|
35
36
|
});
|
|
36
37
|
}
|
|
37
38
|
}
|
|
39
|
+
recordPassed(this.root, (results.testResults || []).flatMap((f) =>
|
|
40
|
+
(f.testResults || f.assertionResults || []).filter((a) => a.status === 'passed').map((a) => ({ file: f.testFilePath, title: a.title }))));
|
|
38
41
|
const durationSec = Math.round((results.startTime ? Date.now() - results.startTime : 0) / 1000);
|
|
39
42
|
const list = Object.values(suites);
|
|
43
|
+
recordSummary(this.root, this.category, list);
|
|
40
44
|
// Same reasoning as the Playwright reporter: no suites means nothing ran,
|
|
41
45
|
// and a published 0/0/0 run reads as a clean result.
|
|
42
46
|
if (!list.length) {
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
// TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to playwright.config.js reporter array:
|
|
3
3
|
// ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle, recordPassed, recordSummary } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgePlaywrightReporter {
|
|
7
7
|
constructor(options) {
|
|
8
8
|
this.category = (options && options.category) || 'web';
|
|
9
9
|
this.suites = {};
|
|
10
|
+
this.passed = [];
|
|
10
11
|
this.start = Date.now();
|
|
11
12
|
this.root = findRoot(process.cwd());
|
|
12
13
|
this.testRoot = testRoot(this.root, this.category);
|
|
@@ -41,6 +42,7 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
41
42
|
const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' || notProduct ? 'skipped' : 'failed';
|
|
42
43
|
const pend = (result.annotations || []).concat(test.annotations || []).filter((x) => x.type === 'skip' || x.type === 'fixme');
|
|
43
44
|
const skipAnno = pend.find((x) => x.description) || pend[0];
|
|
45
|
+
if (status === 'passed') this.passed.push({ file, title: test.title });
|
|
44
46
|
su.tests.push({
|
|
45
47
|
name: plainTitle(test.title),
|
|
46
48
|
status,
|
|
@@ -50,7 +52,7 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
50
52
|
checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
|
|
51
53
|
|| testChecks(test.location && test.location.file, test.location && test.location.line),
|
|
52
54
|
...(status === 'failed' ? this.failed(test, result) : {}),
|
|
53
|
-
...(status === 'failed' ? this.verdictOf(test.location && test.location.file, test.title) : {}),
|
|
55
|
+
...(status === 'failed' ? this.verdictOf(test.location && test.location.file, test.title, raw) : {}),
|
|
54
56
|
});
|
|
55
57
|
}
|
|
56
58
|
failed(test, result) {
|
|
@@ -63,7 +65,9 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
63
65
|
}
|
|
64
66
|
async onEnd(result) {
|
|
65
67
|
const durationSec = Math.round(((result && result.duration) || (Date.now() - this.start)) / 1000);
|
|
68
|
+
recordPassed(this.root, this.passed);
|
|
66
69
|
const suites = Object.values(this.suites);
|
|
70
|
+
recordSummary(this.root, this.category, suites);
|
|
67
71
|
// A run that collected no suites did not test anything: global setup failed,
|
|
68
72
|
// or the filter matched nothing. Publishing it records a 0/0/0 result that
|
|
69
73
|
// reads as "the suite ran and found nothing wrong" — the most misleading
|
package/reporters/run-data.cjs
CHANGED
|
@@ -19,8 +19,13 @@ function runId() {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
/** A value no other run produces: label + the run id's last 6 characters (letters and digits only). */
|
|
22
|
+
const uses = new Map();
|
|
22
23
|
function unique(label = '') {
|
|
23
|
-
|
|
24
|
+
// The same label asked for again in one test file gets a new value: two
|
|
25
|
+
// rows built from one label (an email, a name) must not collide.
|
|
26
|
+
const n = (uses.get(label) || 0) + 1;
|
|
27
|
+
uses.set(label, n);
|
|
28
|
+
return `${label}${runId().slice(-6)}${n > 1 ? n : ''}`;
|
|
24
29
|
}
|
|
25
30
|
|
|
26
31
|
/**
|