@activate-core/test-forge 2.2.1 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -12
- package/dist/cli/index.js +1 -0
- package/dist/core/run/name-scan.js +235 -0
- package/dist/core/run/run-tests.js +17 -20
- package/package.json +1 -1
- package/reporters/_shared.cjs +7 -1
- package/reporters/interface.cjs +29 -4
- package/reporters/jest-reporter.cjs +5 -3
- package/reporters/playwright-reporter.cjs +5 -3
package/README.md
CHANGED
|
@@ -105,7 +105,7 @@ never touched.
|
|
|
105
105
|
| `testforge map` | Rebuild the project map (`testforge.md`) only. |
|
|
106
106
|
| `testforge discover` | Redraft the per-journey docs in `docs/` only. |
|
|
107
107
|
| `testforge env` | Rebuild `.env.testing.example` from the docs only. |
|
|
108
|
-
| `testforge run` | Run the generated tests (`--journey` / `--area` / `--env-file`).
|
|
108
|
+
| `testforge run` | Run the generated tests (`--journey` / `--area` / `--env-file`). First fills the names tests wait for from the code (only where each kind of name lives, only when those lines changed); `--no-scan` skips that. |
|
|
109
109
|
| `testforge bind` | After a feature ships: which test files the code is ready for, which wait for the feature, which use a name the code spells differently. `--accept "doc=code"` takes the code's wording. No AI. |
|
|
110
110
|
| `testforge review` | Ask an AI reviewer whether the last failures are defects, test mistakes or the environment. On demand only. |
|
|
111
111
|
| `testforge report` | Publish results to the dashboard — `--results` after a run, `--inventory` to register every generated check. |
|
|
@@ -119,7 +119,7 @@ never touched.
|
|
|
119
119
|
| `--force` | generate | Regenerate even when the doc has not changed. |
|
|
120
120
|
| `--plan-only` | generate | Plan and check each journey (`testforge/memory/<area>/<id>/plan-check.md`), write no tests. |
|
|
121
121
|
| `--refresh` | discover | Fetch the reference again instead of the copy kept from the last run. |
|
|
122
|
-
| `--
|
|
122
|
+
| `--no-scan` | run | Do not look for blank interface names in the code before the run. |
|
|
123
123
|
| `--refresh-docs` | update | Re-draft the docs even when nothing detectably changed — see below. |
|
|
124
124
|
| `--reference <file\|url>` | setup | Base the docs on a spec instead of the code map. |
|
|
125
125
|
| `--source map` | setup | Base the docs on `testforge.md`, no prompt. |
|
|
@@ -410,16 +410,26 @@ route, a button's label, a command, a table or field. When the code already
|
|
|
410
410
|
has one, the test uses it. When it does not (yet), the test asks for it by a
|
|
411
411
|
plain name — `need('order summary URL')` from
|
|
412
412
|
`@activate-core/test-forge/reporters/interface.cjs` — and the name is listed
|
|
413
|
-
in `testforge/interface.json` with an empty value.
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
413
|
+
in `testforge/interface.json` with an empty value. TestForge never overwrites
|
|
414
|
+
a value somebody wrote.
|
|
415
|
+
|
|
416
|
+
**Every run runs everything.** Before it starts, `testforge run` looks for the
|
|
417
|
+
blank names in the code — a URL only in the route files, a command only in the
|
|
418
|
+
console commands, a table only in the migrations and models, a field only in
|
|
419
|
+
the resources and controllers, and only on lines that share two of the name's
|
|
420
|
+
words. Nothing found, or the same lines as last time: no AI call. New lines:
|
|
421
|
+
one small call without tools picks each value, kept only if the code really
|
|
422
|
+
has it, and printed with what was found. Tests, docs and plans are never
|
|
423
|
+
touched. A test whose name is still blank is reported as "waiting for the
|
|
424
|
+
product", not as a defect. `--no-scan` skips the look-up.
|
|
425
|
+
|
|
426
|
+
Every run of `generate` ends with its AI cost per step (write, grounding, seed, harness …).
|
|
427
|
+
|
|
428
|
+
**A feature that is not live yet.** Write `Status: not built` or
|
|
429
|
+
`Status: behind flag <name>` in the journey doc's header if you like: no seed is
|
|
430
|
+
written for it (its tests make their own data), and its files carry a
|
|
431
|
+
`// testforge-pending: …` note. Its tests still run; until the feature is there
|
|
432
|
+
they fail or wait for its names, and the report says which.
|
|
423
433
|
|
|
424
434
|
`generate` runs nothing. `testforge run` runs the suite; `testforge review`
|
|
425
435
|
judges failures when you ask for it.
|
package/dist/cli/index.js
CHANGED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — fill the names tests wait for, from the code, at run time.
|
|
3
|
+
*
|
|
4
|
+
* A test written from the doc asks the product for a name it did not have
|
|
5
|
+
* yet: need('assignment mode switch URL'). Before a run, each blank name is
|
|
6
|
+
* looked for only where that kind of name lives — a URL in the route files, a
|
|
7
|
+
* command in the console commands, a table in the migrations, a field in the
|
|
8
|
+
* resources and controllers — and only on the lines that share a word with
|
|
9
|
+
* it. No line, no call. Lines unchanged since the last look, no call. When
|
|
10
|
+
* there are new lines, one small call without tools picks the value, and it
|
|
11
|
+
* is kept only if the code really contains it. Tests are never touched.
|
|
12
|
+
*/
|
|
13
|
+
import { createHash } from 'node:crypto';
|
|
14
|
+
import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { dirname, join, relative } from 'node:path';
|
|
16
|
+
import { STATE_DIR } from '../paths.js';
|
|
17
|
+
import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
18
|
+
import { runAiSync } from '../generate/ai-usage.js';
|
|
19
|
+
import { cleanOutput } from '../generate/ai-generator.js';
|
|
20
|
+
import { readInterface, recordInterface } from '../generate/interface-file.js';
|
|
21
|
+
const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
|
|
22
|
+
const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
|
|
23
|
+
const CODE = /\.(php|ts|tsx|js|jsx|mjs|cjs|py|rb|go|vue|svelte|prisma|sql)$/;
|
|
24
|
+
const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls']);
|
|
25
|
+
/** Where a name of this kind lives, judged from how the name ends. */
|
|
26
|
+
export function kindOf(name) {
|
|
27
|
+
const n = name.trim().toLowerCase();
|
|
28
|
+
if (/\b(?:url|urls|path|prefix|endpoint|route)$/.test(n))
|
|
29
|
+
return 'route';
|
|
30
|
+
if (/\bcommand$/.test(n))
|
|
31
|
+
return 'command';
|
|
32
|
+
if (/\b(?:table|column|model)$/.test(n))
|
|
33
|
+
return 'schema';
|
|
34
|
+
if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|setting)$/.test(n))
|
|
35
|
+
return 'field';
|
|
36
|
+
if (/\blabel$/.test(n))
|
|
37
|
+
return 'label';
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
40
|
+
/** Files of a kind: a route file, a console command, a migration or model, a resource or controller, a component. */
|
|
41
|
+
function fileMatches(kind, rel) {
|
|
42
|
+
switch (kind) {
|
|
43
|
+
case 'route': return /(^|\/)(routes?|router|urls)(\/|\.[a-z]+$)|\.routes?\.[a-z]+$|\/routes?\.[a-z]+$|(^|\/)app\/.*\/(page|route)\.[jt]sx?$/i.test(rel);
|
|
44
|
+
case 'command': return /(^|\/)(console|commands?|cli|bin)(\/|$)|command/i.test(rel);
|
|
45
|
+
case 'schema': return /(^|\/)(migrations?|models?|entities|schema)(\/|\.)|\.prisma$|\.sql$/i.test(rel);
|
|
46
|
+
case 'field': return /(^|\/)(resources?|transformers?|serializers?|controllers?|dto|types?|api)(\/|\.)|resource|serializer|controller/i.test(rel);
|
|
47
|
+
case 'label': return /\.(tsx|jsx|vue|svelte)$/.test(rel);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** What a line that DEFINES a name of each kind looks like. */
|
|
51
|
+
const SHAPE = {
|
|
52
|
+
route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]/,
|
|
53
|
+
command: /\$signature\s*=|\$name\s*=\s*['"]|->command\(\s*['"]|\.command\(\s*['"]|add_parser\(|@click\.command/,
|
|
54
|
+
schema: /Schema::(?:create|table)\(|\$table->\w+\(\s*['"]|create_table|\bclass\s+\w+\s+extends\s+\w*Model\b|protected\s+\$(?:table|fillable)|\bmodel\s+\w+\s*\{|@(?:Entity|Column|Table)\b/,
|
|
55
|
+
field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]/,
|
|
56
|
+
label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]/,
|
|
57
|
+
};
|
|
58
|
+
/** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
|
|
59
|
+
function candidateLines(name, kind, files, root) {
|
|
60
|
+
const words = [...new Set(name.toLowerCase().match(/[a-z]{3,}/g) ?? [])]
|
|
61
|
+
.filter((w) => !GENERIC.has(w))
|
|
62
|
+
.map((w) => (w.length > 5 ? w.replace(/(?:ings?|ed|es|s)$/, '') : w));
|
|
63
|
+
if (!words.length)
|
|
64
|
+
return [];
|
|
65
|
+
// One shared common word ("add", "error") points at nothing: a line must
|
|
66
|
+
// carry two of the name's words (or its only one).
|
|
67
|
+
const need = Math.min(2, words.length);
|
|
68
|
+
const hits = [];
|
|
69
|
+
const prefixes = [];
|
|
70
|
+
for (const f of files) {
|
|
71
|
+
const rel = relative(root, f);
|
|
72
|
+
if (!fileMatches(kind, rel))
|
|
73
|
+
continue;
|
|
74
|
+
let lines;
|
|
75
|
+
try {
|
|
76
|
+
lines = readFileSync(f, 'utf-8').split('\n');
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
lines.forEach((l, i) => {
|
|
82
|
+
if (/^\s*(?:\/\/|#|\*)/.test(l) || l.length > 300)
|
|
83
|
+
return;
|
|
84
|
+
const low = l.toLowerCase();
|
|
85
|
+
if (kind === 'route' && /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>/.test(l))
|
|
86
|
+
prefixes.push(`${rel}:${i + 1}: ${l.trim()}`);
|
|
87
|
+
if (!SHAPE[kind].test(l))
|
|
88
|
+
return;
|
|
89
|
+
const score = words.filter((w) => low.includes(w)).length;
|
|
90
|
+
if (score >= need)
|
|
91
|
+
hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score });
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
if (!hits.length)
|
|
95
|
+
return [];
|
|
96
|
+
const best = hits.sort((a, b) => b.score - a.score).slice(0, 8).map((h) => h.line);
|
|
97
|
+
// A route's full path needs the prefix of the group it sits in: those of its own files ride along.
|
|
98
|
+
const files_ = new Set(best.map((l) => l.split(':')[0]));
|
|
99
|
+
return [...best, ...prefixes.filter((p) => files_.has(p.split(':')[0])).slice(0, 6)];
|
|
100
|
+
}
|
|
101
|
+
function codeFiles(root) {
|
|
102
|
+
const out = [];
|
|
103
|
+
const walk = (d) => {
|
|
104
|
+
let entries;
|
|
105
|
+
try {
|
|
106
|
+
entries = readdirSync(d);
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
for (const e of entries) {
|
|
112
|
+
const p = join(d, e);
|
|
113
|
+
let st;
|
|
114
|
+
try {
|
|
115
|
+
st = statSync(p);
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (st.isDirectory()) {
|
|
121
|
+
if (!SKIP.test(e))
|
|
122
|
+
walk(p);
|
|
123
|
+
}
|
|
124
|
+
else if (CODE.test(e) && st.size < 400_000)
|
|
125
|
+
out.push(p);
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
walk(root);
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
131
|
+
/** A value is kept only when the code really has it: each part of a path, or the value itself. */
|
|
132
|
+
export function foundInCode(value, lines) {
|
|
133
|
+
const text = lines.join('\n');
|
|
134
|
+
const v = value.trim();
|
|
135
|
+
if (!v)
|
|
136
|
+
return false;
|
|
137
|
+
if (/^\/|^[\w-]+\//.test(v)) {
|
|
138
|
+
const parts = v.split('/').filter((p) => p && p !== 'api' && !/^[{:<]/.test(p));
|
|
139
|
+
return parts.length > 0 && parts.every((p) => text.includes(p));
|
|
140
|
+
}
|
|
141
|
+
return text.includes(v);
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* The blank names worth one lookup call, each with its candidate lines; the
|
|
145
|
+
* memo records what each name's lines looked like, so unchanged ones are not
|
|
146
|
+
* asked again. No AI.
|
|
147
|
+
*/
|
|
148
|
+
export function namesToAsk(blank, root, memo) {
|
|
149
|
+
const files = blank.length ? codeFiles(root) : [];
|
|
150
|
+
const ask = [];
|
|
151
|
+
for (const name of blank) {
|
|
152
|
+
const kind = kindOf(name);
|
|
153
|
+
const lines = kind ? candidateLines(name, kind, files, root) : [];
|
|
154
|
+
const print = createHash('sha1').update(lines.join('\n')).digest('hex').slice(0, 16);
|
|
155
|
+
// Nothing that could be it, or nothing new since the last look: no call.
|
|
156
|
+
if (!lines.length || memo[name] === print) {
|
|
157
|
+
memo[name] = print;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
memo[name] = print;
|
|
161
|
+
ask.push({ name, lines });
|
|
162
|
+
}
|
|
163
|
+
return ask;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Look for the blank names in the code and fill what is found. `blank` are
|
|
167
|
+
* the names the tests to run still wait for. Returns what was filled.
|
|
168
|
+
*/
|
|
169
|
+
export function scanNames(blank, root = '.') {
|
|
170
|
+
const memo = (() => {
|
|
171
|
+
try {
|
|
172
|
+
return JSON.parse(readFileSync(SCAN_PATH, 'utf-8'));
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return {};
|
|
176
|
+
}
|
|
177
|
+
})();
|
|
178
|
+
const all = namesToAsk(blank, root, memo);
|
|
179
|
+
// One call stays small: names past the budget are asked on the next run.
|
|
180
|
+
const ask = [];
|
|
181
|
+
let size = 0;
|
|
182
|
+
for (const a of all) {
|
|
183
|
+
const n = a.name.length + a.lines.join('\n').length;
|
|
184
|
+
if (ask.length && size + n > 60_000) {
|
|
185
|
+
delete memo[a.name];
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
ask.push(a);
|
|
189
|
+
size += n;
|
|
190
|
+
}
|
|
191
|
+
const filled = [];
|
|
192
|
+
let calls = 0;
|
|
193
|
+
if (ask.length) {
|
|
194
|
+
const known = Object.entries(readInterface()).filter(([, v]) => String(v).trim()).slice(0, 8);
|
|
195
|
+
const prompt = [
|
|
196
|
+
'You match names a test needs to values in code. For each NAME below, pick its value from ITS lines only, or null.',
|
|
197
|
+
'Answer with ONE JSON object {"<name>": "<value>" | null} and nothing else.',
|
|
198
|
+
'A value is copied from the lines, never invented. A URL is the path a client calls, with "/" first and every route prefix the lines show',
|
|
199
|
+
'(a route inside a prefix group gets the prefix). A command is the command name a person types. A field, key or column is the exact key.',
|
|
200
|
+
'If no line clearly is that thing, the value is null.',
|
|
201
|
+
...(known.length ? ['', 'Values already filled, for their format:', ...known.map(([k, v]) => `- ${k}: ${v}`)] : []),
|
|
202
|
+
'',
|
|
203
|
+
...ask.flatMap((a) => [`NAME: ${a.name}`, ...a.lines.map((l) => ` ${l}`), '']),
|
|
204
|
+
].join('\n');
|
|
205
|
+
try {
|
|
206
|
+
const { program, args } = withNoTools(splitCommand(loadConfig().aiCommand), 'Answer only with the JSON object asked for.');
|
|
207
|
+
calls = 1;
|
|
208
|
+
const raw = cleanOutput(runAiSync(program, args, { input: prompt, encoding: 'utf-8', maxBuffer: 4 * 1024 * 1024, timeout: 120_000 }));
|
|
209
|
+
const json = JSON.parse(raw.slice(raw.indexOf('{'), raw.lastIndexOf('}') + 1));
|
|
210
|
+
const keep = {};
|
|
211
|
+
for (const a of ask) {
|
|
212
|
+
const v = json[a.name];
|
|
213
|
+
if (typeof v === 'string' && foundInCode(v, a.lines))
|
|
214
|
+
keep[a.name] = v.trim();
|
|
215
|
+
}
|
|
216
|
+
recordInterface(keep);
|
|
217
|
+
for (const [name, value] of Object.entries(keep))
|
|
218
|
+
filled.push({ name, value });
|
|
219
|
+
}
|
|
220
|
+
catch (e) {
|
|
221
|
+
console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
|
|
222
|
+
for (const a of ask)
|
|
223
|
+
delete memo[a.name]; // look again next time
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
try {
|
|
227
|
+
mkdirSync(dirname(SCAN_PATH), { recursive: true });
|
|
228
|
+
writeFileSync(SCAN_PATH, `${JSON.stringify(memo, null, 2)}\n`, 'utf-8');
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
// best-effort memo
|
|
232
|
+
}
|
|
233
|
+
const done = new Set(filled.map((f) => f.name));
|
|
234
|
+
return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
|
|
235
|
+
}
|
|
@@ -12,10 +12,11 @@
|
|
|
12
12
|
* It runs from the area's nearest package root (where package.json /
|
|
13
13
|
* playwright.config live), with config.envFile merged into the env.
|
|
14
14
|
*/
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
15
|
+
import { testFilesIn } from './readiness.js';
|
|
16
|
+
import { scanNames } from './name-scan.js';
|
|
17
|
+
import { INTERFACE_PATH, blankNames, neededNames } from '../generate/interface-file.js';
|
|
17
18
|
import { spawnSync } from 'node:child_process';
|
|
18
|
-
import { existsSync } from 'node:fs';
|
|
19
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
19
20
|
import { join, dirname, resolve, relative } from 'node:path';
|
|
20
21
|
import { loadConfig, loadEnvFile, testDirForType, resolveFramework, } from '../project/config.js';
|
|
21
22
|
/** Nearest ancestor dir containing a package.json (the runner's cwd). */
|
|
@@ -132,25 +133,21 @@ export function runTests(opts = {}) {
|
|
|
132
133
|
const framework = resolveFramework(config, area, area);
|
|
133
134
|
const root = packageRoot(testDir);
|
|
134
135
|
const rel = relative(root, testDir) || '.';
|
|
135
|
-
//
|
|
136
|
-
//
|
|
137
|
-
|
|
138
|
-
//
|
|
139
|
-
|
|
140
|
-
if (!opts.
|
|
136
|
+
// Every test runs: a feature that is not there yet fails, and its report
|
|
137
|
+
// says so. Before the run, the names those tests wait for are looked for
|
|
138
|
+
// in the code — only where that kind of name lives, only when those lines
|
|
139
|
+
// changed — and filled; the tests themselves are never touched.
|
|
140
|
+
const targets = undefined;
|
|
141
|
+
if (!opts.noScan) {
|
|
141
142
|
const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
|
|
142
143
|
const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
|
|
143
|
-
const
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
console.log(` not
|
|
150
|
-
targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
|
|
151
|
-
if (!targets.length) {
|
|
152
|
-
console.log(`[skip] ${area}: nothing to run yet`);
|
|
153
|
-
continue;
|
|
144
|
+
const blank = blankNames([...new Set(files.flatMap((f) => neededNames(readFileSync(f, 'utf-8'))))]);
|
|
145
|
+
if (blank.length) {
|
|
146
|
+
const res = scanNames(blank, root);
|
|
147
|
+
for (const f of res.filled)
|
|
148
|
+
console.log(` found in the code: ${f.name} = ${f.value}`);
|
|
149
|
+
if (res.stillBlank.length) {
|
|
150
|
+
console.log(` ${res.stillBlank.length} name(s) not in the code yet — their tests report "waiting for the product" (${INTERFACE_PATH}): ${res.stillBlank.slice(0, 6).join(', ')}${res.stillBlank.length > 6 ? ', …' : ''}`);
|
|
154
151
|
}
|
|
155
152
|
}
|
|
156
153
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.3.1",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -394,4 +394,10 @@ function reviewVerdicts(root) {
|
|
|
394
394
|
return (file, title) => (file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined);
|
|
395
395
|
}
|
|
396
396
|
|
|
397
|
-
|
|
397
|
+
/** The name a test is still waiting for (need() threw for a blank interface name), else undefined. */
|
|
398
|
+
function waitingFor(raw) {
|
|
399
|
+
const m = /\[testforge:waiting\] "([^"]+)"/.exec(String(raw || ''));
|
|
400
|
+
return m ? m[1] : undefined;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor };
|
package/reporters/interface.cjs
CHANGED
|
@@ -10,9 +10,8 @@
|
|
|
10
10
|
* const { need } = require('@activate-core/test-forge/reporters/interface.cjs');
|
|
11
11
|
* const ORDER_SUMMARY = need('order summary URL');
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* test with what to fill, never with a guess.
|
|
13
|
+
* A blank name fails each test that needs it with what to fill, never with a
|
|
14
|
+
* guess; the reporters file it as "Waiting for the product".
|
|
16
15
|
*/
|
|
17
16
|
const fs = require('fs');
|
|
18
17
|
const path = require('path');
|
|
@@ -36,11 +35,37 @@ function values() {
|
|
|
36
35
|
return cache;
|
|
37
36
|
}
|
|
38
37
|
|
|
38
|
+
/**
|
|
39
|
+
* While a test file loads (need() at its top, outside any test), a throw would
|
|
40
|
+
* fail the whole file and none of its tests would be reported. Instead each of
|
|
41
|
+
* its tests fails with the waiting error, from a beforeEach. Inside a running
|
|
42
|
+
* test the caller throws as usual.
|
|
43
|
+
*/
|
|
44
|
+
function inRunningTest() {
|
|
45
|
+
try { if (globalThis.expect && globalThis.expect.getState().currentTestName) return true; } catch { /* not Jest */ }
|
|
46
|
+
try { require('@playwright/test').test.info(); return true; } catch { return false; } // throws outside a test
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function failTestsOfThisFile(message) {
|
|
50
|
+
if (inRunningTest()) return false;
|
|
51
|
+
const hook = () => { throw new Error(message); };
|
|
52
|
+
try {
|
|
53
|
+
if (typeof globalThis.beforeEach === 'function') { globalThis.beforeEach(hook); return true; } // Jest
|
|
54
|
+
require('@playwright/test').test.beforeEach(hook); // Playwright
|
|
55
|
+
return true;
|
|
56
|
+
} catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
39
61
|
/** The product's value for `name`, or an error naming what to fill. */
|
|
40
62
|
function need(name) {
|
|
41
63
|
const v = values()[name];
|
|
42
64
|
if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
|
|
43
|
-
|
|
65
|
+
// The prefix lets the reporters file this as "waiting for the product", not as a defect.
|
|
66
|
+
const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json): this test runs for real once it is there.`;
|
|
67
|
+
if (failTestsOfThisFile(message)) return `<waiting: ${name}>`;
|
|
68
|
+
throw new Error(message);
|
|
44
69
|
}
|
|
45
70
|
|
|
46
71
|
module.exports = { need };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to jest.config.js:
|
|
3
3
|
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgeJestReporter {
|
|
7
7
|
constructor(_globalConfig, options) {
|
|
@@ -15,15 +15,17 @@ module.exports = class TestForgeJestReporter {
|
|
|
15
15
|
const name = suiteName(f.testFilePath || f.name || 'suite', this.testRoot);
|
|
16
16
|
const su = suites[name] || (suites[name] = { name, category: this.category, tests: [] });
|
|
17
17
|
for (const a of (f.testResults || f.assertionResults || [])) {
|
|
18
|
-
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
|
|
19
18
|
const raw = (a.failureMessages || [])[0];
|
|
19
|
+
// A test waiting for a name the product does not have yet is not a defect.
|
|
20
|
+
const waiting = a.status === 'failed' ? waitingFor(raw) : undefined;
|
|
21
|
+
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' && !waiting ? 'failed' : 'skipped';
|
|
20
22
|
su.tests.push({
|
|
21
23
|
name: a.fullName || a.title,
|
|
22
24
|
status,
|
|
23
25
|
error: status === 'failed' ? cleanError(raw) : undefined,
|
|
24
26
|
sig: status === 'failed' ? sig(raw) : undefined,
|
|
25
27
|
note: status === 'skipped'
|
|
26
|
-
? (a.status === 'todo' ? 'Marked to-do — this scenario isn’t implemented yet' : 'Skipped — not run in this execution')
|
|
28
|
+
? (waiting ? `Waiting for the product: "${waiting}" is not in it yet` : a.status === 'todo' ? 'Marked to-do — this scenario isn’t implemented yet' : 'Skipped — not run in this execution')
|
|
27
29
|
: undefined,
|
|
28
30
|
checks: testChecks(f.testFilePath, a.location && a.location.line),
|
|
29
31
|
...(status === 'failed' ? expectation(raw) : {}),
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to playwright.config.js reporter array:
|
|
3
3
|
// ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgePlaywrightReporter {
|
|
7
7
|
constructor(options) {
|
|
@@ -35,8 +35,10 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
35
35
|
const file = (test.location && test.location.file) || 'suite';
|
|
36
36
|
const name = suiteName(file, this.testRoot);
|
|
37
37
|
const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
|
|
38
|
-
const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
|
|
39
38
|
const raw = result.error && result.error.message;
|
|
39
|
+
// A test waiting for a name the product does not have yet is not a defect.
|
|
40
|
+
const waiting = result.status !== 'passed' && result.status !== 'skipped' ? waitingFor(raw) : undefined;
|
|
41
|
+
const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' || waiting ? 'skipped' : 'failed';
|
|
40
42
|
const pend = (result.annotations || []).concat(test.annotations || []).filter((x) => x.type === 'skip' || x.type === 'fixme');
|
|
41
43
|
const skipAnno = pend.find((x) => x.description) || pend[0];
|
|
42
44
|
su.tests.push({
|
|
@@ -44,7 +46,7 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
44
46
|
status,
|
|
45
47
|
error: status === 'failed' ? cleanError(raw) : undefined,
|
|
46
48
|
sig: status === 'failed' ? sig(raw) : undefined,
|
|
47
|
-
note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
|
|
49
|
+
note: status === 'skipped' ? (waiting ? `Waiting for the product: "${waiting}" is not in it yet` : (skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
|
|
48
50
|
checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
|
|
49
51
|
|| testChecks(test.location && test.location.file, test.location && test.location.line),
|
|
50
52
|
...(status === 'failed' ? this.failed(test, result) : {}),
|