@activate-core/test-forge 1.0.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +27 -2
- package/dist/core/docs/doc-guidance.js +25 -5
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +17 -1
- package/dist/core/generate/ai-generator.js +208 -283
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +166 -236
- package/dist/core/generate/pending.js +122 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +63 -68
- package/dist/core/generate/suite-planner.js +52 -3
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
|
@@ -12,12 +12,14 @@
|
|
|
12
12
|
* is the repair prompt: the runner's own error is far better evidence than
|
|
13
13
|
* anything we could infer.
|
|
14
14
|
*/
|
|
15
|
-
import { spawnSync } from 'node:child_process';
|
|
16
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
15
|
+
import { spawnSync, spawn } from 'node:child_process';
|
|
16
|
+
import { existsSync, readFileSync, mkdtempSync, rmSync, openSync, closeSync } from 'node:fs';
|
|
17
|
+
import { tmpdir } from 'node:os';
|
|
17
18
|
import { createRequire } from 'node:module';
|
|
18
|
-
import { join, relative } from 'node:path';
|
|
19
|
+
import { join, relative, dirname, resolve } from 'node:path';
|
|
19
20
|
import { loadEnvFile, testDirForType, resolveFramework, } from '../project/config.js';
|
|
20
21
|
import { runnerCommand, packageRoot } from '../run/run-tests.js';
|
|
22
|
+
import { DEFECT_MARK } from './ai-generator.js';
|
|
21
23
|
/** How much runner output to carry into a repair prompt. */
|
|
22
24
|
const MAX_OUTPUT = 12_000;
|
|
23
25
|
/**
|
|
@@ -30,21 +32,6 @@ function trimOutput(s) {
|
|
|
30
32
|
? clean.trim()
|
|
31
33
|
: `…(${clean.length - MAX_OUTPUT} chars trimmed)…\n${clean.slice(-MAX_OUTPUT).trim()}`;
|
|
32
34
|
}
|
|
33
|
-
/**
|
|
34
|
-
* What the runner actually executed. Both Jest and Playwright print these
|
|
35
|
-
* counts; anything we cannot read comes back null and is not guessed at.
|
|
36
|
-
*/
|
|
37
|
-
function countsFrom(output) {
|
|
38
|
-
const n = (re) => {
|
|
39
|
-
const m = output.match(re);
|
|
40
|
-
return m ? Number(m[1]) : null;
|
|
41
|
-
};
|
|
42
|
-
const passed = n(/(\d+)\s+passed/i) ?? n(/Tests:.*?(\d+)\s+passed/i);
|
|
43
|
-
const skipped = n(/(\d+)\s+skipped/i) ?? n(/(\d+)\s+todo/i) ?? n(/(\d+)\s+pending/i);
|
|
44
|
-
if (passed === null && skipped === null)
|
|
45
|
-
return null;
|
|
46
|
-
return { passed: passed ?? 0, skipped: skipped ?? 0 };
|
|
47
|
-
}
|
|
48
35
|
/**
|
|
49
36
|
* Failures no test edit can fix. Sending one to the repair prompt burns AI
|
|
50
37
|
* calls rewriting correct tests against an error they cannot cause (one
|
|
@@ -78,70 +65,206 @@ export function browserPreflight(root) {
|
|
|
78
65
|
const output = `${res.stdout ?? ''}\n${res.stderr ?? ''}`;
|
|
79
66
|
return infraFailure(output) ?? (output.trim().split('\n').find((l) => l.trim()) || 'the test browser could not start');
|
|
80
67
|
}
|
|
68
|
+
const stripAnsi = (s) => s.replace(/\u001b\[[0-9;]*m/g, '');
|
|
69
|
+
/** Results of a Playwright JSON report (`--reporter=json`). */
|
|
70
|
+
export function fromPlaywright(json) {
|
|
71
|
+
const out = { ran: true, passed: 0, passes: [], failures: [], skips: [] };
|
|
72
|
+
// Spec files are recorded relative to the config's test directory.
|
|
73
|
+
const at = (f) => resolve(json.config?.rootDir ?? '.', f ?? '');
|
|
74
|
+
let specs = 0;
|
|
75
|
+
const walk = (suite) => {
|
|
76
|
+
for (const spec of suite.specs ?? []) {
|
|
77
|
+
for (const t of spec.tests ?? []) {
|
|
78
|
+
specs += 1;
|
|
79
|
+
const r = (t.results ?? [])[(t.results ?? []).length - 1] ?? {};
|
|
80
|
+
const status = r.status ?? t.status;
|
|
81
|
+
if (status === 'passed') {
|
|
82
|
+
out.passed += 1;
|
|
83
|
+
out.passes.push({ file: at(spec.file), line: spec.line, title: spec.title });
|
|
84
|
+
}
|
|
85
|
+
else if (status === 'skipped') {
|
|
86
|
+
const anns = [...(r.annotations ?? []), ...(t.annotations ?? [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
|
|
87
|
+
// A declared pending test's annotation points at the declaration
|
|
88
|
+
// itself; a skip the test decided while running points at the call
|
|
89
|
+
// inside its body. Only the second kind says anything about this run.
|
|
90
|
+
const runtime = anns.find((a) => a.location && (a.location.line !== spec.line || a.location.column !== spec.column));
|
|
91
|
+
if (runtime || !anns.length) {
|
|
92
|
+
out.skips.push({
|
|
93
|
+
file: at(spec.file),
|
|
94
|
+
line: spec.line,
|
|
95
|
+
title: spec.title,
|
|
96
|
+
detail: runtime?.description || (anns.length ? '' : 'not run — an earlier test in a serial block failed'),
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
const msg = stripAnsi(r.error?.message ?? (r.errors ?? []).map((e) => e.message).join('\n'));
|
|
102
|
+
out.failures.push({ file: at(spec.file), line: spec.line, title: spec.title, detail: msg.split('\n').slice(0, 40).join('\n') });
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
for (const s of suite.suites ?? [])
|
|
107
|
+
walk(s);
|
|
108
|
+
};
|
|
109
|
+
for (const s of json.suites ?? [])
|
|
110
|
+
walk(s);
|
|
111
|
+
if (!specs) {
|
|
112
|
+
const errors = (json.errors ?? []).map((e) => stripAnsi(e.message ?? '')).join('\n').trim();
|
|
113
|
+
if (!errors)
|
|
114
|
+
return null;
|
|
115
|
+
return { ran: false, reason: `the suite could not start: ${errors.split('\n')[0]}`, passed: 0, passes: [], failures: [], skips: [], infra: infraFailure(errors) ?? undefined };
|
|
116
|
+
}
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
/** Results of a Jest JSON report (`--json --testLocationInResults`). */
|
|
120
|
+
function fromJest(json) {
|
|
121
|
+
const out = { ran: true, passed: 0, passes: [], failures: [], skips: [] };
|
|
122
|
+
const files = json.testResults ?? [];
|
|
123
|
+
if (!files.length)
|
|
124
|
+
return null;
|
|
125
|
+
for (const f of files) {
|
|
126
|
+
if (!(f.assertionResults ?? []).length && f.message) {
|
|
127
|
+
return { ran: false, reason: `the suite could not start: ${stripAnsi(f.message).trim().split('\n')[0]}`, passed: 0, passes: [], failures: [], skips: [], infra: infraFailure(f.message) ?? undefined };
|
|
128
|
+
}
|
|
129
|
+
for (const a of f.assertionResults ?? []) {
|
|
130
|
+
if (a.status === 'passed') {
|
|
131
|
+
out.passed += 1;
|
|
132
|
+
out.passes.push({ file: f.name, line: a.location?.line ?? 0, title: a.fullName ?? a.title });
|
|
133
|
+
}
|
|
134
|
+
else if (a.status === 'failed') {
|
|
135
|
+
out.failures.push({
|
|
136
|
+
file: f.name,
|
|
137
|
+
line: a.location?.line ?? 0,
|
|
138
|
+
title: a.fullName ?? a.title,
|
|
139
|
+
detail: stripAnsi((a.failureMessages ?? []).join('\n')).split('\n').slice(0, 40).join('\n'),
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
// pending / todo are declared in the file, not decided by this run
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return out;
|
|
146
|
+
}
|
|
81
147
|
/**
|
|
82
|
-
* Run
|
|
83
|
-
*
|
|
148
|
+
* Run ONE generated file and read its results from the runner's own JSON
|
|
149
|
+
* report — per test, not per journey. Never throws.
|
|
150
|
+
*
|
|
151
|
+
* Per file because a journey can hold hundreds of tests: one run for all of
|
|
152
|
+
* them outlived any sane timeout, the step reported "not verified" once, and
|
|
153
|
+
* a whole suite reached the project without a single test having run. Per
|
|
154
|
+
* test because a journey's repair must touch only the files that failed, and
|
|
155
|
+
* must be told which failures they were.
|
|
84
156
|
*/
|
|
85
|
-
export function
|
|
86
|
-
|
|
87
|
-
|
|
157
|
+
export function verifyFile(config, area, file) {
|
|
158
|
+
return verifyRun(config, area, [file]);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Run several files in ONE runner invocation, the way a team or CI runs a
|
|
162
|
+
* suite. Files that pass one at a time can still break each other when they
|
|
163
|
+
* share a run — one signs in as the shared account and ends the session
|
|
164
|
+
* another relies on — and only a run together shows it. Never throws.
|
|
165
|
+
*/
|
|
166
|
+
export async function verifyRun(config, area, files, timeoutMs) {
|
|
167
|
+
const none = (reason, infra) => ({ ran: false, reason, infra, passed: 0, passes: [], failures: [], skips: [] });
|
|
168
|
+
const present = files.filter((f) => existsSync(f));
|
|
169
|
+
if (!present.length)
|
|
170
|
+
return none('the file is not on disk');
|
|
88
171
|
const testDir = testDirForType(config, area);
|
|
89
172
|
const root = packageRoot(testDir);
|
|
90
173
|
const rel = relative(root, testDir) || '.';
|
|
91
174
|
const framework = resolveFramework(config, area, area);
|
|
92
|
-
const present = files.filter((f) => existsSync(f));
|
|
93
|
-
if (!present.length) {
|
|
94
|
-
return { ran: false, ok: false, output: '', reason: 'generated files are not on disk' };
|
|
95
|
-
}
|
|
96
|
-
const cmd = runnerCommand(config, area, framework, root, rel, undefined, present);
|
|
97
175
|
const envFile = config.envFile ?? '.env.testing';
|
|
98
176
|
const envVars = loadEnvFile(envFile);
|
|
99
177
|
if (!Object.keys(envVars).length && !existsSync(envFile)) {
|
|
100
|
-
return {
|
|
101
|
-
ran: false,
|
|
102
|
-
ok: false,
|
|
103
|
-
output: '',
|
|
104
|
-
reason: `no ${envFile} — cannot reach the app to verify`,
|
|
105
|
-
};
|
|
178
|
+
return none(`no ${envFile} — cannot reach the app to verify`);
|
|
106
179
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
180
|
+
// The project's own testCommand is for whole runs; here we name the files
|
|
181
|
+
// and ask for a machine-readable report (which also keeps a verify run from
|
|
182
|
+
// being published to the dashboard as if it were a real one).
|
|
183
|
+
const base = runnerCommand({ ...config, testCommand: undefined }, area, framework, root, rel, undefined, present);
|
|
184
|
+
const report = join(mkdtempSync(join(tmpdir(), 'tf-verify-')), 'report.json');
|
|
185
|
+
const timeout = timeoutMs ?? config.verifyTimeoutMs ?? 600_000;
|
|
186
|
+
// The runner enforces the limit itself where it can: Playwright stops at its
|
|
187
|
+
// global timeout and still writes the report for the tests that ran. The
|
|
188
|
+
// process limit below is only the backstop — and it must end the WHOLE tree:
|
|
189
|
+
// killing the shell alone left the runner and its browser alive, and a
|
|
190
|
+
// verify waited on them for hours.
|
|
191
|
+
const out = join(dirname(report), 'output.txt');
|
|
192
|
+
const cmd = framework === 'playwright'
|
|
193
|
+
? `${base} --reporter=json --global-timeout=${timeout}`
|
|
194
|
+
: `${base} --json --testLocationInResults --forceExit --outputFile=${report}`;
|
|
195
|
+
const res = await runTree(cmd, out, { cwd: root, env: { ...process.env, ...envVars, PLAYWRIGHT_JSON_OUTPUT_NAME: report } }, timeout + 120_000);
|
|
196
|
+
let raw = '';
|
|
197
|
+
try {
|
|
198
|
+
raw = readFileSync(out, 'utf-8');
|
|
117
199
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
200
|
+
catch {
|
|
201
|
+
// no output at all
|
|
202
|
+
}
|
|
203
|
+
const output = trimOutput(raw);
|
|
204
|
+
if (res.timedOut) {
|
|
205
|
+
return none(`timed out after ${Math.round(timeout / 60000)} min (raise verifyTimeoutMs)`);
|
|
123
206
|
}
|
|
124
207
|
if (/command not found|Cannot find module '@playwright|is not recognized/i.test(output)) {
|
|
125
|
-
return
|
|
208
|
+
return none('test runner is not installed', 'test runner is not installed');
|
|
126
209
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
if (counts && counts.passed === 0 && counts.skipped > 0) {
|
|
138
|
-
return {
|
|
139
|
-
ran: true,
|
|
140
|
-
ok: false,
|
|
141
|
-
output: `Every test was SKIPPED (${counts.skipped} skipped, 0 passed) — nothing was actually verified.\nThe precondition or seed below could not be established; fix that so the tests run.\n\n${output}`,
|
|
142
|
-
};
|
|
210
|
+
let parsed = null;
|
|
211
|
+
try {
|
|
212
|
+
const json = JSON.parse(readFileSync(report, 'utf-8'));
|
|
213
|
+
parsed = framework === 'playwright' ? fromPlaywright(json) : fromJest(json);
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
parsed = null;
|
|
217
|
+
}
|
|
218
|
+
finally {
|
|
219
|
+
rmSync(dirname(report), { recursive: true, force: true });
|
|
143
220
|
}
|
|
144
|
-
|
|
221
|
+
if (!parsed) {
|
|
222
|
+
const infra = infraFailure(output);
|
|
223
|
+
return none(infra ?? (output.split('\n').find((l) => l.trim()) || 'the runner produced no results'), infra ?? undefined);
|
|
224
|
+
}
|
|
225
|
+
// Environment failures are judged per test: one test that stubs a refused
|
|
226
|
+
// connection on purpose must not make the whole file look like a dead app.
|
|
227
|
+
if (parsed.failures.length && parsed.failures.every((f) => infraFailure(f.detail))) {
|
|
228
|
+
parsed.infra = infraFailure(parsed.failures[0].detail) ?? undefined;
|
|
229
|
+
}
|
|
230
|
+
return parsed;
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Run a shell command in its own process group, output to a file, and kill the
|
|
234
|
+
* whole group if it outlives the limit — the runner, its workers and browsers
|
|
235
|
+
* included, not just the shell that started them.
|
|
236
|
+
*/
|
|
237
|
+
export function runTree(cmd, outFile, opts, limitMs) {
|
|
238
|
+
return new Promise((done) => {
|
|
239
|
+
const fd = openSync(outFile, 'w');
|
|
240
|
+
const child = spawn('sh', ['-c', cmd], { cwd: opts.cwd, env: opts.env, detached: true, stdio: ['ignore', fd, fd] });
|
|
241
|
+
let timedOut = false;
|
|
242
|
+
const timer = setTimeout(() => {
|
|
243
|
+
timedOut = true;
|
|
244
|
+
try {
|
|
245
|
+
if (child.pid)
|
|
246
|
+
process.kill(-child.pid, 'SIGKILL');
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
// already gone
|
|
250
|
+
}
|
|
251
|
+
}, limitMs);
|
|
252
|
+
const finish = (code) => {
|
|
253
|
+
clearTimeout(timer);
|
|
254
|
+
closeSync(fd);
|
|
255
|
+
done({ code, timedOut });
|
|
256
|
+
};
|
|
257
|
+
child.on('error', () => finish(null));
|
|
258
|
+
child.on('close', (code) => finish(code));
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* A skip that says only that the environment lacks something — an env var,
|
|
263
|
+
* an opt-in flag — is the environment's to fix. Any other runtime skip is the
|
|
264
|
+
* test reporting something it observed, and its own setup is a suspect.
|
|
265
|
+
*/
|
|
266
|
+
export function environmentSkip(reason) {
|
|
267
|
+
return /\b[A-Z][A-Z0-9_]{2,}\b[^.]*\b(not set|is not set|unset|missing|absent|empty|not configured)\b|\b(set|provide|export)\s+[A-Z][A-Z0-9_]{2,}\b|TESTFORGE_ALLOW_/.test(reason);
|
|
145
268
|
}
|
|
146
269
|
/**
|
|
147
270
|
* Packages the generated harness declares it needs (REQUIRED_PACKAGES) that
|
|
@@ -170,3 +293,40 @@ export function missingPackages(harnessFile, root) {
|
|
|
170
293
|
}
|
|
171
294
|
});
|
|
172
295
|
}
|
|
296
|
+
/**
|
|
297
|
+
* Declaration lines (1-based) of tests a repair judged to be failing on the
|
|
298
|
+
* application, with the claim it wrote — read from the comment directly above.
|
|
299
|
+
*/
|
|
300
|
+
export function claimedDefects(content) {
|
|
301
|
+
const lines = content.split('\n');
|
|
302
|
+
const claims = new Map();
|
|
303
|
+
for (let i = 0; i < lines.length; i++) {
|
|
304
|
+
if (!/\b(?:test|it)(?:\.\w+)?\s*\(/.test(lines[i]))
|
|
305
|
+
continue;
|
|
306
|
+
for (let j = i - 1; j >= 0 && lines[j].trim().startsWith('//'); j--) {
|
|
307
|
+
const at = lines[j].indexOf(DEFECT_MARK);
|
|
308
|
+
if (at !== -1)
|
|
309
|
+
claims.set(i + 1, lines[j].slice(at + DEFECT_MARK.length).trim());
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
return claims;
|
|
313
|
+
}
|
|
314
|
+
const titles = (s) => new Set([...s.matchAll(/\b(?:test|it)(?:\.(?:only|fixme|skip|fail|slow))?\s*\(\s*(['"`])((?:\\.|(?!\1)[\s\S])*?)\1/g)].map((m) => m[2]));
|
|
315
|
+
const count = (s, re) => (s.match(re) || []).length;
|
|
316
|
+
/**
|
|
317
|
+
* What a repair may not do, checked mechanically rather than trusted: drop or
|
|
318
|
+
* rename a test, add a skip, or lose an assertion. Each of those turns a
|
|
319
|
+
* caught defect into a silent one. Returns why the repair is refused, or null.
|
|
320
|
+
*/
|
|
321
|
+
export function repairGuard(before, after) {
|
|
322
|
+
const gone = [...titles(before)].filter((t) => !titles(after).has(t));
|
|
323
|
+
if (gone.length)
|
|
324
|
+
return `it removed or renamed ${gone.length} test(s), e.g. "${gone[0].slice(0, 60)}"`;
|
|
325
|
+
const skips = /\b(?:test|it)\.(?:skip|fixme)\s*\(/g;
|
|
326
|
+
if (count(after, skips) > count(before, skips))
|
|
327
|
+
return 'it added a skip';
|
|
328
|
+
const expects = /\bexpect(?:\.soft)?\s*\(/g;
|
|
329
|
+
if (count(after, expects) < count(before, expects))
|
|
330
|
+
return 'it removed an assertion';
|
|
331
|
+
return null;
|
|
332
|
+
}
|
|
@@ -27,7 +27,9 @@ function extractEnvVars(text) {
|
|
|
27
27
|
vars.add(m[1]);
|
|
28
28
|
}
|
|
29
29
|
// A generated suite often reads env through a small helper: env('TEST_X', default).
|
|
30
|
-
|
|
30
|
+
// The helper's name varies (env, envVal, readEnv…), so take any call whose
|
|
31
|
+
// first argument is an env-shaped name.
|
|
32
|
+
for (const m of text.matchAll(/\b[A-Za-z_$][\w$]*\(\s*['"]([A-Z][A-Z0-9]*_[A-Z0-9_]+)['"]/g)) {
|
|
31
33
|
vars.add(m[1]);
|
|
32
34
|
}
|
|
33
35
|
// backtick-wrapped ALL_CAPS names in "env var" lines
|
package/dist/core/run/report.js
CHANGED
|
@@ -7,8 +7,13 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import fs from 'node:fs';
|
|
9
9
|
import path from 'node:path';
|
|
10
|
+
import { createRequire } from 'node:module';
|
|
11
|
+
import { packageRoot } from './run-tests.js';
|
|
10
12
|
import os from 'node:os';
|
|
11
13
|
import { execSync } from 'node:child_process';
|
|
14
|
+
// The reporters' helpers, so a report published from a results file reads the
|
|
15
|
+
// same as one the reporter published during the run.
|
|
16
|
+
const shared = createRequire(import.meta.url)('../../../reporters/_shared.cjs');
|
|
12
17
|
/**
|
|
13
18
|
* Load TESTFORGE_KEY (and any other vars) from a gitignored `.testforge.env`
|
|
14
19
|
* (then `.env`) at the project root, so a developer can keep the key in a file
|
|
@@ -16,7 +21,14 @@ import { execSync } from 'node:child_process';
|
|
|
16
21
|
* environment (a real `export` still wins). Add `.testforge.env` to .gitignore.
|
|
17
22
|
*/
|
|
18
23
|
export function loadDotenv(cwd = process.cwd()) {
|
|
19
|
-
|
|
24
|
+
let envFile = '';
|
|
25
|
+
try {
|
|
26
|
+
envFile = JSON.parse(fs.readFileSync(path.join(cwd, 'testforge.config.json'), 'utf8')).envFile || '';
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// no config — the fixed names below still apply
|
|
30
|
+
}
|
|
31
|
+
for (const file of [envFile, '.testforge.env', '.env'].filter(Boolean)) {
|
|
20
32
|
const p = path.join(cwd, file);
|
|
21
33
|
if (!fs.existsSync(p))
|
|
22
34
|
continue;
|
|
@@ -50,20 +62,39 @@ export function loadReportConfig(cwd = process.cwd()) {
|
|
|
50
62
|
return {};
|
|
51
63
|
}
|
|
52
64
|
const strip = (s) => (s || '').replace(/\x1b\[[0-9;]*m/g, '');
|
|
53
|
-
/**
|
|
54
|
-
function
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
65
|
+
/** Map a Playwright `--reporter=json` report to the dashboard run contract. */
|
|
66
|
+
export function mapPlaywright(json, category = 'web') {
|
|
67
|
+
const suites = {};
|
|
68
|
+
// Spec files are relative to the config's testDir; the report records the root it used.
|
|
69
|
+
const rootDir = (json.config && json.config.rootDir) || process.cwd();
|
|
70
|
+
const walk = (suite) => {
|
|
71
|
+
for (const spec of suite.specs || []) {
|
|
72
|
+
const name = String(spec.file || suite.file || 'suite').split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
|
|
73
|
+
const su = (suites[name] ||= { name, category, tests: [] });
|
|
74
|
+
for (const t of spec.tests || []) {
|
|
75
|
+
const r = (t.results || [])[(t.results || []).length - 1] || {};
|
|
76
|
+
const st = r.status || t.status;
|
|
77
|
+
const status = st === 'passed' ? 'passed' : st === 'skipped' ? 'skipped' : 'failed';
|
|
78
|
+
const raw = r.error && r.error.message;
|
|
79
|
+
const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
|
|
80
|
+
const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
|
|
81
|
+
su.tests.push({
|
|
82
|
+
name: spec.title,
|
|
83
|
+
status,
|
|
84
|
+
checks: shared.testChecks(loc, spec.line),
|
|
85
|
+
...(status === 'failed' ? shared.expectation(raw) : {}),
|
|
86
|
+
error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
|
|
87
|
+
sig: status === 'failed' ? shared.sig(raw) : undefined,
|
|
88
|
+
note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
for (const c of suite.suites || [])
|
|
93
|
+
walk(c);
|
|
94
|
+
};
|
|
95
|
+
for (const s of json.suites || [])
|
|
96
|
+
walk(s);
|
|
97
|
+
return { suites: Object.values(suites), durationSec: Math.round(((json.stats && json.stats.duration) || 0) / 1000) };
|
|
67
98
|
}
|
|
68
99
|
/** Map a Jest `--json` report to the dashboard run contract. */
|
|
69
100
|
export function mapJest(json, category = 'api') {
|
|
@@ -81,7 +112,7 @@ export function mapJest(json, category = 'api') {
|
|
|
81
112
|
name: a.title,
|
|
82
113
|
status,
|
|
83
114
|
error: status === 'failed' ? err.slice(0, 200) : undefined,
|
|
84
|
-
sig: status === 'failed' ?
|
|
115
|
+
sig: status === 'failed' ? shared.sig(raw) : undefined,
|
|
85
116
|
});
|
|
86
117
|
}
|
|
87
118
|
}
|
|
@@ -90,8 +121,11 @@ export function mapJest(json, category = 'api') {
|
|
|
90
121
|
}
|
|
91
122
|
export async function publishReport(opts) {
|
|
92
123
|
const json = JSON.parse(fs.readFileSync(opts.resultsPath, 'utf8'));
|
|
93
|
-
|
|
94
|
-
const
|
|
124
|
+
// Jest's report has testResults; Playwright's has suites.
|
|
125
|
+
const { suites, durationSec } = Array.isArray(json.testResults)
|
|
126
|
+
? mapJest(json, opts.category || 'api')
|
|
127
|
+
: mapPlaywright(json, opts.category || 'web');
|
|
128
|
+
const body = { project: opts.project, env: opts.env || process.env.TESTFORGE_ENV || 'local', durationSec, suites };
|
|
95
129
|
const res = await fetch(opts.dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
|
|
96
130
|
method: 'POST',
|
|
97
131
|
headers: { Authorization: `Bearer ${opts.apiKey}`, 'Content-Type': 'application/json' },
|
|
@@ -320,8 +354,9 @@ export function scanInventory(cwd, testOutput, ext, framework = {}) {
|
|
|
320
354
|
if (!fs.existsSync(dir))
|
|
321
355
|
continue;
|
|
322
356
|
if (framework[category] === 'playwright') {
|
|
323
|
-
//
|
|
324
|
-
|
|
357
|
+
// The config sits at the package root (or, beside a project's own
|
|
358
|
+
// config, in the tests' parent folder) — list from wherever it is.
|
|
359
|
+
if (playwrightList(packageRoot(dir), category, suites) || playwrightList(path.dirname(dir), category, suites)) {
|
|
325
360
|
attachPending(dir, ext, suites);
|
|
326
361
|
continue;
|
|
327
362
|
}
|
|
@@ -35,8 +35,17 @@ export function packageRoot(dir) {
|
|
|
35
35
|
* journey); without it the whole area runs.
|
|
36
36
|
*/
|
|
37
37
|
export function runnerCommand(config, area, framework, root, rel, journey, targets) {
|
|
38
|
-
|
|
38
|
+
let explicit = config.testCommand?.[area];
|
|
39
39
|
const paths = (targets ?? []).map((t) => relative(root, t) || t);
|
|
40
|
+
if (explicit && (paths.length || journey)) {
|
|
41
|
+
// A command that already names the whole test folder ("npx playwright
|
|
42
|
+
// test tests/e2e") would still run all of it next to a narrower path:
|
|
43
|
+
// the runner treats every path argument as one more filter to include.
|
|
44
|
+
explicit = explicit
|
|
45
|
+
.split(/\s+/)
|
|
46
|
+
.filter((tok) => tok.replace(/^\.\//, '').replace(/\/$/, '') !== rel)
|
|
47
|
+
.join(' ');
|
|
48
|
+
}
|
|
40
49
|
if (explicit) {
|
|
41
50
|
if (paths.length)
|
|
42
51
|
return `${explicit} ${paths.join(' ')}`;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -17,9 +17,10 @@ function findRoot(cwd) {
|
|
|
17
17
|
return cwd;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
// Load TESTFORGE_KEY from
|
|
20
|
+
// Load TESTFORGE_KEY from the project's test env file (envFile in testforge.config.json),
|
|
21
|
+
// or a gitignored .testforge.env / .env. Never overrides a real env var.
|
|
21
22
|
function loadEnv(root) {
|
|
22
|
-
for (const f of ['.testforge.env', '.env']) {
|
|
23
|
+
for (const f of [loadConfig(root).envFile, '.testforge.env', '.env'].filter(Boolean)) {
|
|
23
24
|
const p = path.join(root, f);
|
|
24
25
|
if (!fs.existsSync(p)) continue;
|
|
25
26
|
for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
|
|
@@ -86,6 +87,56 @@ function cleanError(raw) {
|
|
|
86
87
|
return (lead + (rest ? '\n\n' + rest : '')).slice(0, 700);
|
|
87
88
|
}
|
|
88
89
|
|
|
90
|
+
// What a test checks, in its own words: the requirement each assertion cites in an
|
|
91
|
+
// "// oracle: …" comment (every TestForge test carries them), read from the test's
|
|
92
|
+
// body in its source file. Runners report the declaration line, so read from there
|
|
93
|
+
// to the end of that test. Undefined when the file has no such comments.
|
|
94
|
+
const fileLines = new Map();
|
|
95
|
+
function testChecks(file, line) {
|
|
96
|
+
if (!file || !line) return undefined;
|
|
97
|
+
let lines = fileLines.get(file);
|
|
98
|
+
if (!lines) {
|
|
99
|
+
try { lines = fs.readFileSync(file, 'utf8').split('\n'); } catch { lines = []; }
|
|
100
|
+
fileLines.set(file, lines);
|
|
101
|
+
}
|
|
102
|
+
const decl = lines[line - 1];
|
|
103
|
+
if (decl === undefined) return undefined;
|
|
104
|
+
const indent = decl.match(/^\s*/)[0].length;
|
|
105
|
+
const found = [];
|
|
106
|
+
let cur = null;
|
|
107
|
+
for (let i = line; i < lines.length; i++) {
|
|
108
|
+
const l = lines[i];
|
|
109
|
+
const t = l.trim();
|
|
110
|
+
// the test ends at the first line back at (or left of) its own indentation
|
|
111
|
+
if (t && l.match(/^\s*/)[0].length <= indent) break;
|
|
112
|
+
const c = t.match(/^\/\/\s?(.*)$/);
|
|
113
|
+
if (c && /^oracle\s*:/i.test(c[1])) { cur = c[1].replace(/^oracle\s*:\s*/i, ''); found.push(cur); continue; }
|
|
114
|
+
if (c && cur !== null) { found[found.length - 1] += ' ' + c[1].trim(); continue; }
|
|
115
|
+
cur = null;
|
|
116
|
+
}
|
|
117
|
+
const uniq = [...new Set(found.map((x) => x.replace(/\s+/g, ' ').trim()).filter(Boolean))];
|
|
118
|
+
return uniq.length ? uniq.join('\n').slice(0, 1200) : undefined;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// For a failed check: what it wanted and what it got, from the runner's own message.
|
|
122
|
+
// A test's own failure message ("no visible reason, only a red field") says what the
|
|
123
|
+
// app did better than a bare true/false, so it wins over Received when present.
|
|
124
|
+
function expectation(raw) {
|
|
125
|
+
const s = strip(raw || '');
|
|
126
|
+
if (!s) return {};
|
|
127
|
+
const pick = (re) => { const m = s.match(re); return m ? m[1].trim() : undefined; };
|
|
128
|
+
const expected = pick(/^\s*Expected(?: [a-z ]+)?:\s*(.+)$/m);
|
|
129
|
+
let actual = pick(/^\s*Received(?: [a-z ]+)?:\s*(.+)$/m);
|
|
130
|
+
if (/element\(s\) not found/.test(s)) actual = 'the element was not found on the page';
|
|
131
|
+
const first = (s.split('\n').find((l) => l.trim()) || '').replace(/^\s*Error:\s*/, '').trim();
|
|
132
|
+
const own = first && !/^expect[.(]|^Test timeout|^locator\./i.test(first) ? first : undefined;
|
|
133
|
+
if (/^Test timeout of/i.test(first)) return { expected: plain(expected), actual: first };
|
|
134
|
+
// A bare true/false says nothing once the test's own message explains it.
|
|
135
|
+
return { expected: own && /^(true|false)$/.test(expected || '') ? undefined : plain(expected), actual: own || plain(actual) };
|
|
136
|
+
}
|
|
137
|
+
const PLAIN = { visible: 'shown on screen', hidden: 'not shown on screen (hidden)', enabled: 'enabled', disabled: 'disabled' };
|
|
138
|
+
const plain = (v) => (v && PLAIN[v]) || v;
|
|
139
|
+
|
|
89
140
|
function sh(cmd, cwd) {
|
|
90
141
|
try { return (execSync(cmd, { cwd, stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim()) || undefined; }
|
|
91
142
|
catch { return undefined; }
|
|
@@ -166,4 +217,4 @@ async function publish(category, suites, durationSec) {
|
|
|
166
217
|
}
|
|
167
218
|
}
|
|
168
219
|
|
|
169
|
-
module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv };
|
|
220
|
+
module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to jest.config.js:
|
|
3
3
|
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
-
const { cleanError, sig, publish } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgeJestReporter {
|
|
7
7
|
constructor(_globalConfig, options) {
|
|
@@ -23,6 +23,8 @@ module.exports = class TestForgeJestReporter {
|
|
|
23
23
|
note: status === 'skipped'
|
|
24
24
|
? (a.status === 'todo' ? 'Marked to-do — this scenario isn’t implemented yet' : 'Skipped — not run in this execution')
|
|
25
25
|
: undefined,
|
|
26
|
+
checks: testChecks(f.testFilePath, a.location && a.location.line),
|
|
27
|
+
...(status === 'failed' ? expectation(raw) : {}),
|
|
26
28
|
});
|
|
27
29
|
}
|
|
28
30
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to playwright.config.js reporter array:
|
|
3
3
|
// ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
|
|
4
|
-
const { cleanError, sig, publish } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgePlaywrightReporter {
|
|
7
7
|
constructor(options) {
|
|
@@ -15,13 +15,17 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
15
15
|
const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
|
|
16
16
|
const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
|
|
17
17
|
const raw = result.error && result.error.message;
|
|
18
|
-
const
|
|
18
|
+
const pend = (result.annotations || []).concat(test.annotations || []).filter((x) => x.type === 'skip' || x.type === 'fixme');
|
|
19
|
+
const skipAnno = pend.find((x) => x.description) || pend[0];
|
|
19
20
|
su.tests.push({
|
|
20
21
|
name: test.title,
|
|
21
22
|
status,
|
|
22
23
|
error: status === 'failed' ? cleanError(raw) : undefined,
|
|
23
24
|
sig: status === 'failed' ? sig(raw) : undefined,
|
|
24
25
|
note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
|
|
26
|
+
checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
|
|
27
|
+
|| testChecks(test.location && test.location.file, test.location && test.location.line),
|
|
28
|
+
...(status === 'failed' ? expectation(raw) : {}),
|
|
25
29
|
});
|
|
26
30
|
}
|
|
27
31
|
async onEnd(result) {
|