@activate-core/test-forge 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/gates/validate.js +25 -2
- package/dist/core/generate/defect-review.js +28 -4
- package/dist/core/generate/run-generate.js +3 -3
- package/dist/core/generate/verify-journey.js +12 -0
- package/dist/core/run/report.js +26 -17
- package/package.json +1 -1
- package/reporters/_shared.cjs +170 -14
- package/reporters/jest-reporter.cjs +4 -2
- package/reporters/playwright-reporter.cjs +32 -3
|
@@ -17,8 +17,8 @@
|
|
|
17
17
|
* false-rejecting valid TypeScript.
|
|
18
18
|
*/
|
|
19
19
|
import { execFileSync } from 'node:child_process';
|
|
20
|
-
import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
|
21
|
-
import { join } from 'node:path';
|
|
20
|
+
import { writeFileSync, mkdtempSync, rmSync, existsSync } from 'node:fs';
|
|
21
|
+
import { join, dirname, resolve } from 'node:path';
|
|
22
22
|
import { tmpdir } from 'node:os';
|
|
23
23
|
import { reasonlessPending, pendingDeclarations } from '../generate/pending.js';
|
|
24
24
|
/**
|
|
@@ -169,6 +169,13 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
169
169
|
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
170
170
|
return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
|
|
171
171
|
}
|
|
172
|
+
// F17 — relative imports resolve from where the file is written.
|
|
173
|
+
if (ctx.filePath) {
|
|
174
|
+
const missing = unresolvedImports(content, ctx.filePath);
|
|
175
|
+
if (missing.length) {
|
|
176
|
+
return `relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
172
179
|
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
173
180
|
// as a bare "fixme", indistinguishable from the tool giving up. Runs on
|
|
174
181
|
// content normalisePending() has already given every reason it could find.
|
|
@@ -386,3 +393,19 @@ function looksStructurallyClean(content) {
|
|
|
386
393
|
count(content, '(') === count(content, ')');
|
|
387
394
|
return balanced;
|
|
388
395
|
}
|
|
396
|
+
const RESOLVE_EXTS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json'];
|
|
397
|
+
/** Relative module specifiers (import / require / dynamic import) that reach no file from `filePath`. */
|
|
398
|
+
export function unresolvedImports(content, filePath) {
|
|
399
|
+
const specs = new Set();
|
|
400
|
+
const re = /(?:\bfrom\s*|\brequire\s*\(\s*|\bimport\s*\(\s*|^\s*import\s+)(['"])(\.{1,2}\/[^'"]*)\1/gm;
|
|
401
|
+
for (const m of content.matchAll(re))
|
|
402
|
+
specs.add(m[2]);
|
|
403
|
+
const base = dirname(resolve(filePath));
|
|
404
|
+
return [...specs].filter((spec) => {
|
|
405
|
+
const target = resolve(base, spec);
|
|
406
|
+
// A compiled .js specifier may point at the .ts source.
|
|
407
|
+
const stem = target.replace(/\.(c|m)?js$/, '');
|
|
408
|
+
return !RESOLVE_EXTS.some((e) => existsSync(target + e) || existsSync(stem + e))
|
|
409
|
+
&& !RESOLVE_EXTS.slice(1).some((e) => existsSync(join(target, 'index' + e)));
|
|
410
|
+
});
|
|
411
|
+
}
|
|
@@ -20,6 +20,7 @@ import { existsSync, readFileSync } from 'node:fs';
|
|
|
20
20
|
import { relative } from 'node:path';
|
|
21
21
|
import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
|
|
22
22
|
import { runWithStdin } from './ai-generator.js';
|
|
23
|
+
import { aiUnavailableReason, noteAiFailure, noteAiSuccess, waitForAi } from './ai-session.js';
|
|
23
24
|
const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'unproven'];
|
|
24
25
|
/**
|
|
25
26
|
* Group failing tests by the claim marked above them. A parametrised test
|
|
@@ -149,24 +150,47 @@ export function splitReview(group, parsed) {
|
|
|
149
150
|
out.unshift({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence });
|
|
150
151
|
return out;
|
|
151
152
|
}
|
|
152
|
-
/**
|
|
153
|
-
|
|
153
|
+
/**
|
|
154
|
+
* Review every claim group, a few at a time. Never throws.
|
|
155
|
+
*
|
|
156
|
+
* A review that failed because the AI is gone (usage limit, logged out) is
|
|
157
|
+
* not a verdict. On the CleanMe booking-summary run the limit landed exactly
|
|
158
|
+
* here: all 18 claims came back "unproven — the review could not be read",
|
|
159
|
+
* the journey-level wait only noticed at the next AI call, and the summary
|
|
160
|
+
* reported 0 defects. So the group goes back on the queue, every worker
|
|
161
|
+
* shares one wait, and the review is asked again once the AI answers.
|
|
162
|
+
*/
|
|
163
|
+
export async function reviewDefects(journey, groups, passes, io) {
|
|
154
164
|
const config = loadConfig();
|
|
155
165
|
const decisions = decisionsSource(config.decisions);
|
|
156
166
|
// A linked list (an internal Notion page) needs the reference tools to be read.
|
|
157
167
|
const tools = decisions.link ? withReferenceTools : withReadOnlyTools;
|
|
158
168
|
const { program, args } = tools(splitCommand(config.aiCommand));
|
|
169
|
+
const ask = io?.ask ?? ((prompt) => runWithStdin(program, args, prompt, config.aiTimeoutMs));
|
|
170
|
+
const wait = io?.wait ?? (() => ((config.waitForLimit ?? true) ? waitForAi(config) : Promise.resolve(false)));
|
|
159
171
|
const out = [];
|
|
160
172
|
const queue = [...groups];
|
|
173
|
+
let waiting = null;
|
|
174
|
+
let gaveUp = false; // one failed wait is enough; the rest are recorded as unproven
|
|
161
175
|
const worker = async () => {
|
|
162
176
|
for (let g = queue.shift(); g; g = queue.shift()) {
|
|
163
177
|
const files = new Set(g.tests.map((t) => t.file));
|
|
164
178
|
const siblings = passes.filter((p) => files.has(p.file));
|
|
165
179
|
let raw = '';
|
|
166
180
|
try {
|
|
167
|
-
raw = await
|
|
181
|
+
raw = await ask(buildDefectReviewPrompt(journey, g, siblings, decisions));
|
|
182
|
+
noteAiSuccess();
|
|
168
183
|
}
|
|
169
|
-
catch {
|
|
184
|
+
catch (e) {
|
|
185
|
+
noteAiFailure(e);
|
|
186
|
+
if (aiUnavailableReason() && !gaveUp) {
|
|
187
|
+
waiting ??= wait().finally(() => (waiting = null));
|
|
188
|
+
if (await waiting) {
|
|
189
|
+
queue.unshift(g);
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
gaveUp = true;
|
|
193
|
+
}
|
|
170
194
|
// an unanswered review proves nothing
|
|
171
195
|
}
|
|
172
196
|
out.push(...splitReview(g, parseReview(raw)));
|
|
@@ -408,8 +408,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
408
408
|
});
|
|
409
409
|
// A freshly written file also may not hold placeholders; a repair is held to
|
|
410
410
|
// the repair guard instead (it may not add a skip), so old files stay repairable.
|
|
411
|
-
const genGate = (content) => testGate(content) ??
|
|
412
|
-
checkTestGates(content, ext, { framework, forbidPlaceholders: true });
|
|
411
|
+
const genGate = (content, filePath) => testGate(content) ??
|
|
412
|
+
checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath });
|
|
413
413
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
414
414
|
// to retry (a resume pass fills those in without touching real files).
|
|
415
415
|
if (!force &&
|
|
@@ -517,7 +517,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
517
517
|
let status;
|
|
518
518
|
if (useAI) {
|
|
519
519
|
const facts = await grounding();
|
|
520
|
-
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, genGate);
|
|
520
|
+
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, (c) => genGate(c, filePath));
|
|
521
521
|
if (ai !== null) {
|
|
522
522
|
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
523
523
|
status = 'ai';
|
|
@@ -65,6 +65,18 @@ export async function verifyJourney(ctx) {
|
|
|
65
65
|
let outcome = await run();
|
|
66
66
|
for (let attempt = 0;; attempt += 1) {
|
|
67
67
|
if (!usable(outcome)) {
|
|
68
|
+
// The file itself does not load (a bad import, a syntax the runner
|
|
69
|
+
// rejects): that is the file's fault, so it is repaired like a
|
|
70
|
+
// failing test. Only an environment problem (infra) is left alone.
|
|
71
|
+
const loadError = !outcome.ran && !outcome.infra;
|
|
72
|
+
if (loadError && attempt < ctx.attempts) {
|
|
73
|
+
console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
74
|
+
const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
|
|
75
|
+
if (await repairOnce(target, [issue], [])) {
|
|
76
|
+
outcome = await run();
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
68
80
|
notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
|
|
69
81
|
break;
|
|
70
82
|
}
|
package/dist/core/run/report.js
CHANGED
|
@@ -69,7 +69,7 @@ export function mapPlaywright(json, category = 'web') {
|
|
|
69
69
|
const rootDir = (json.config && json.config.rootDir) || process.cwd();
|
|
70
70
|
const walk = (suite) => {
|
|
71
71
|
for (const spec of suite.specs || []) {
|
|
72
|
-
const name =
|
|
72
|
+
const name = shared.suiteName(spec.file || suite.file || 'suite');
|
|
73
73
|
const su = (suites[name] ||= { name, category, tests: [] });
|
|
74
74
|
for (const t of spec.tests || []) {
|
|
75
75
|
const r = (t.results || [])[(t.results || []).length - 1] || {};
|
|
@@ -78,11 +78,20 @@ export function mapPlaywright(json, category = 'web') {
|
|
|
78
78
|
const raw = r.error && r.error.message;
|
|
79
79
|
const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
|
|
80
80
|
const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
|
|
81
|
+
const err = r.error || {};
|
|
82
|
+
const detail = status === 'failed'
|
|
83
|
+
? shared.failureDetail(raw, {
|
|
84
|
+
file: err.location && err.location.file, line: err.location && err.location.line, column: err.location && err.location.column,
|
|
85
|
+
snippet: err.snippet, timeoutMs: undefined, browser: t.projectName,
|
|
86
|
+
attachments: (r.attachments || []).map((a) => ({ name: a.name, path: a.path })), root: shared.findRoot(rootDir),
|
|
87
|
+
})
|
|
88
|
+
: undefined;
|
|
81
89
|
su.tests.push({
|
|
82
90
|
name: spec.title,
|
|
83
91
|
status,
|
|
84
92
|
checks: shared.testChecks(loc, spec.line),
|
|
85
93
|
...(status === 'failed' ? shared.expectation(raw) : {}),
|
|
94
|
+
...(detail ? { detail } : {}),
|
|
86
95
|
error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
|
|
87
96
|
sig: status === 'failed' ? shared.sig(raw) : undefined,
|
|
88
97
|
note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
|
|
@@ -97,12 +106,10 @@ export function mapPlaywright(json, category = 'web') {
|
|
|
97
106
|
return { suites: Object.values(suites), durationSec: Math.round(((json.stats && json.stats.duration) || 0) / 1000) };
|
|
98
107
|
}
|
|
99
108
|
/** Map a Jest `--json` report to the dashboard run contract. */
|
|
100
|
-
export function mapJest(json, category = 'api') {
|
|
109
|
+
export function mapJest(json, category = 'api', testDir) {
|
|
101
110
|
const suites = {};
|
|
102
111
|
for (const f of json.testResults || []) {
|
|
103
|
-
const name =
|
|
104
|
-
.split('/').pop()
|
|
105
|
-
.replace(/\.(test|spec)\.[jt]s$/, '');
|
|
112
|
+
const name = shared.suiteName(f.name || f.testFilePath || 'suite', testDir);
|
|
106
113
|
const su = (suites[name] ||= { name, category, tests: [] });
|
|
107
114
|
for (const a of f.assertionResults || []) {
|
|
108
115
|
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
|
|
@@ -123,7 +130,7 @@ export async function publishReport(opts) {
|
|
|
123
130
|
const json = JSON.parse(fs.readFileSync(opts.resultsPath, 'utf8'));
|
|
124
131
|
// Jest's report has testResults; Playwright's has suites.
|
|
125
132
|
const { suites, durationSec } = Array.isArray(json.testResults)
|
|
126
|
-
? mapJest(json, opts.category || 'api')
|
|
133
|
+
? mapJest(json, opts.category || 'api', shared.testRoot(shared.findRoot(process.cwd()), opts.category || 'api'))
|
|
127
134
|
: mapPlaywright(json, opts.category || 'web');
|
|
128
135
|
const body = { project: opts.project, env: opts.env || process.env.TESTFORGE_ENV || 'local', durationSec, suites };
|
|
129
136
|
const res = await fetch(opts.dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
|
|
@@ -187,7 +194,7 @@ function attachPending(dir, ext, suites) {
|
|
|
187
194
|
}
|
|
188
195
|
const byName = new Map();
|
|
189
196
|
for (const f of files)
|
|
190
|
-
byName.set(suiteName(f), f);
|
|
197
|
+
byName.set(suiteName(f, dir), f);
|
|
191
198
|
for (const s of suites) {
|
|
192
199
|
const f = byName.get(s.name);
|
|
193
200
|
if (!f || s.pending)
|
|
@@ -206,14 +213,14 @@ function walk(dir, ext, out) {
|
|
|
206
213
|
out.push(full);
|
|
207
214
|
}
|
|
208
215
|
}
|
|
209
|
-
const suiteName = (file) =>
|
|
216
|
+
const suiteName = (file, dir) => shared.suiteName(file, dir);
|
|
210
217
|
const staticCount = (file) => (fs.readFileSync(file, 'utf8').match(/\b(it|test)\s*\(/g) || []).length;
|
|
211
218
|
/** Static fallback: count `it(`/`test(` occurrences per file (undercounts parameterized tests). */
|
|
212
219
|
function jestStatic(dir, ext, category, out) {
|
|
213
220
|
const files = [];
|
|
214
221
|
walk(dir, ext, files);
|
|
215
222
|
for (const f of files)
|
|
216
|
-
out.push({ name: suiteName(f), category, total: staticCount(f) });
|
|
223
|
+
out.push({ name: suiteName(f, dir), category, total: staticCount(f) });
|
|
217
224
|
}
|
|
218
225
|
// setupFilesAfterEnv shim: runs after jest installs the test globals, replaces
|
|
219
226
|
// every test BODY and before/after hook with a no-op. Collection still runs
|
|
@@ -282,13 +289,13 @@ function jestCount(dir, ext, category, out) {
|
|
|
282
289
|
const j = JSON.parse(fs.readFileSync(outJson, 'utf8'));
|
|
283
290
|
const seen = new Set();
|
|
284
291
|
for (const f of j.testResults || []) {
|
|
285
|
-
const n = suiteName(String(f.name || f.testFilePath || 'suite'));
|
|
292
|
+
const n = suiteName(String(f.name || f.testFilePath || 'suite'), dir);
|
|
286
293
|
seen.add(n);
|
|
287
294
|
out.push({ name: n, category, total: (f.assertionResults || []).length });
|
|
288
295
|
}
|
|
289
296
|
// any file jest couldn't even load → static, and say so loudly
|
|
290
297
|
for (const f of files) {
|
|
291
|
-
const n = suiteName(f);
|
|
298
|
+
const n = suiteName(f, dir);
|
|
292
299
|
if (seen.has(n))
|
|
293
300
|
continue;
|
|
294
301
|
const total = staticCount(f);
|
|
@@ -315,7 +322,7 @@ function jestCount(dir, ext, category, out) {
|
|
|
315
322
|
* EXPANDS parameterized tests (loops / .each), so we get the real case count
|
|
316
323
|
* per file. Returns false if it couldn't run (caller falls back to static).
|
|
317
324
|
*/
|
|
318
|
-
function playwrightList(pkgDir, category, out) {
|
|
325
|
+
function playwrightList(pkgDir, dir, category, out) {
|
|
319
326
|
try {
|
|
320
327
|
// `--reporter=list` so the project's own reporters — ours included — do
|
|
321
328
|
// not run during a count. Listing publishes nothing on its own, but a
|
|
@@ -326,10 +333,12 @@ function playwrightList(pkgDir, category, out) {
|
|
|
326
333
|
const counts = {};
|
|
327
334
|
for (const line of res.split('\n')) {
|
|
328
335
|
const m = line.match(/([^\s›»]+\.(?:test|spec)\.[jt]sx?):\d+:\d+/);
|
|
329
|
-
if (m)
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
336
|
+
if (!m)
|
|
337
|
+
continue;
|
|
338
|
+
// Playwright lists files relative to testDir; older versions relative to the config dir.
|
|
339
|
+
const abs = [path.resolve(dir, m[1]), path.resolve(pkgDir, m[1])].find((f) => fs.existsSync(f)) ?? path.resolve(dir, m[1]);
|
|
340
|
+
const n = suiteName(abs, dir);
|
|
341
|
+
counts[n] = (counts[n] || 0) + 1;
|
|
333
342
|
}
|
|
334
343
|
const names = Object.keys(counts);
|
|
335
344
|
if (!names.length)
|
|
@@ -356,7 +365,7 @@ export function scanInventory(cwd, testOutput, ext, framework = {}) {
|
|
|
356
365
|
if (framework[category] === 'playwright') {
|
|
357
366
|
// The config sits at the package root (or, beside a project's own
|
|
358
367
|
// config, in the tests' parent folder) — list from wherever it is.
|
|
359
|
-
if (playwrightList(packageRoot(dir), category, suites) || playwrightList(path.dirname(dir), category, suites)) {
|
|
368
|
+
if (playwrightList(packageRoot(dir), dir, category, suites) || playwrightList(path.dirname(dir), dir, category, suites)) {
|
|
360
369
|
attachPending(dir, ext, suites);
|
|
361
370
|
continue;
|
|
362
371
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.2",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -118,21 +118,156 @@ function testChecks(file, line) {
|
|
|
118
118
|
return uniq.length ? uniq.join('\n').slice(0, 1200) : undefined;
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
-
//
|
|
122
|
-
//
|
|
123
|
-
//
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
121
|
+
// ---- Failure detail -------------------------------------------------------
|
|
122
|
+
// The runner's message holds four different things that a report must keep
|
|
123
|
+
// apart: the test's OWN message (`expect(x, 'a visible reason under Password')`
|
|
124
|
+
// — what the check wanted, in the test's words), the matcher and its
|
|
125
|
+
// Expected/Received values, the locator and what it resolved to (the
|
|
126
|
+
// evidence of the element's real state), and the call log. The old mapping
|
|
127
|
+
// put the test's own message in "actual", so a report read "got: a visible
|
|
128
|
+
// reason under Password" — the expectation, dressed up as the observation.
|
|
129
|
+
|
|
130
|
+
const STATE_MATCHERS = /^toBe(Visible|Hidden|Enabled|Disabled|Checked|Focused|Editable|Empty|Attached|InViewport)$/;
|
|
131
|
+
const SUBJECT_OF = {
|
|
132
|
+
toHaveText: 'Text', toContainText: 'Text', toHaveURL: 'Page URL', toHaveTitle: 'Page title',
|
|
133
|
+
toHaveCount: 'Element count', toHaveValue: 'Field value', toHaveValues: 'Field values',
|
|
134
|
+
toHaveAttribute: 'Attribute', toHaveClass: 'Class', toHaveCSS: 'Style', toHaveId: 'Id',
|
|
135
|
+
toHaveScreenshot: 'Screenshot', toBeOK: 'Response', toHaveLength: 'Length',
|
|
136
|
+
};
|
|
137
|
+
const KIND_LABEL = {
|
|
138
|
+
hidden: 'Element hidden', 'not-found': 'Element not found', ambiguous: 'Ambiguous selector',
|
|
139
|
+
timeout: 'Test timeout', state: 'Wrong element state', mismatch: 'Value mismatch',
|
|
140
|
+
assertion: 'Condition false', error: 'Error thrown',
|
|
141
|
+
};
|
|
142
|
+
const short = (v, n = 90) => { v = String(v == null ? '' : v).trim(); return v.length > n ? v.slice(0, n - 1) + '…' : v; };
|
|
143
|
+
const cap = (v) => (v ? v[0].toUpperCase() + v.slice(1) : v);
|
|
144
|
+
const secs = (ms) => (ms % 1000 === 0 ? ms / 1000 + 's' : ms + 'ms');
|
|
145
|
+
const relTo = (root, p) => (p && root && p.startsWith(root + path.sep) ? p.slice(root.length + 1) : p);
|
|
146
|
+
|
|
147
|
+
// Inner text of the element Playwright printed ("<span class="x">Password format is incorrect.</span>").
|
|
148
|
+
function elementText(html) {
|
|
149
|
+
const m = (html || '').match(/^<([a-zA-Z0-9-]+)[^>]*>([\s\S]*?)<\/\1>\s*$/) || (html || '').match(/^<([a-zA-Z0-9-]+)[^>]*>([\s\S]*)$/);
|
|
150
|
+
if (!m) return { tag: ((html || '').match(/^<([a-zA-Z0-9-]+)/) || [])[1] };
|
|
151
|
+
const text = m[2].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim().replace(/…$/, '');
|
|
152
|
+
return { tag: m[1], text: text || undefined };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Structured detail of one failure, from the runner's message plus what the
|
|
157
|
+
* reporter knows (location, browser, timeout, artifacts). Every field comes
|
|
158
|
+
* from evidence; nothing is inferred from the wording of a message.
|
|
159
|
+
*
|
|
160
|
+
* ctx: { file, line, column, snippet, browser, timeoutMs, attachments: [{name, path}], root }
|
|
161
|
+
*/
|
|
162
|
+
function failureDetail(raw, ctx) {
|
|
163
|
+
ctx = ctx || {};
|
|
164
|
+
const s = strip(raw || '').replace(/\r/g, '');
|
|
165
|
+
if (!s.trim()) return undefined;
|
|
166
|
+
const lines = s.split('\n');
|
|
127
167
|
const pick = (re) => { const m = s.match(re); return m ? m[1].trim() : undefined; };
|
|
168
|
+
const first = (lines.find((l) => l.trim()) || '').trim().replace(/^Error:\s*/, '');
|
|
169
|
+
|
|
170
|
+
const assertionLine = lines.map((l) => l.trim()).find((l) => /^(Error:\s*)?expect\(.*\)\.\w+\(/.test(l));
|
|
171
|
+
const assertion = assertionLine ? assertionLine.replace(/^Error:\s*/, '').replace(/\s*(failed|\/\/.*)$/, '').trim() : undefined;
|
|
172
|
+
const matcher = assertion ? (assertion.match(/\.(\w+)\(/) || [])[1] : undefined;
|
|
173
|
+
const locator = pick(/^\s*Locator:\s*(.+)$/m);
|
|
128
174
|
const expected = pick(/^\s*Expected(?: [a-z ]+)?:\s*(.+)$/m);
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
175
|
+
const received = pick(/^\s*Received(?: [a-z ]+)?:\s*(.+)$/m);
|
|
176
|
+
const testTimeout = s.match(/Test timeout of (\d+) ?ms exceeded/i);
|
|
177
|
+
const timeoutMs = testTimeout ? Number(testTimeout[1])
|
|
178
|
+
: Number(pick(/^\s*Timeout:?\s*(\d+)\s*ms/m) || (s.match(/Timeout (\d+) ?ms exceeded/) || [])[1] || ctx.timeoutMs || 0) || undefined;
|
|
179
|
+
const resolved = s.match(/(\d+) × locator resolved to (.+)$/m);
|
|
180
|
+
const strict = s.match(/strict mode violation: (.+?) resolved to (\d+) elements/);
|
|
181
|
+
const callLog = (s.split(/\n\s*Call log:\s*\n/i)[1] || '').replace(/\n\s*at\s.*$/gm, '').trim() || undefined;
|
|
182
|
+
|
|
183
|
+
// The test's own message: the first line when it is not the runner's own wording.
|
|
184
|
+
const runnerWording = /^(expect[.(]|Test timeout|Timeout|locator\.|page\.|strict mode violation|element\(s\) not found|Target (page|context|browser)|Error: )/i;
|
|
185
|
+
const subject = first && first !== assertion && !runnerWording.test(first) ? first : undefined;
|
|
186
|
+
const element = resolved ? elementText(resolved[2]) : {};
|
|
187
|
+
|
|
188
|
+
let kind;
|
|
189
|
+
if (testTimeout) kind = 'timeout';
|
|
190
|
+
else if (strict) kind = 'ambiguous';
|
|
191
|
+
else if (/element\(s\) not found/.test(s) || (locator && /waiting for/.test(s) && !resolved && matcher && STATE_MATCHERS.test(matcher))) kind = 'not-found';
|
|
192
|
+
else if (matcher === 'toBeVisible' && received === 'hidden') kind = 'hidden';
|
|
193
|
+
else if (matcher && STATE_MATCHERS.test(matcher)) kind = 'state';
|
|
194
|
+
else if (/^(true|false)$/.test(expected || '') && /^(true|false)$/.test(received || '')) kind = 'assertion';
|
|
195
|
+
else if (matcher && (expected !== undefined || received !== undefined)) kind = 'mismatch';
|
|
196
|
+
else if (matcher) kind = 'assertion';
|
|
197
|
+
else kind = 'error';
|
|
198
|
+
|
|
199
|
+
const what = subject || (locator ? `the element ${short(locator, 60)}` : undefined);
|
|
200
|
+
const waited = timeoutMs ? ` Waited ${secs(timeoutMs)}.` : '';
|
|
201
|
+
let headline, explain, exp, act;
|
|
202
|
+
switch (kind) {
|
|
203
|
+
case 'timeout':
|
|
204
|
+
headline = `Test timed out after ${secs(timeoutMs)}`;
|
|
205
|
+
explain = `The runner stopped the test after ${secs(timeoutMs)}, before it finished.`;
|
|
206
|
+
act = `the test did not finish within ${secs(timeoutMs)}`;
|
|
207
|
+
break;
|
|
208
|
+
case 'ambiguous':
|
|
209
|
+
headline = `${cap(subject || `Selector ${short(strict[1], 60)}`)} matched ${strict[2]} elements`;
|
|
210
|
+
explain = `The selector matched ${strict[2]} elements; Playwright strict mode needs exactly one.`;
|
|
211
|
+
exp = 'exactly one matching element'; act = `${strict[2]} elements matched`;
|
|
212
|
+
break;
|
|
213
|
+
case 'not-found':
|
|
214
|
+
headline = `${cap(what || 'The element')} was not found on the page`;
|
|
215
|
+
explain = `No element matched the selector: either it was never rendered, or the selector does not match the page.${waited}`;
|
|
216
|
+
exp = plain(expected) || 'present on the page'; act = 'not found on the page';
|
|
217
|
+
break;
|
|
218
|
+
case 'hidden':
|
|
219
|
+
headline = `${cap(what || 'The element')} is on the page but hidden`;
|
|
220
|
+
explain = `The element exists${element.text ? ` and reads "${short(element.text, 120)}"` : ''}, but it is not displayed, so the customer cannot see it.${waited}`;
|
|
221
|
+
exp = 'shown on screen'; act = `on the page but hidden${element.text ? ` ("${short(element.text, 80)}")` : ''}`;
|
|
222
|
+
break;
|
|
223
|
+
case 'state':
|
|
224
|
+
headline = `${cap(what || 'The element')}: expected ${plain(expected)}, got ${plain(received)}`;
|
|
225
|
+
explain = `Expected the element to be ${plain(expected)}; it was ${plain(received)}.${waited}`;
|
|
226
|
+
exp = plain(expected); act = plain(received);
|
|
227
|
+
break;
|
|
228
|
+
case 'mismatch': {
|
|
229
|
+
const noun = subject || SUBJECT_OF[matcher] || 'Value';
|
|
230
|
+
const e = expected === 'predicate to succeed' ? "a value matching the test's pattern" : expected;
|
|
231
|
+
headline = `${cap(noun)}: expected ${short(e, 60)}, got ${short(received, 60)}`;
|
|
232
|
+
explain = `Expected ${e}; the app gave ${received}.${waited}`;
|
|
233
|
+
exp = e; act = received;
|
|
234
|
+
break;
|
|
235
|
+
}
|
|
236
|
+
case 'assertion':
|
|
237
|
+
headline = subject ? cap(subject) : `Check failed: ${assertion}`;
|
|
238
|
+
explain = 'The test\'s condition was false. The runner recorded no element or value for it, so the exact on-screen state was not captured.';
|
|
239
|
+
exp = 'the check to pass'; act = 'not captured (the condition was false)';
|
|
240
|
+
break;
|
|
241
|
+
default:
|
|
242
|
+
headline = short(first, 140);
|
|
243
|
+
explain = 'The test stopped with this error; no expected or actual value was recorded.';
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const root = ctx.root;
|
|
247
|
+
const d = {
|
|
248
|
+
kind, kindLabel: KIND_LABEL[kind], headline, explain,
|
|
249
|
+
subject, assertion, matcher, locator,
|
|
250
|
+
expected: exp, actual: act, rawExpected: expected, rawReceived: received,
|
|
251
|
+
element: element.tag ? { tag: element.tag, text: element.text, html: short(resolved[2], 300), count: Number(resolved[1]) } : undefined,
|
|
252
|
+
timeoutMs, callLog: callLog && callLog.slice(0, 3000), message: s.slice(0, 4000),
|
|
253
|
+
file: relTo(root, ctx.file), line: ctx.line, column: ctx.column, snippet: ctx.snippet && strip(ctx.snippet).slice(0, 1200),
|
|
254
|
+
browser: ctx.browser || undefined,
|
|
255
|
+
artifacts: (ctx.attachments || []).filter((a) => a && a.path).map((a) => ({ name: a.name, path: relTo(root, a.path) })),
|
|
256
|
+
};
|
|
257
|
+
for (const k of Object.keys(d)) if (d[k] === undefined || (Array.isArray(d[k]) && !d[k].length)) delete d[k];
|
|
258
|
+
return d;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// For a failed check: what it wanted and what it got, as two short strings.
|
|
262
|
+
// Kept for the older report fields; the values come from failureDetail, so
|
|
263
|
+
// "actual" is always the observed state, never the test's own message.
|
|
264
|
+
function expectation(raw) {
|
|
265
|
+
const d = failureDetail(raw);
|
|
266
|
+
if (!d) return {};
|
|
267
|
+
const out = {};
|
|
268
|
+
if (d.expected) out.expected = d.expected;
|
|
269
|
+
if (d.actual) out.actual = d.actual;
|
|
270
|
+
return out;
|
|
136
271
|
}
|
|
137
272
|
const PLAIN = { visible: 'shown on screen', hidden: 'not shown on screen (hidden)', enabled: 'enabled', disabled: 'disabled' };
|
|
138
273
|
const plain = (v) => (v && PLAIN[v]) || v;
|
|
@@ -217,4 +352,25 @@ async function publish(category, suites, durationSec) {
|
|
|
217
352
|
}
|
|
218
353
|
}
|
|
219
354
|
|
|
220
|
-
|
|
355
|
+
// A suite is named by its path under the test root — "booking-summary/keyboard-access" —
|
|
356
|
+
// because several journeys may each own a keyboard-access.test.ts. Keyed by basename
|
|
357
|
+
// alone, three such files became one suite of 22 and a single-journey run showed 13
|
|
358
|
+
// phantom skips. A relative path is kept as given (Playwright's JSON already reports
|
|
359
|
+
// spec files relative to testDir); an absolute path outside the root falls back to the
|
|
360
|
+
// bare file name.
|
|
361
|
+
function suiteName(file, root) {
|
|
362
|
+
let rel = String(file || 'suite').replace(/\\/g, '/');
|
|
363
|
+
if (path.isAbsolute(rel)) {
|
|
364
|
+
const r = root ? path.relative(root, rel).replace(/\\/g, '/') : '';
|
|
365
|
+
rel = r && !r.startsWith('..') ? r : rel.split('/').pop();
|
|
366
|
+
}
|
|
367
|
+
return rel.replace(/\.(test|spec)\.[jt]sx?$/, '');
|
|
368
|
+
}
|
|
369
|
+
// The configured test folder for a category (testOutput in testforge.config.json), absolute.
|
|
370
|
+
function testRoot(root, category) {
|
|
371
|
+
const out = loadConfig(root).testOutput;
|
|
372
|
+
const d = out && out[category];
|
|
373
|
+
return d ? path.resolve(root, d) : undefined;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot };
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to jest.config.js:
|
|
3
3
|
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgeJestReporter {
|
|
7
7
|
constructor(_globalConfig, options) {
|
|
8
8
|
this.category = (options && options.category) || 'api';
|
|
9
|
+
this.testRoot = testRoot(findRoot(process.cwd()), this.category);
|
|
9
10
|
}
|
|
10
11
|
async onRunComplete(_contexts, results) {
|
|
11
12
|
const suites = {};
|
|
12
13
|
for (const f of results.testResults || []) {
|
|
13
|
-
const name =
|
|
14
|
+
const name = suiteName(f.testFilePath || f.name || 'suite', this.testRoot);
|
|
14
15
|
const su = suites[name] || (suites[name] = { name, category: this.category, tests: [] });
|
|
15
16
|
for (const a of (f.testResults || f.assertionResults || [])) {
|
|
16
17
|
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
|
|
@@ -25,6 +26,7 @@ module.exports = class TestForgeJestReporter {
|
|
|
25
26
|
: undefined,
|
|
26
27
|
checks: testChecks(f.testFilePath, a.location && a.location.line),
|
|
27
28
|
...(status === 'failed' ? expectation(raw) : {}),
|
|
29
|
+
detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
|
|
28
30
|
});
|
|
29
31
|
}
|
|
30
32
|
}
|
|
@@ -1,17 +1,38 @@
|
|
|
1
1
|
// TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to playwright.config.js reporter array:
|
|
3
3
|
// ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
|
|
4
|
-
const { cleanError, sig, publish, testChecks,
|
|
4
|
+
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgePlaywrightReporter {
|
|
7
7
|
constructor(options) {
|
|
8
8
|
this.category = (options && options.category) || 'web';
|
|
9
9
|
this.suites = {};
|
|
10
10
|
this.start = Date.now();
|
|
11
|
+
this.root = findRoot(process.cwd());
|
|
12
|
+
this.testRoot = testRoot(this.root, this.category);
|
|
13
|
+
}
|
|
14
|
+
onBegin(config) {
|
|
15
|
+
// Playwright's own testDir is the authority when it has one.
|
|
16
|
+
const td = config && config.projects && config.projects[0] && config.projects[0].testDir;
|
|
17
|
+
if (td) this.testRoot = td;
|
|
18
|
+
}
|
|
19
|
+
// Everything the runner knows about a failure beyond its message: where it
|
|
20
|
+
// happened, the browser, the test's timeout and the artifacts it left.
|
|
21
|
+
detail(test, result) {
|
|
22
|
+
const err = result.error || {};
|
|
23
|
+
const loc = err.location || test.location || {};
|
|
24
|
+
const project = test.parent && typeof test.parent.project === 'function' ? test.parent.project() : undefined;
|
|
25
|
+
const d = failureDetail(err.message, {
|
|
26
|
+
file: loc.file, line: loc.line, column: loc.column, snippet: err.snippet,
|
|
27
|
+
browser: project && project.name, timeoutMs: test.timeout,
|
|
28
|
+
attachments: (result.attachments || []).map((a) => ({ name: a.name, path: a.path })),
|
|
29
|
+
root: this.root,
|
|
30
|
+
});
|
|
31
|
+
return d || undefined;
|
|
11
32
|
}
|
|
12
33
|
onTestEnd(test, result) {
|
|
13
34
|
const file = (test.location && test.location.file) || 'suite';
|
|
14
|
-
const name =
|
|
35
|
+
const name = suiteName(file, this.testRoot);
|
|
15
36
|
const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
|
|
16
37
|
const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
|
|
17
38
|
const raw = result.error && result.error.message;
|
|
@@ -25,9 +46,17 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
25
46
|
note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
|
|
26
47
|
checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
|
|
27
48
|
|| testChecks(test.location && test.location.file, test.location && test.location.line),
|
|
28
|
-
...(status === 'failed' ?
|
|
49
|
+
...(status === 'failed' ? this.failed(test, result) : {}),
|
|
29
50
|
});
|
|
30
51
|
}
|
|
52
|
+
failed(test, result) {
|
|
53
|
+
const detail = this.detail(test, result);
|
|
54
|
+
if (!detail) return {};
|
|
55
|
+
const out = { detail };
|
|
56
|
+
if (detail.expected) out.expected = detail.expected;
|
|
57
|
+
if (detail.actual) out.actual = detail.actual;
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
31
60
|
async onEnd(result) {
|
|
32
61
|
const durationSec = Math.round(((result && result.duration) || (Date.now() - this.start)) / 1000);
|
|
33
62
|
const suites = Object.values(this.suites);
|