@activate-core/test-forge 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/gates/validate.js +25 -2
- package/dist/core/generate/defect-review.js +28 -4
- package/dist/core/generate/run-generate.js +3 -3
- package/dist/core/generate/verify-journey.js +12 -0
- package/dist/core/run/report.js +9 -0
- package/package.json +1 -1
- package/reporters/_shared.cjs +149 -14
- package/reporters/jest-reporter.cjs +2 -1
- package/reporters/playwright-reporter.cjs +25 -2
|
@@ -17,8 +17,8 @@
|
|
|
17
17
|
* false-rejecting valid TypeScript.
|
|
18
18
|
*/
|
|
19
19
|
import { execFileSync } from 'node:child_process';
|
|
20
|
-
import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
|
21
|
-
import { join } from 'node:path';
|
|
20
|
+
import { writeFileSync, mkdtempSync, rmSync, existsSync } from 'node:fs';
|
|
21
|
+
import { join, dirname, resolve } from 'node:path';
|
|
22
22
|
import { tmpdir } from 'node:os';
|
|
23
23
|
import { reasonlessPending, pendingDeclarations } from '../generate/pending.js';
|
|
24
24
|
/**
|
|
@@ -169,6 +169,13 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
169
169
|
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
170
170
|
return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
|
|
171
171
|
}
|
|
172
|
+
// F17 — relative imports resolve from where the file is written.
|
|
173
|
+
if (ctx.filePath) {
|
|
174
|
+
const missing = unresolvedImports(content, ctx.filePath);
|
|
175
|
+
if (missing.length) {
|
|
176
|
+
return `relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
172
179
|
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
173
180
|
// as a bare "fixme", indistinguishable from the tool giving up. Runs on
|
|
174
181
|
// content normalisePending() has already given every reason it could find.
|
|
@@ -386,3 +393,19 @@ function looksStructurallyClean(content) {
|
|
|
386
393
|
count(content, '(') === count(content, ')');
|
|
387
394
|
return balanced;
|
|
388
395
|
}
|
|
396
|
+
const RESOLVE_EXTS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json'];
|
|
397
|
+
/** Relative module specifiers (import / require / dynamic import) that reach no file from `filePath`. */
|
|
398
|
+
export function unresolvedImports(content, filePath) {
|
|
399
|
+
const specs = new Set();
|
|
400
|
+
const re = /(?:\bfrom\s*|\brequire\s*\(\s*|\bimport\s*\(\s*|^\s*import\s+)(['"])(\.{1,2}\/[^'"]*)\1/gm;
|
|
401
|
+
for (const m of content.matchAll(re))
|
|
402
|
+
specs.add(m[2]);
|
|
403
|
+
const base = dirname(resolve(filePath));
|
|
404
|
+
return [...specs].filter((spec) => {
|
|
405
|
+
const target = resolve(base, spec);
|
|
406
|
+
// A compiled .js specifier may point at the .ts source.
|
|
407
|
+
const stem = target.replace(/\.(c|m)?js$/, '');
|
|
408
|
+
return !RESOLVE_EXTS.some((e) => existsSync(target + e) || existsSync(stem + e))
|
|
409
|
+
&& !RESOLVE_EXTS.slice(1).some((e) => existsSync(join(target, 'index' + e)));
|
|
410
|
+
});
|
|
411
|
+
}
|
|
@@ -20,6 +20,7 @@ import { existsSync, readFileSync } from 'node:fs';
|
|
|
20
20
|
import { relative } from 'node:path';
|
|
21
21
|
import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
|
|
22
22
|
import { runWithStdin } from './ai-generator.js';
|
|
23
|
+
import { aiUnavailableReason, noteAiFailure, noteAiSuccess, waitForAi } from './ai-session.js';
|
|
23
24
|
const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'unproven'];
|
|
24
25
|
/**
|
|
25
26
|
* Group failing tests by the claim marked above them. A parametrised test
|
|
@@ -149,24 +150,47 @@ export function splitReview(group, parsed) {
|
|
|
149
150
|
out.unshift({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence });
|
|
150
151
|
return out;
|
|
151
152
|
}
|
|
152
|
-
/**
|
|
153
|
-
|
|
153
|
+
/**
|
|
154
|
+
* Review every claim group, a few at a time. Never throws.
|
|
155
|
+
*
|
|
156
|
+
* A review that failed because the AI is gone (usage limit, logged out) is
|
|
157
|
+
* not a verdict. On the CleanMe booking-summary run the limit landed exactly
|
|
158
|
+
* here: all 18 claims came back "unproven — the review could not be read",
|
|
159
|
+
* the journey-level wait only noticed at the next AI call, and the summary
|
|
160
|
+
* reported 0 defects. So the group goes back on the queue, every worker
|
|
161
|
+
* shares one wait, and the review is asked again once the AI answers.
|
|
162
|
+
*/
|
|
163
|
+
export async function reviewDefects(journey, groups, passes, io) {
|
|
154
164
|
const config = loadConfig();
|
|
155
165
|
const decisions = decisionsSource(config.decisions);
|
|
156
166
|
// A linked list (an internal Notion page) needs the reference tools to be read.
|
|
157
167
|
const tools = decisions.link ? withReferenceTools : withReadOnlyTools;
|
|
158
168
|
const { program, args } = tools(splitCommand(config.aiCommand));
|
|
169
|
+
const ask = io?.ask ?? ((prompt) => runWithStdin(program, args, prompt, config.aiTimeoutMs));
|
|
170
|
+
const wait = io?.wait ?? (() => ((config.waitForLimit ?? true) ? waitForAi(config) : Promise.resolve(false)));
|
|
159
171
|
const out = [];
|
|
160
172
|
const queue = [...groups];
|
|
173
|
+
let waiting = null;
|
|
174
|
+
let gaveUp = false; // one failed wait is enough; the rest are recorded as unproven
|
|
161
175
|
const worker = async () => {
|
|
162
176
|
for (let g = queue.shift(); g; g = queue.shift()) {
|
|
163
177
|
const files = new Set(g.tests.map((t) => t.file));
|
|
164
178
|
const siblings = passes.filter((p) => files.has(p.file));
|
|
165
179
|
let raw = '';
|
|
166
180
|
try {
|
|
167
|
-
raw = await
|
|
181
|
+
raw = await ask(buildDefectReviewPrompt(journey, g, siblings, decisions));
|
|
182
|
+
noteAiSuccess();
|
|
168
183
|
}
|
|
169
|
-
catch {
|
|
184
|
+
catch (e) {
|
|
185
|
+
noteAiFailure(e);
|
|
186
|
+
if (aiUnavailableReason() && !gaveUp) {
|
|
187
|
+
waiting ??= wait().finally(() => (waiting = null));
|
|
188
|
+
if (await waiting) {
|
|
189
|
+
queue.unshift(g);
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
gaveUp = true;
|
|
193
|
+
}
|
|
170
194
|
// an unanswered review proves nothing
|
|
171
195
|
}
|
|
172
196
|
out.push(...splitReview(g, parseReview(raw)));
|
|
@@ -408,8 +408,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
408
408
|
});
|
|
409
409
|
// A freshly written file also may not hold placeholders; a repair is held to
|
|
410
410
|
// the repair guard instead (it may not add a skip), so old files stay repairable.
|
|
411
|
-
const genGate = (content) => testGate(content) ??
|
|
412
|
-
checkTestGates(content, ext, { framework, forbidPlaceholders: true });
|
|
411
|
+
const genGate = (content, filePath) => testGate(content) ??
|
|
412
|
+
checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath });
|
|
413
413
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
414
414
|
// to retry (a resume pass fills those in without touching real files).
|
|
415
415
|
if (!force &&
|
|
@@ -517,7 +517,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
517
517
|
let status;
|
|
518
518
|
if (useAI) {
|
|
519
519
|
const facts = await grounding();
|
|
520
|
-
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, genGate);
|
|
520
|
+
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, (c) => genGate(c, filePath));
|
|
521
521
|
if (ai !== null) {
|
|
522
522
|
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
523
523
|
status = 'ai';
|
|
@@ -65,6 +65,18 @@ export async function verifyJourney(ctx) {
|
|
|
65
65
|
let outcome = await run();
|
|
66
66
|
for (let attempt = 0;; attempt += 1) {
|
|
67
67
|
if (!usable(outcome)) {
|
|
68
|
+
// The file itself does not load (a bad import, a syntax the runner
|
|
69
|
+
// rejects): that is the file's fault, so it is repaired like a
|
|
70
|
+
// failing test. Only an environment problem (infra) is left alone.
|
|
71
|
+
const loadError = !outcome.ran && !outcome.infra;
|
|
72
|
+
if (loadError && attempt < ctx.attempts) {
|
|
73
|
+
console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
74
|
+
const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
|
|
75
|
+
if (await repairOnce(target, [issue], [])) {
|
|
76
|
+
outcome = await run();
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
68
80
|
notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
|
|
69
81
|
break;
|
|
70
82
|
}
|
package/dist/core/run/report.js
CHANGED
|
@@ -78,11 +78,20 @@ export function mapPlaywright(json, category = 'web') {
|
|
|
78
78
|
const raw = r.error && r.error.message;
|
|
79
79
|
const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
|
|
80
80
|
const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
|
|
81
|
+
const err = r.error || {};
|
|
82
|
+
const detail = status === 'failed'
|
|
83
|
+
? shared.failureDetail(raw, {
|
|
84
|
+
file: err.location && err.location.file, line: err.location && err.location.line, column: err.location && err.location.column,
|
|
85
|
+
snippet: err.snippet, timeoutMs: undefined, browser: t.projectName,
|
|
86
|
+
attachments: (r.attachments || []).map((a) => ({ name: a.name, path: a.path })), root: shared.findRoot(rootDir),
|
|
87
|
+
})
|
|
88
|
+
: undefined;
|
|
81
89
|
su.tests.push({
|
|
82
90
|
name: spec.title,
|
|
83
91
|
status,
|
|
84
92
|
checks: shared.testChecks(loc, spec.line),
|
|
85
93
|
...(status === 'failed' ? shared.expectation(raw) : {}),
|
|
94
|
+
...(detail ? { detail } : {}),
|
|
86
95
|
error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
|
|
87
96
|
sig: status === 'failed' ? shared.sig(raw) : undefined,
|
|
88
97
|
note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.1",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -118,21 +118,156 @@ function testChecks(file, line) {
|
|
|
118
118
|
return uniq.length ? uniq.join('\n').slice(0, 1200) : undefined;
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
-
//
|
|
122
|
-
//
|
|
123
|
-
//
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
121
|
+
// ---- Failure detail -------------------------------------------------------
|
|
122
|
+
// The runner's message holds four different things that a report must keep
|
|
123
|
+
// apart: the test's OWN message (`expect(x, 'a visible reason under Password')`
|
|
124
|
+
// — what the check wanted, in the test's words), the matcher and its
|
|
125
|
+
// Expected/Received values, the locator and what it resolved to (the
|
|
126
|
+
// evidence of the element's real state), and the call log. The old mapping
|
|
127
|
+
// put the test's own message in "actual", so a report read "got: a visible
|
|
128
|
+
// reason under Password" — the expectation, dressed up as the observation.
|
|
129
|
+
|
|
130
|
+
const STATE_MATCHERS = /^toBe(Visible|Hidden|Enabled|Disabled|Checked|Focused|Editable|Empty|Attached|InViewport)$/;
|
|
131
|
+
const SUBJECT_OF = {
|
|
132
|
+
toHaveText: 'Text', toContainText: 'Text', toHaveURL: 'Page URL', toHaveTitle: 'Page title',
|
|
133
|
+
toHaveCount: 'Element count', toHaveValue: 'Field value', toHaveValues: 'Field values',
|
|
134
|
+
toHaveAttribute: 'Attribute', toHaveClass: 'Class', toHaveCSS: 'Style', toHaveId: 'Id',
|
|
135
|
+
toHaveScreenshot: 'Screenshot', toBeOK: 'Response', toHaveLength: 'Length',
|
|
136
|
+
};
|
|
137
|
+
const KIND_LABEL = {
|
|
138
|
+
hidden: 'Element hidden', 'not-found': 'Element not found', ambiguous: 'Ambiguous selector',
|
|
139
|
+
timeout: 'Test timeout', state: 'Wrong element state', mismatch: 'Value mismatch',
|
|
140
|
+
assertion: 'Condition false', error: 'Error thrown',
|
|
141
|
+
};
|
|
142
|
+
const short = (v, n = 90) => { v = String(v == null ? '' : v).trim(); return v.length > n ? v.slice(0, n - 1) + '…' : v; };
|
|
143
|
+
const cap = (v) => (v ? v[0].toUpperCase() + v.slice(1) : v);
|
|
144
|
+
const secs = (ms) => (ms % 1000 === 0 ? ms / 1000 + 's' : ms + 'ms');
|
|
145
|
+
const relTo = (root, p) => (p && root && p.startsWith(root + path.sep) ? p.slice(root.length + 1) : p);
|
|
146
|
+
|
|
147
|
+
// Inner text of the element Playwright printed ("<span class="x">Password format is incorrect.</span>").
|
|
148
|
+
function elementText(html) {
|
|
149
|
+
const m = (html || '').match(/^<([a-zA-Z0-9-]+)[^>]*>([\s\S]*?)<\/\1>\s*$/) || (html || '').match(/^<([a-zA-Z0-9-]+)[^>]*>([\s\S]*)$/);
|
|
150
|
+
if (!m) return { tag: ((html || '').match(/^<([a-zA-Z0-9-]+)/) || [])[1] };
|
|
151
|
+
const text = m[2].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim().replace(/…$/, '');
|
|
152
|
+
return { tag: m[1], text: text || undefined };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Structured detail of one failure, from the runner's message plus what the
|
|
157
|
+
* reporter knows (location, browser, timeout, artifacts). Every field comes
|
|
158
|
+
* from evidence; nothing is inferred from the wording of a message.
|
|
159
|
+
*
|
|
160
|
+
* ctx: { file, line, column, snippet, browser, timeoutMs, attachments: [{name, path}], root }
|
|
161
|
+
*/
|
|
162
|
+
function failureDetail(raw, ctx) {
|
|
163
|
+
ctx = ctx || {};
|
|
164
|
+
const s = strip(raw || '').replace(/\r/g, '');
|
|
165
|
+
if (!s.trim()) return undefined;
|
|
166
|
+
const lines = s.split('\n');
|
|
127
167
|
const pick = (re) => { const m = s.match(re); return m ? m[1].trim() : undefined; };
|
|
168
|
+
const first = (lines.find((l) => l.trim()) || '').trim().replace(/^Error:\s*/, '');
|
|
169
|
+
|
|
170
|
+
const assertionLine = lines.map((l) => l.trim()).find((l) => /^(Error:\s*)?expect\(.*\)\.\w+\(/.test(l));
|
|
171
|
+
const assertion = assertionLine ? assertionLine.replace(/^Error:\s*/, '').replace(/\s*(failed|\/\/.*)$/, '').trim() : undefined;
|
|
172
|
+
const matcher = assertion ? (assertion.match(/\.(\w+)\(/) || [])[1] : undefined;
|
|
173
|
+
const locator = pick(/^\s*Locator:\s*(.+)$/m);
|
|
128
174
|
const expected = pick(/^\s*Expected(?: [a-z ]+)?:\s*(.+)$/m);
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
175
|
+
const received = pick(/^\s*Received(?: [a-z ]+)?:\s*(.+)$/m);
|
|
176
|
+
const testTimeout = s.match(/Test timeout of (\d+) ?ms exceeded/i);
|
|
177
|
+
const timeoutMs = testTimeout ? Number(testTimeout[1])
|
|
178
|
+
: Number(pick(/^\s*Timeout:?\s*(\d+)\s*ms/m) || (s.match(/Timeout (\d+) ?ms exceeded/) || [])[1] || ctx.timeoutMs || 0) || undefined;
|
|
179
|
+
const resolved = s.match(/(\d+) × locator resolved to (.+)$/m);
|
|
180
|
+
const strict = s.match(/strict mode violation: (.+?) resolved to (\d+) elements/);
|
|
181
|
+
const callLog = (s.split(/\n\s*Call log:\s*\n/i)[1] || '').replace(/\n\s*at\s.*$/gm, '').trim() || undefined;
|
|
182
|
+
|
|
183
|
+
// The test's own message: the first line when it is not the runner's own wording.
|
|
184
|
+
const runnerWording = /^(expect[.(]|Test timeout|Timeout|locator\.|page\.|strict mode violation|element\(s\) not found|Target (page|context|browser)|Error: )/i;
|
|
185
|
+
const subject = first && first !== assertion && !runnerWording.test(first) ? first : undefined;
|
|
186
|
+
const element = resolved ? elementText(resolved[2]) : {};
|
|
187
|
+
|
|
188
|
+
let kind;
|
|
189
|
+
if (testTimeout) kind = 'timeout';
|
|
190
|
+
else if (strict) kind = 'ambiguous';
|
|
191
|
+
else if (/element\(s\) not found/.test(s) || (locator && /waiting for/.test(s) && !resolved && matcher && STATE_MATCHERS.test(matcher))) kind = 'not-found';
|
|
192
|
+
else if (matcher === 'toBeVisible' && received === 'hidden') kind = 'hidden';
|
|
193
|
+
else if (matcher && STATE_MATCHERS.test(matcher)) kind = 'state';
|
|
194
|
+
else if (/^(true|false)$/.test(expected || '') && /^(true|false)$/.test(received || '')) kind = 'assertion';
|
|
195
|
+
else if (matcher && (expected !== undefined || received !== undefined)) kind = 'mismatch';
|
|
196
|
+
else if (matcher) kind = 'assertion';
|
|
197
|
+
else kind = 'error';
|
|
198
|
+
|
|
199
|
+
const what = subject || (locator ? `the element ${short(locator, 60)}` : undefined);
|
|
200
|
+
const waited = timeoutMs ? ` Waited ${secs(timeoutMs)}.` : '';
|
|
201
|
+
let headline, explain, exp, act;
|
|
202
|
+
switch (kind) {
|
|
203
|
+
case 'timeout':
|
|
204
|
+
headline = `Test timed out after ${secs(timeoutMs)}`;
|
|
205
|
+
explain = `The runner stopped the test after ${secs(timeoutMs)}, before it finished.`;
|
|
206
|
+
act = `the test did not finish within ${secs(timeoutMs)}`;
|
|
207
|
+
break;
|
|
208
|
+
case 'ambiguous':
|
|
209
|
+
headline = `${cap(subject || `Selector ${short(strict[1], 60)}`)} matched ${strict[2]} elements`;
|
|
210
|
+
explain = `The selector matched ${strict[2]} elements; Playwright strict mode needs exactly one.`;
|
|
211
|
+
exp = 'exactly one matching element'; act = `${strict[2]} elements matched`;
|
|
212
|
+
break;
|
|
213
|
+
case 'not-found':
|
|
214
|
+
headline = `${cap(what || 'The element')} was not found on the page`;
|
|
215
|
+
explain = `No element matched the selector: either it was never rendered, or the selector does not match the page.${waited}`;
|
|
216
|
+
exp = plain(expected) || 'present on the page'; act = 'not found on the page';
|
|
217
|
+
break;
|
|
218
|
+
case 'hidden':
|
|
219
|
+
headline = `${cap(what || 'The element')} is on the page but hidden`;
|
|
220
|
+
explain = `The element exists${element.text ? ` and reads "${short(element.text, 120)}"` : ''}, but it is not displayed, so the customer cannot see it.${waited}`;
|
|
221
|
+
exp = 'shown on screen'; act = `on the page but hidden${element.text ? ` ("${short(element.text, 80)}")` : ''}`;
|
|
222
|
+
break;
|
|
223
|
+
case 'state':
|
|
224
|
+
headline = `${cap(what || 'The element')}: expected ${plain(expected)}, got ${plain(received)}`;
|
|
225
|
+
explain = `Expected the element to be ${plain(expected)}; it was ${plain(received)}.${waited}`;
|
|
226
|
+
exp = plain(expected); act = plain(received);
|
|
227
|
+
break;
|
|
228
|
+
case 'mismatch': {
|
|
229
|
+
const noun = subject || SUBJECT_OF[matcher] || 'Value';
|
|
230
|
+
const e = expected === 'predicate to succeed' ? "a value matching the test's pattern" : expected;
|
|
231
|
+
headline = `${cap(noun)}: expected ${short(e, 60)}, got ${short(received, 60)}`;
|
|
232
|
+
explain = `Expected ${e}; the app gave ${received}.${waited}`;
|
|
233
|
+
exp = e; act = received;
|
|
234
|
+
break;
|
|
235
|
+
}
|
|
236
|
+
case 'assertion':
|
|
237
|
+
headline = subject ? cap(subject) : `Check failed: ${assertion}`;
|
|
238
|
+
explain = 'The test\'s condition was false. The runner recorded no element or value for it, so the exact on-screen state was not captured.';
|
|
239
|
+
exp = 'the check to pass'; act = 'not captured (the condition was false)';
|
|
240
|
+
break;
|
|
241
|
+
default:
|
|
242
|
+
headline = short(first, 140);
|
|
243
|
+
explain = 'The test stopped with this error; no expected or actual value was recorded.';
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const root = ctx.root;
|
|
247
|
+
const d = {
|
|
248
|
+
kind, kindLabel: KIND_LABEL[kind], headline, explain,
|
|
249
|
+
subject, assertion, matcher, locator,
|
|
250
|
+
expected: exp, actual: act, rawExpected: expected, rawReceived: received,
|
|
251
|
+
element: element.tag ? { tag: element.tag, text: element.text, html: short(resolved[2], 300), count: Number(resolved[1]) } : undefined,
|
|
252
|
+
timeoutMs, callLog: callLog && callLog.slice(0, 3000), message: s.slice(0, 4000),
|
|
253
|
+
file: relTo(root, ctx.file), line: ctx.line, column: ctx.column, snippet: ctx.snippet && strip(ctx.snippet).slice(0, 1200),
|
|
254
|
+
browser: ctx.browser || undefined,
|
|
255
|
+
artifacts: (ctx.attachments || []).filter((a) => a && a.path).map((a) => ({ name: a.name, path: relTo(root, a.path) })),
|
|
256
|
+
};
|
|
257
|
+
for (const k of Object.keys(d)) if (d[k] === undefined || (Array.isArray(d[k]) && !d[k].length)) delete d[k];
|
|
258
|
+
return d;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// For a failed check: what it wanted and what it got, as two short strings.
|
|
262
|
+
// Kept for the older report fields; the values come from failureDetail, so
|
|
263
|
+
// "actual" is always the observed state, never the test's own message.
|
|
264
|
+
function expectation(raw) {
|
|
265
|
+
const d = failureDetail(raw);
|
|
266
|
+
if (!d) return {};
|
|
267
|
+
const out = {};
|
|
268
|
+
if (d.expected) out.expected = d.expected;
|
|
269
|
+
if (d.actual) out.actual = d.actual;
|
|
270
|
+
return out;
|
|
136
271
|
}
|
|
137
272
|
const PLAIN = { visible: 'shown on screen', hidden: 'not shown on screen (hidden)', enabled: 'enabled', disabled: 'disabled' };
|
|
138
273
|
const plain = (v) => (v && PLAIN[v]) || v;
|
|
@@ -217,4 +352,4 @@ async function publish(category, suites, durationSec) {
|
|
|
217
352
|
}
|
|
218
353
|
}
|
|
219
354
|
|
|
220
|
-
module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation };
|
|
355
|
+
module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to jest.config.js:
|
|
3
3
|
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation, failureDetail } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgeJestReporter {
|
|
7
7
|
constructor(_globalConfig, options) {
|
|
@@ -25,6 +25,7 @@ module.exports = class TestForgeJestReporter {
|
|
|
25
25
|
: undefined,
|
|
26
26
|
checks: testChecks(f.testFilePath, a.location && a.location.line),
|
|
27
27
|
...(status === 'failed' ? expectation(raw) : {}),
|
|
28
|
+
detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
|
|
28
29
|
});
|
|
29
30
|
}
|
|
30
31
|
}
|
|
@@ -1,13 +1,28 @@
|
|
|
1
1
|
// TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to playwright.config.js reporter array:
|
|
3
3
|
// ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
|
|
4
|
-
const { cleanError, sig, publish, testChecks,
|
|
4
|
+
const { cleanError, sig, publish, testChecks, failureDetail, findRoot } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgePlaywrightReporter {
|
|
7
7
|
constructor(options) {
|
|
8
8
|
this.category = (options && options.category) || 'web';
|
|
9
9
|
this.suites = {};
|
|
10
10
|
this.start = Date.now();
|
|
11
|
+
this.root = findRoot(process.cwd());
|
|
12
|
+
}
|
|
13
|
+
// Everything the runner knows about a failure beyond its message: where it
|
|
14
|
+
// happened, the browser, the test's timeout and the artifacts it left.
|
|
15
|
+
detail(test, result) {
|
|
16
|
+
const err = result.error || {};
|
|
17
|
+
const loc = err.location || test.location || {};
|
|
18
|
+
const project = test.parent && typeof test.parent.project === 'function' ? test.parent.project() : undefined;
|
|
19
|
+
const d = failureDetail(err.message, {
|
|
20
|
+
file: loc.file, line: loc.line, column: loc.column, snippet: err.snippet,
|
|
21
|
+
browser: project && project.name, timeoutMs: test.timeout,
|
|
22
|
+
attachments: (result.attachments || []).map((a) => ({ name: a.name, path: a.path })),
|
|
23
|
+
root: this.root,
|
|
24
|
+
});
|
|
25
|
+
return d || undefined;
|
|
11
26
|
}
|
|
12
27
|
onTestEnd(test, result) {
|
|
13
28
|
const file = (test.location && test.location.file) || 'suite';
|
|
@@ -25,9 +40,17 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
25
40
|
note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
|
|
26
41
|
checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
|
|
27
42
|
|| testChecks(test.location && test.location.file, test.location && test.location.line),
|
|
28
|
-
...(status === 'failed' ?
|
|
43
|
+
...(status === 'failed' ? this.failed(test, result) : {}),
|
|
29
44
|
});
|
|
30
45
|
}
|
|
46
|
+
failed(test, result) {
|
|
47
|
+
const detail = this.detail(test, result);
|
|
48
|
+
if (!detail) return {};
|
|
49
|
+
const out = { detail };
|
|
50
|
+
if (detail.expected) out.expected = detail.expected;
|
|
51
|
+
if (detail.actual) out.actual = detail.actual;
|
|
52
|
+
return out;
|
|
53
|
+
}
|
|
31
54
|
async onEnd(result) {
|
|
32
55
|
const durationSec = Math.round(((result && result.duration) || (Date.now() - this.start)) / 1000);
|
|
33
56
|
const suites = Object.values(this.suites);
|