@activate-core/test-forge 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,8 +17,8 @@
17
17
  * false-rejecting valid TypeScript.
18
18
  */
19
19
  import { execFileSync } from 'node:child_process';
20
- import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
21
- import { join } from 'node:path';
20
+ import { writeFileSync, mkdtempSync, rmSync, existsSync } from 'node:fs';
21
+ import { join, dirname, resolve } from 'node:path';
22
22
  import { tmpdir } from 'node:os';
23
23
  import { reasonlessPending, pendingDeclarations } from '../generate/pending.js';
24
24
  /**
@@ -169,6 +169,13 @@ export function checkTestGates(content, _ext, ctx) {
169
169
  if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
170
170
  return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
171
171
  }
172
+ // F17 — relative imports resolve from where the file is written.
173
+ if (ctx.filePath) {
174
+ const missing = unresolvedImports(content, ctx.filePath);
175
+ if (missing.length) {
176
+ return `relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`;
177
+ }
178
+ }
172
179
  // F15 — a pending test says why. A reasonless fixme/skip reaches the report
173
180
  // as a bare "fixme", indistinguishable from the tool giving up. Runs on
174
181
  // content normalisePending() has already given every reason it could find.
@@ -386,3 +393,19 @@ function looksStructurallyClean(content) {
386
393
  count(content, '(') === count(content, ')');
387
394
  return balanced;
388
395
  }
396
+ const RESOLVE_EXTS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json'];
397
+ /** Relative module specifiers (import / require / dynamic import) that reach no file from `filePath`. */
398
+ export function unresolvedImports(content, filePath) {
399
+ const specs = new Set();
400
+ const re = /(?:\bfrom\s*|\brequire\s*\(\s*|\bimport\s*\(\s*|^\s*import\s+)(['"])(\.{1,2}\/[^'"]*)\1/gm;
401
+ for (const m of content.matchAll(re))
402
+ specs.add(m[2]);
403
+ const base = dirname(resolve(filePath));
404
+ return [...specs].filter((spec) => {
405
+ const target = resolve(base, spec);
406
+ // A compiled .js specifier may point at the .ts source.
407
+ const stem = target.replace(/\.(c|m)?js$/, '');
408
+ return !RESOLVE_EXTS.some((e) => existsSync(target + e) || existsSync(stem + e))
409
+ && !RESOLVE_EXTS.slice(1).some((e) => existsSync(join(target, 'index' + e)));
410
+ });
411
+ }
@@ -20,6 +20,7 @@ import { existsSync, readFileSync } from 'node:fs';
20
20
  import { relative } from 'node:path';
21
21
  import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
22
22
  import { runWithStdin } from './ai-generator.js';
23
+ import { aiUnavailableReason, noteAiFailure, noteAiSuccess, waitForAi } from './ai-session.js';
23
24
  const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'unproven'];
24
25
  /**
25
26
  * Group failing tests by the claim marked above them. A parametrised test
@@ -149,24 +150,47 @@ export function splitReview(group, parsed) {
149
150
  out.unshift({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence });
150
151
  return out;
151
152
  }
152
- /** Review every claim group, a few at a time. Never throws. */
153
- export async function reviewDefects(journey, groups, passes) {
153
+ /**
154
+ * Review every claim group, a few at a time. Never throws.
155
+ *
156
+ * A review that failed because the AI is gone (usage limit, logged out) is
157
+ * not a verdict. On the CleanMe booking-summary run the limit landed exactly
158
+ * here: all 18 claims came back "unproven — the review could not be read",
159
+ * the journey-level wait only noticed at the next AI call, and the summary
160
+ * reported 0 defects. So the group goes back on the queue, every worker
161
+ * shares one wait, and the review is asked again once the AI answers.
162
+ */
163
+ export async function reviewDefects(journey, groups, passes, io) {
154
164
  const config = loadConfig();
155
165
  const decisions = decisionsSource(config.decisions);
156
166
  // A linked list (an internal Notion page) needs the reference tools to be read.
157
167
  const tools = decisions.link ? withReferenceTools : withReadOnlyTools;
158
168
  const { program, args } = tools(splitCommand(config.aiCommand));
169
+ const ask = io?.ask ?? ((prompt) => runWithStdin(program, args, prompt, config.aiTimeoutMs));
170
+ const wait = io?.wait ?? (() => ((config.waitForLimit ?? true) ? waitForAi(config) : Promise.resolve(false)));
159
171
  const out = [];
160
172
  const queue = [...groups];
173
+ let waiting = null;
174
+ let gaveUp = false; // one failed wait is enough; the rest are recorded as unproven
161
175
  const worker = async () => {
162
176
  for (let g = queue.shift(); g; g = queue.shift()) {
163
177
  const files = new Set(g.tests.map((t) => t.file));
164
178
  const siblings = passes.filter((p) => files.has(p.file));
165
179
  let raw = '';
166
180
  try {
167
- raw = await runWithStdin(program, args, buildDefectReviewPrompt(journey, g, siblings, decisions), config.aiTimeoutMs);
181
+ raw = await ask(buildDefectReviewPrompt(journey, g, siblings, decisions));
182
+ noteAiSuccess();
168
183
  }
169
- catch {
184
+ catch (e) {
185
+ noteAiFailure(e);
186
+ if (aiUnavailableReason() && !gaveUp) {
187
+ waiting ??= wait().finally(() => (waiting = null));
188
+ if (await waiting) {
189
+ queue.unshift(g);
190
+ continue;
191
+ }
192
+ gaveUp = true;
193
+ }
170
194
  // an unanswered review proves nothing
171
195
  }
172
196
  out.push(...splitReview(g, parseReview(raw)));
@@ -408,8 +408,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
408
408
  });
409
409
  // A freshly written file also may not hold placeholders; a repair is held to
410
410
  // the repair guard instead (it may not add a skip), so old files stay repairable.
411
- const genGate = (content) => testGate(content) ??
412
- checkTestGates(content, ext, { framework, forbidPlaceholders: true });
411
+ const genGate = (content, filePath) => testGate(content) ??
412
+ checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath });
413
413
  // Skip if unchanged — but NOT if the journey still has deferred stubs
414
414
  // to retry (a resume pass fills those in without touching real files).
415
415
  if (!force &&
@@ -517,7 +517,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
517
517
  let status;
518
518
  if (useAI) {
519
519
  const facts = await grounding();
520
- const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, genGate);
520
+ const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, (c) => genGate(c, filePath));
521
521
  if (ai !== null) {
522
522
  content = framework === 'playwright' ? normalisePending(ai) : ai;
523
523
  status = 'ai';
@@ -65,6 +65,18 @@ export async function verifyJourney(ctx) {
65
65
  let outcome = await run();
66
66
  for (let attempt = 0;; attempt += 1) {
67
67
  if (!usable(outcome)) {
68
+ // The file itself does not load (a bad import, a syntax the runner
69
+ // rejects): that is the file's fault, so it is repaired like a
70
+ // failing test. Only an environment problem (infra) is left alone.
71
+ const loadError = !outcome.ran && !outcome.infra;
72
+ if (loadError && attempt < ctx.attempts) {
73
+ console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
74
+ const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
75
+ if (await repairOnce(target, [issue], [])) {
76
+ outcome = await run();
77
+ continue;
78
+ }
79
+ }
68
80
  notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
69
81
  break;
70
82
  }
@@ -69,7 +69,7 @@ export function mapPlaywright(json, category = 'web') {
69
69
  const rootDir = (json.config && json.config.rootDir) || process.cwd();
70
70
  const walk = (suite) => {
71
71
  for (const spec of suite.specs || []) {
72
- const name = String(spec.file || suite.file || 'suite').split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
72
+ const name = shared.suiteName(spec.file || suite.file || 'suite');
73
73
  const su = (suites[name] ||= { name, category, tests: [] });
74
74
  for (const t of spec.tests || []) {
75
75
  const r = (t.results || [])[(t.results || []).length - 1] || {};
@@ -78,11 +78,20 @@ export function mapPlaywright(json, category = 'web') {
78
78
  const raw = r.error && r.error.message;
79
79
  const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
80
80
  const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
81
+ const err = r.error || {};
82
+ const detail = status === 'failed'
83
+ ? shared.failureDetail(raw, {
84
+ file: err.location && err.location.file, line: err.location && err.location.line, column: err.location && err.location.column,
85
+ snippet: err.snippet, timeoutMs: undefined, browser: t.projectName,
86
+ attachments: (r.attachments || []).map((a) => ({ name: a.name, path: a.path })), root: shared.findRoot(rootDir),
87
+ })
88
+ : undefined;
81
89
  su.tests.push({
82
90
  name: spec.title,
83
91
  status,
84
92
  checks: shared.testChecks(loc, spec.line),
85
93
  ...(status === 'failed' ? shared.expectation(raw) : {}),
94
+ ...(detail ? { detail } : {}),
86
95
  error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
87
96
  sig: status === 'failed' ? shared.sig(raw) : undefined,
88
97
  note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
@@ -97,12 +106,10 @@ export function mapPlaywright(json, category = 'web') {
97
106
  return { suites: Object.values(suites), durationSec: Math.round(((json.stats && json.stats.duration) || 0) / 1000) };
98
107
  }
99
108
  /** Map a Jest `--json` report to the dashboard run contract. */
100
- export function mapJest(json, category = 'api') {
109
+ export function mapJest(json, category = 'api', testDir) {
101
110
  const suites = {};
102
111
  for (const f of json.testResults || []) {
103
- const name = String(f.name || f.testFilePath || 'suite')
104
- .split('/').pop()
105
- .replace(/\.(test|spec)\.[jt]s$/, '');
112
+ const name = shared.suiteName(f.name || f.testFilePath || 'suite', testDir);
106
113
  const su = (suites[name] ||= { name, category, tests: [] });
107
114
  for (const a of f.assertionResults || []) {
108
115
  const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
@@ -123,7 +130,7 @@ export async function publishReport(opts) {
123
130
  const json = JSON.parse(fs.readFileSync(opts.resultsPath, 'utf8'));
124
131
  // Jest's report has testResults; Playwright's has suites.
125
132
  const { suites, durationSec } = Array.isArray(json.testResults)
126
- ? mapJest(json, opts.category || 'api')
133
+ ? mapJest(json, opts.category || 'api', shared.testRoot(shared.findRoot(process.cwd()), opts.category || 'api'))
127
134
  : mapPlaywright(json, opts.category || 'web');
128
135
  const body = { project: opts.project, env: opts.env || process.env.TESTFORGE_ENV || 'local', durationSec, suites };
129
136
  const res = await fetch(opts.dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
@@ -187,7 +194,7 @@ function attachPending(dir, ext, suites) {
187
194
  }
188
195
  const byName = new Map();
189
196
  for (const f of files)
190
- byName.set(suiteName(f), f);
197
+ byName.set(suiteName(f, dir), f);
191
198
  for (const s of suites) {
192
199
  const f = byName.get(s.name);
193
200
  if (!f || s.pending)
@@ -206,14 +213,14 @@ function walk(dir, ext, out) {
206
213
  out.push(full);
207
214
  }
208
215
  }
209
- const suiteName = (file) => path.basename(file).replace(/\.(test|spec)\.[jt]sx?$/, '');
216
+ const suiteName = (file, dir) => shared.suiteName(file, dir);
210
217
  const staticCount = (file) => (fs.readFileSync(file, 'utf8').match(/\b(it|test)\s*\(/g) || []).length;
211
218
  /** Static fallback: count `it(`/`test(` occurrences per file (undercounts parameterized tests). */
212
219
  function jestStatic(dir, ext, category, out) {
213
220
  const files = [];
214
221
  walk(dir, ext, files);
215
222
  for (const f of files)
216
- out.push({ name: suiteName(f), category, total: staticCount(f) });
223
+ out.push({ name: suiteName(f, dir), category, total: staticCount(f) });
217
224
  }
218
225
  // setupFilesAfterEnv shim: runs after jest installs the test globals, replaces
219
226
  // every test BODY and before/after hook with a no-op. Collection still runs
@@ -282,13 +289,13 @@ function jestCount(dir, ext, category, out) {
282
289
  const j = JSON.parse(fs.readFileSync(outJson, 'utf8'));
283
290
  const seen = new Set();
284
291
  for (const f of j.testResults || []) {
285
- const n = suiteName(String(f.name || f.testFilePath || 'suite'));
292
+ const n = suiteName(String(f.name || f.testFilePath || 'suite'), dir);
286
293
  seen.add(n);
287
294
  out.push({ name: n, category, total: (f.assertionResults || []).length });
288
295
  }
289
296
  // any file jest couldn't even load → static, and say so loudly
290
297
  for (const f of files) {
291
- const n = suiteName(f);
298
+ const n = suiteName(f, dir);
292
299
  if (seen.has(n))
293
300
  continue;
294
301
  const total = staticCount(f);
@@ -315,7 +322,7 @@ function jestCount(dir, ext, category, out) {
315
322
  * EXPANDS parameterized tests (loops / .each), so we get the real case count
316
323
  * per file. Returns false if it couldn't run (caller falls back to static).
317
324
  */
318
- function playwrightList(pkgDir, category, out) {
325
+ function playwrightList(pkgDir, dir, category, out) {
319
326
  try {
320
327
  // `--reporter=list` so the project's own reporters — ours included — do
321
328
  // not run during a count. Listing publishes nothing on its own, but a
@@ -326,10 +333,12 @@ function playwrightList(pkgDir, category, out) {
326
333
  const counts = {};
327
334
  for (const line of res.split('\n')) {
328
335
  const m = line.match(/([^\s›»]+\.(?:test|spec)\.[jt]sx?):\d+:\d+/);
329
- if (m) {
330
- const n = suiteName(m[1]);
331
- counts[n] = (counts[n] || 0) + 1;
332
- }
336
+ if (!m)
337
+ continue;
338
+ // Playwright lists files relative to testDir; older versions relative to the config dir.
339
+ const abs = [path.resolve(dir, m[1]), path.resolve(pkgDir, m[1])].find((f) => fs.existsSync(f)) ?? path.resolve(dir, m[1]);
340
+ const n = suiteName(abs, dir);
341
+ counts[n] = (counts[n] || 0) + 1;
333
342
  }
334
343
  const names = Object.keys(counts);
335
344
  if (!names.length)
@@ -356,7 +365,7 @@ export function scanInventory(cwd, testOutput, ext, framework = {}) {
356
365
  if (framework[category] === 'playwright') {
357
366
  // The config sits at the package root (or, beside a project's own
358
367
  // config, in the tests' parent folder) — list from wherever it is.
359
- if (playwrightList(packageRoot(dir), category, suites) || playwrightList(path.dirname(dir), category, suites)) {
368
+ if (playwrightList(packageRoot(dir), dir, category, suites) || playwrightList(path.dirname(dir), dir, category, suites)) {
360
369
  attachPending(dir, ext, suites);
361
370
  continue;
362
371
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "1.2.0",
3
+ "version": "1.2.2",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -118,21 +118,156 @@ function testChecks(file, line) {
118
118
  return uniq.length ? uniq.join('\n').slice(0, 1200) : undefined;
119
119
  }
120
120
 
121
- // For a failed check: what it wanted and what it got, from the runner's own message.
122
- // A test's own failure message ("no visible reason, only a red field") says what the
123
- // app did better than a bare true/false, so it wins over Received when present.
124
- function expectation(raw) {
125
- const s = strip(raw || '');
126
- if (!s) return {};
121
+ // ---- Failure detail -------------------------------------------------------
122
+ // The runner's message holds four different things that a report must keep
123
+ // apart: the test's OWN message (`expect(x, 'a visible reason under Password')`
124
+ // — what the check wanted, in the test's words), the matcher and its
125
+ // Expected/Received values, the locator and what it resolved to (the
126
+ // evidence of the element's real state), and the call log. The old mapping
127
+ // put the test's own message in "actual", so a report read "got: a visible
128
+ // reason under Password" — the expectation, dressed up as the observation.
129
+
130
+ const STATE_MATCHERS = /^toBe(Visible|Hidden|Enabled|Disabled|Checked|Focused|Editable|Empty|Attached|InViewport)$/;
131
+ const SUBJECT_OF = {
132
+ toHaveText: 'Text', toContainText: 'Text', toHaveURL: 'Page URL', toHaveTitle: 'Page title',
133
+ toHaveCount: 'Element count', toHaveValue: 'Field value', toHaveValues: 'Field values',
134
+ toHaveAttribute: 'Attribute', toHaveClass: 'Class', toHaveCSS: 'Style', toHaveId: 'Id',
135
+ toHaveScreenshot: 'Screenshot', toBeOK: 'Response', toHaveLength: 'Length',
136
+ };
137
+ const KIND_LABEL = {
138
+ hidden: 'Element hidden', 'not-found': 'Element not found', ambiguous: 'Ambiguous selector',
139
+ timeout: 'Test timeout', state: 'Wrong element state', mismatch: 'Value mismatch',
140
+ assertion: 'Condition false', error: 'Error thrown',
141
+ };
142
+ const short = (v, n = 90) => { v = String(v == null ? '' : v).trim(); return v.length > n ? v.slice(0, n - 1) + '…' : v; };
143
+ const cap = (v) => (v ? v[0].toUpperCase() + v.slice(1) : v);
144
+ const secs = (ms) => (ms % 1000 === 0 ? ms / 1000 + 's' : ms + 'ms');
145
+ const relTo = (root, p) => (p && root && p.startsWith(root + path.sep) ? p.slice(root.length + 1) : p);
146
+
147
+ // Inner text of the element Playwright printed ("<span class="x">Password format is incorrect.</span>").
148
+ function elementText(html) {
149
+ const m = (html || '').match(/^<([a-zA-Z0-9-]+)[^>]*>([\s\S]*?)<\/\1>\s*$/) || (html || '').match(/^<([a-zA-Z0-9-]+)[^>]*>([\s\S]*)$/);
150
+ if (!m) return { tag: ((html || '').match(/^<([a-zA-Z0-9-]+)/) || [])[1] };
151
+ const text = m[2].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim().replace(/…$/, '');
152
+ return { tag: m[1], text: text || undefined };
153
+ }
154
+
155
+ /**
156
+ * Structured detail of one failure, from the runner's message plus what the
157
+ * reporter knows (location, browser, timeout, artifacts). Every field comes
158
+ * from evidence; nothing is inferred from the wording of a message.
159
+ *
160
+ * ctx: { file, line, column, snippet, browser, timeoutMs, attachments: [{name, path}], root }
161
+ */
162
+ function failureDetail(raw, ctx) {
163
+ ctx = ctx || {};
164
+ const s = strip(raw || '').replace(/\r/g, '');
165
+ if (!s.trim()) return undefined;
166
+ const lines = s.split('\n');
127
167
  const pick = (re) => { const m = s.match(re); return m ? m[1].trim() : undefined; };
168
+ const first = (lines.find((l) => l.trim()) || '').trim().replace(/^Error:\s*/, '');
169
+
170
+ const assertionLine = lines.map((l) => l.trim()).find((l) => /^(Error:\s*)?expect\(.*\)\.\w+\(/.test(l));
171
+ const assertion = assertionLine ? assertionLine.replace(/^Error:\s*/, '').replace(/\s*(failed|\/\/.*)$/, '').trim() : undefined;
172
+ const matcher = assertion ? (assertion.match(/\.(\w+)\(/) || [])[1] : undefined;
173
+ const locator = pick(/^\s*Locator:\s*(.+)$/m);
128
174
  const expected = pick(/^\s*Expected(?: [a-z ]+)?:\s*(.+)$/m);
129
- let actual = pick(/^\s*Received(?: [a-z ]+)?:\s*(.+)$/m);
130
- if (/element\(s\) not found/.test(s)) actual = 'the element was not found on the page';
131
- const first = (s.split('\n').find((l) => l.trim()) || '').replace(/^\s*Error:\s*/, '').trim();
132
- const own = first && !/^expect[.(]|^Test timeout|^locator\./i.test(first) ? first : undefined;
133
- if (/^Test timeout of/i.test(first)) return { expected: plain(expected), actual: first };
134
- // A bare true/false says nothing once the test's own message explains it.
135
- return { expected: own && /^(true|false)$/.test(expected || '') ? undefined : plain(expected), actual: own || plain(actual) };
175
+ const received = pick(/^\s*Received(?: [a-z ]+)?:\s*(.+)$/m);
176
+ const testTimeout = s.match(/Test timeout of (\d+) ?ms exceeded/i);
177
+ const timeoutMs = testTimeout ? Number(testTimeout[1])
178
+ : Number(pick(/^\s*Timeout:?\s*(\d+)\s*ms/m) || (s.match(/Timeout (\d+) ?ms exceeded/) || [])[1] || ctx.timeoutMs || 0) || undefined;
179
+ const resolved = s.match(/(\d+) × locator resolved to (.+)$/m);
180
+ const strict = s.match(/strict mode violation: (.+?) resolved to (\d+) elements/);
181
+ const callLog = (s.split(/\n\s*Call log:\s*\n/i)[1] || '').replace(/\n\s*at\s.*$/gm, '').trim() || undefined;
182
+
183
+ // The test's own message: the first line when it is not the runner's own wording.
184
+ const runnerWording = /^(expect[.(]|Test timeout|Timeout|locator\.|page\.|strict mode violation|element\(s\) not found|Target (page|context|browser)|Error: )/i;
185
+ const subject = first && first !== assertion && !runnerWording.test(first) ? first : undefined;
186
+ const element = resolved ? elementText(resolved[2]) : {};
187
+
188
+ let kind;
189
+ if (testTimeout) kind = 'timeout';
190
+ else if (strict) kind = 'ambiguous';
191
+ else if (/element\(s\) not found/.test(s) || (locator && /waiting for/.test(s) && !resolved && matcher && STATE_MATCHERS.test(matcher))) kind = 'not-found';
192
+ else if (matcher === 'toBeVisible' && received === 'hidden') kind = 'hidden';
193
+ else if (matcher && STATE_MATCHERS.test(matcher)) kind = 'state';
194
+ else if (/^(true|false)$/.test(expected || '') && /^(true|false)$/.test(received || '')) kind = 'assertion';
195
+ else if (matcher && (expected !== undefined || received !== undefined)) kind = 'mismatch';
196
+ else if (matcher) kind = 'assertion';
197
+ else kind = 'error';
198
+
199
+ const what = subject || (locator ? `the element ${short(locator, 60)}` : undefined);
200
+ const waited = timeoutMs ? ` Waited ${secs(timeoutMs)}.` : '';
201
+ let headline, explain, exp, act;
202
+ switch (kind) {
203
+ case 'timeout':
204
+ headline = `Test timed out after ${secs(timeoutMs)}`;
205
+ explain = `The runner stopped the test after ${secs(timeoutMs)}, before it finished.`;
206
+ act = `the test did not finish within ${secs(timeoutMs)}`;
207
+ break;
208
+ case 'ambiguous':
209
+ headline = `${cap(subject || `Selector ${short(strict[1], 60)}`)} matched ${strict[2]} elements`;
210
+ explain = `The selector matched ${strict[2]} elements; Playwright strict mode needs exactly one.`;
211
+ exp = 'exactly one matching element'; act = `${strict[2]} elements matched`;
212
+ break;
213
+ case 'not-found':
214
+ headline = `${cap(what || 'The element')} was not found on the page`;
215
+ explain = `No element matched the selector: either it was never rendered, or the selector does not match the page.${waited}`;
216
+ exp = plain(expected) || 'present on the page'; act = 'not found on the page';
217
+ break;
218
+ case 'hidden':
219
+ headline = `${cap(what || 'The element')} is on the page but hidden`;
220
+ explain = `The element exists${element.text ? ` and reads "${short(element.text, 120)}"` : ''}, but it is not displayed, so the customer cannot see it.${waited}`;
221
+ exp = 'shown on screen'; act = `on the page but hidden${element.text ? ` ("${short(element.text, 80)}")` : ''}`;
222
+ break;
223
+ case 'state':
224
+ headline = `${cap(what || 'The element')}: expected ${plain(expected)}, got ${plain(received)}`;
225
+ explain = `Expected the element to be ${plain(expected)}; it was ${plain(received)}.${waited}`;
226
+ exp = plain(expected); act = plain(received);
227
+ break;
228
+ case 'mismatch': {
229
+ const noun = subject || SUBJECT_OF[matcher] || 'Value';
230
+ const e = expected === 'predicate to succeed' ? "a value matching the test's pattern" : expected;
231
+ headline = `${cap(noun)}: expected ${short(e, 60)}, got ${short(received, 60)}`;
232
+ explain = `Expected ${e}; the app gave ${received}.${waited}`;
233
+ exp = e; act = received;
234
+ break;
235
+ }
236
+ case 'assertion':
237
+ headline = subject ? cap(subject) : `Check failed: ${assertion}`;
238
+ explain = 'The test\'s condition was false. The runner recorded no element or value for it, so the exact on-screen state was not captured.';
239
+ exp = 'the check to pass'; act = 'not captured (the condition was false)';
240
+ break;
241
+ default:
242
+ headline = short(first, 140);
243
+ explain = 'The test stopped with this error; no expected or actual value was recorded.';
244
+ }
245
+
246
+ const root = ctx.root;
247
+ const d = {
248
+ kind, kindLabel: KIND_LABEL[kind], headline, explain,
249
+ subject, assertion, matcher, locator,
250
+ expected: exp, actual: act, rawExpected: expected, rawReceived: received,
251
+ element: element.tag ? { tag: element.tag, text: element.text, html: short(resolved[2], 300), count: Number(resolved[1]) } : undefined,
252
+ timeoutMs, callLog: callLog && callLog.slice(0, 3000), message: s.slice(0, 4000),
253
+ file: relTo(root, ctx.file), line: ctx.line, column: ctx.column, snippet: ctx.snippet && strip(ctx.snippet).slice(0, 1200),
254
+ browser: ctx.browser || undefined,
255
+ artifacts: (ctx.attachments || []).filter((a) => a && a.path).map((a) => ({ name: a.name, path: relTo(root, a.path) })),
256
+ };
257
+ for (const k of Object.keys(d)) if (d[k] === undefined || (Array.isArray(d[k]) && !d[k].length)) delete d[k];
258
+ return d;
259
+ }
260
+
261
+ // For a failed check: what it wanted and what it got, as two short strings.
262
+ // Kept for the older report fields; the values come from failureDetail, so
263
+ // "actual" is always the observed state, never the test's own message.
264
+ function expectation(raw) {
265
+ const d = failureDetail(raw);
266
+ if (!d) return {};
267
+ const out = {};
268
+ if (d.expected) out.expected = d.expected;
269
+ if (d.actual) out.actual = d.actual;
270
+ return out;
136
271
  }
137
272
  const PLAIN = { visible: 'shown on screen', hidden: 'not shown on screen (hidden)', enabled: 'enabled', disabled: 'disabled' };
138
273
  const plain = (v) => (v && PLAIN[v]) || v;
@@ -217,4 +352,25 @@ async function publish(category, suites, durationSec) {
217
352
  }
218
353
  }
219
354
 
220
- module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation };
355
+ // A suite is named by its path under the test root — "booking-summary/keyboard-access" —
356
+ // because several journeys may each own a keyboard-access.test.ts. Keyed by basename
357
+ // alone, three such files became one suite of 22 and a single-journey run showed 13
358
+ // phantom skips. A relative path is kept as given (Playwright's JSON already reports
359
+ // spec files relative to testDir); an absolute path outside the root falls back to the
360
+ // bare file name.
361
+ function suiteName(file, root) {
362
+ let rel = String(file || 'suite').replace(/\\/g, '/');
363
+ if (path.isAbsolute(rel)) {
364
+ const r = root ? path.relative(root, rel).replace(/\\/g, '/') : '';
365
+ rel = r && !r.startsWith('..') ? r : rel.split('/').pop();
366
+ }
367
+ return rel.replace(/\.(test|spec)\.[jt]sx?$/, '');
368
+ }
369
+ // The configured test folder for a category (testOutput in testforge.config.json), absolute.
370
+ function testRoot(root, category) {
371
+ const out = loadConfig(root).testOutput;
372
+ const d = out && out[category];
373
+ return d ? path.resolve(root, d) : undefined;
374
+ }
375
+
376
+ module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot };
@@ -1,16 +1,17 @@
1
1
  // TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to jest.config.js:
3
3
  // reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
4
- const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgeJestReporter {
7
7
  constructor(_globalConfig, options) {
8
8
  this.category = (options && options.category) || 'api';
9
+ this.testRoot = testRoot(findRoot(process.cwd()), this.category);
9
10
  }
10
11
  async onRunComplete(_contexts, results) {
11
12
  const suites = {};
12
13
  for (const f of results.testResults || []) {
13
- const name = String(f.testFilePath || f.name || 'suite').split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
14
+ const name = suiteName(f.testFilePath || f.name || 'suite', this.testRoot);
14
15
  const su = suites[name] || (suites[name] = { name, category: this.category, tests: [] });
15
16
  for (const a of (f.testResults || f.assertionResults || [])) {
16
17
  const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
@@ -25,6 +26,7 @@ module.exports = class TestForgeJestReporter {
25
26
  : undefined,
26
27
  checks: testChecks(f.testFilePath, a.location && a.location.line),
27
28
  ...(status === 'failed' ? expectation(raw) : {}),
29
+ detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
28
30
  });
29
31
  }
30
32
  }
@@ -1,17 +1,38 @@
1
1
  // TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to playwright.config.js reporter array:
3
3
  // ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
4
- const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgePlaywrightReporter {
7
7
  constructor(options) {
8
8
  this.category = (options && options.category) || 'web';
9
9
  this.suites = {};
10
10
  this.start = Date.now();
11
+ this.root = findRoot(process.cwd());
12
+ this.testRoot = testRoot(this.root, this.category);
13
+ }
14
+ onBegin(config) {
15
+ // Playwright's own testDir is the authority when it has one.
16
+ const td = config && config.projects && config.projects[0] && config.projects[0].testDir;
17
+ if (td) this.testRoot = td;
18
+ }
19
+ // Everything the runner knows about a failure beyond its message: where it
20
+ // happened, the browser, the test's timeout and the artifacts it left.
21
+ detail(test, result) {
22
+ const err = result.error || {};
23
+ const loc = err.location || test.location || {};
24
+ const project = test.parent && typeof test.parent.project === 'function' ? test.parent.project() : undefined;
25
+ const d = failureDetail(err.message, {
26
+ file: loc.file, line: loc.line, column: loc.column, snippet: err.snippet,
27
+ browser: project && project.name, timeoutMs: test.timeout,
28
+ attachments: (result.attachments || []).map((a) => ({ name: a.name, path: a.path })),
29
+ root: this.root,
30
+ });
31
+ return d || undefined;
11
32
  }
12
33
  onTestEnd(test, result) {
13
34
  const file = (test.location && test.location.file) || 'suite';
14
- const name = String(file).split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
35
+ const name = suiteName(file, this.testRoot);
15
36
  const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
16
37
  const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
17
38
  const raw = result.error && result.error.message;
@@ -25,9 +46,17 @@ module.exports = class TestForgePlaywrightReporter {
25
46
  note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
26
47
  checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
27
48
  || testChecks(test.location && test.location.file, test.location && test.location.line),
28
- ...(status === 'failed' ? expectation(raw) : {}),
49
+ ...(status === 'failed' ? this.failed(test, result) : {}),
29
50
  });
30
51
  }
52
+ failed(test, result) {
53
+ const detail = this.detail(test, result);
54
+ if (!detail) return {};
55
+ const out = { detail };
56
+ if (detail.expected) out.expected = detail.expected;
57
+ if (detail.actual) out.actual = detail.actual;
58
+ return out;
59
+ }
31
60
  async onEnd(result) {
32
61
  const durationSec = Math.round(((result && result.duration) || (Date.now() - this.start)) / 1000);
33
62
  const suites = Object.values(this.suites);