@activate-core/test-forge 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -67,9 +67,10 @@ export function mapPlaywright(json, category = 'web') {
67
67
  const suites = {};
68
68
  // Spec files are relative to the config's testDir; the report records the root it used.
69
69
  const rootDir = (json.config && json.config.rootDir) || process.cwd();
70
+ const verdictOf = shared.reviewVerdicts(shared.findRoot(process.cwd()));
70
71
  const walk = (suite) => {
71
72
  for (const spec of suite.specs || []) {
72
- const name = String(spec.file || suite.file || 'suite').split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
73
+ const name = shared.suiteName(spec.file || suite.file || 'suite');
73
74
  const su = (suites[name] ||= { name, category, tests: [] });
74
75
  for (const t of spec.tests || []) {
75
76
  const r = (t.results || [])[(t.results || []).length - 1] || {};
@@ -95,6 +96,7 @@ export function mapPlaywright(json, category = 'web') {
95
96
  error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
96
97
  sig: status === 'failed' ? shared.sig(raw) : undefined,
97
98
  note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
99
+ ...(status === 'failed' ? verdictOf(loc, spec.title) : {}),
98
100
  });
99
101
  }
100
102
  }
@@ -106,12 +108,10 @@ export function mapPlaywright(json, category = 'web') {
106
108
  return { suites: Object.values(suites), durationSec: Math.round(((json.stats && json.stats.duration) || 0) / 1000) };
107
109
  }
108
110
  /** Map a Jest `--json` report to the dashboard run contract. */
109
- export function mapJest(json, category = 'api') {
111
+ export function mapJest(json, category = 'api', testDir) {
110
112
  const suites = {};
111
113
  for (const f of json.testResults || []) {
112
- const name = String(f.name || f.testFilePath || 'suite')
113
- .split('/').pop()
114
- .replace(/\.(test|spec)\.[jt]s$/, '');
114
+ const name = shared.suiteName(f.name || f.testFilePath || 'suite', testDir);
115
115
  const su = (suites[name] ||= { name, category, tests: [] });
116
116
  for (const a of f.assertionResults || []) {
117
117
  const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
@@ -132,7 +132,7 @@ export async function publishReport(opts) {
132
132
  const json = JSON.parse(fs.readFileSync(opts.resultsPath, 'utf8'));
133
133
  // Jest's report has testResults; Playwright's has suites.
134
134
  const { suites, durationSec } = Array.isArray(json.testResults)
135
- ? mapJest(json, opts.category || 'api')
135
+ ? mapJest(json, opts.category || 'api', shared.testRoot(shared.findRoot(process.cwd()), opts.category || 'api'))
136
136
  : mapPlaywright(json, opts.category || 'web');
137
137
  const body = { project: opts.project, env: opts.env || process.env.TESTFORGE_ENV || 'local', durationSec, suites };
138
138
  const res = await fetch(opts.dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
@@ -196,7 +196,7 @@ function attachPending(dir, ext, suites) {
196
196
  }
197
197
  const byName = new Map();
198
198
  for (const f of files)
199
- byName.set(suiteName(f), f);
199
+ byName.set(suiteName(f, dir), f);
200
200
  for (const s of suites) {
201
201
  const f = byName.get(s.name);
202
202
  if (!f || s.pending)
@@ -215,14 +215,14 @@ function walk(dir, ext, out) {
215
215
  out.push(full);
216
216
  }
217
217
  }
218
- const suiteName = (file) => path.basename(file).replace(/\.(test|spec)\.[jt]sx?$/, '');
218
+ const suiteName = (file, dir) => shared.suiteName(file, dir);
219
219
  const staticCount = (file) => (fs.readFileSync(file, 'utf8').match(/\b(it|test)\s*\(/g) || []).length;
220
220
  /** Static fallback: count `it(`/`test(` occurrences per file (undercounts parameterized tests). */
221
221
  function jestStatic(dir, ext, category, out) {
222
222
  const files = [];
223
223
  walk(dir, ext, files);
224
224
  for (const f of files)
225
- out.push({ name: suiteName(f), category, total: staticCount(f) });
225
+ out.push({ name: suiteName(f, dir), category, total: staticCount(f) });
226
226
  }
227
227
  // setupFilesAfterEnv shim: runs after jest installs the test globals, replaces
228
228
  // every test BODY and before/after hook with a no-op. Collection still runs
@@ -291,13 +291,13 @@ function jestCount(dir, ext, category, out) {
291
291
  const j = JSON.parse(fs.readFileSync(outJson, 'utf8'));
292
292
  const seen = new Set();
293
293
  for (const f of j.testResults || []) {
294
- const n = suiteName(String(f.name || f.testFilePath || 'suite'));
294
+ const n = suiteName(String(f.name || f.testFilePath || 'suite'), dir);
295
295
  seen.add(n);
296
296
  out.push({ name: n, category, total: (f.assertionResults || []).length });
297
297
  }
298
298
  // any file jest couldn't even load → static, and say so loudly
299
299
  for (const f of files) {
300
- const n = suiteName(f);
300
+ const n = suiteName(f, dir);
301
301
  if (seen.has(n))
302
302
  continue;
303
303
  const total = staticCount(f);
@@ -324,7 +324,7 @@ function jestCount(dir, ext, category, out) {
324
324
  * EXPANDS parameterized tests (loops / .each), so we get the real case count
325
325
  * per file. Returns false if it couldn't run (caller falls back to static).
326
326
  */
327
- function playwrightList(pkgDir, category, out) {
327
+ function playwrightList(pkgDir, dir, category, out) {
328
328
  try {
329
329
  // `--reporter=list` so the project's own reporters — ours included — do
330
330
  // not run during a count. Listing publishes nothing on its own, but a
@@ -335,10 +335,12 @@ function playwrightList(pkgDir, category, out) {
335
335
  const counts = {};
336
336
  for (const line of res.split('\n')) {
337
337
  const m = line.match(/([^\s›»]+\.(?:test|spec)\.[jt]sx?):\d+:\d+/);
338
- if (m) {
339
- const n = suiteName(m[1]);
340
- counts[n] = (counts[n] || 0) + 1;
341
- }
338
+ if (!m)
339
+ continue;
340
+ // Playwright lists files relative to testDir; older versions relative to the config dir.
341
+ const abs = [path.resolve(dir, m[1]), path.resolve(pkgDir, m[1])].find((f) => fs.existsSync(f)) ?? path.resolve(dir, m[1]);
342
+ const n = suiteName(abs, dir);
343
+ counts[n] = (counts[n] || 0) + 1;
342
344
  }
343
345
  const names = Object.keys(counts);
344
346
  if (!names.length)
@@ -365,7 +367,7 @@ export function scanInventory(cwd, testOutput, ext, framework = {}) {
365
367
  if (framework[category] === 'playwright') {
366
368
  // The config sits at the package root (or, beside a project's own
367
369
  // config, in the tests' parent folder) — list from wherever it is.
368
- if (playwrightList(packageRoot(dir), category, suites) || playwrightList(path.dirname(dir), category, suites)) {
370
+ if (playwrightList(packageRoot(dir), dir, category, suites) || playwrightList(path.dirname(dir), dir, category, suites)) {
369
371
  attachPending(dir, ext, suites);
370
372
  continue;
371
373
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "1.2.1",
3
+ "version": "1.3.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -332,7 +332,8 @@ async function publish(category, suites, durationSec) {
332
332
  const root = findRoot(process.cwd());
333
333
  loadEnv(root);
334
334
  const cfg = loadConfig(root);
335
- const project = cfg.project, dashboardUrl = cfg.dashboardUrl, key = process.env.TESTFORGE_KEY;
335
+ // In CI the deployed dashboard comes from the environment; locally, from the config.
336
+ const project = cfg.project, dashboardUrl = process.env.TESTFORGE_DASHBOARD_URL || cfg.dashboardUrl, key = process.env.TESTFORGE_KEY;
336
337
  if (!project || !dashboardUrl || !key) {
337
338
  console.warn('[testforge] skipped publish — need "project" + "dashboardUrl" in testforge.config.json and TESTFORGE_KEY in .testforge.env');
338
339
  return;
@@ -343,7 +344,7 @@ async function publish(category, suites, durationSec) {
343
344
  const res = await fetch(dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
344
345
  method: 'POST',
345
346
  headers: { Authorization: 'Bearer ' + key, 'Content-Type': 'application/json' },
346
- body: JSON.stringify({ project, env, branch, sha, ciBuild, durationSec, suites }),
347
+ body: JSON.stringify({ project, env, branch, sha, ciBuild, durationSec, suites, runId: process.env.TESTFORGE_RUN_ID || undefined }),
347
348
  });
348
349
  if (res.ok) console.log(`[testforge] ✓ published ${category} run (env: ${env}${branch ? ` · ${branch}` : ''}) → ${dashboardUrl}`);
349
350
  else console.warn(`[testforge] publish failed (${res.status}): ${(await res.text()).slice(0, 200)}`);
@@ -352,4 +353,45 @@ async function publish(category, suites, durationSec) {
352
353
  }
353
354
  }
354
355
 
355
- module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot };
356
+ // A suite is named by its path under the test root — "booking-summary/keyboard-access" —
357
+ // because several journeys may each own a keyboard-access.test.ts. Keyed by basename
358
+ // alone, three such files became one suite of 22 and a single-journey run showed 13
359
+ // phantom skips. A relative path is kept as given (Playwright's JSON already reports
360
+ // spec files relative to testDir); an absolute path outside the root falls back to the
361
+ // bare file name.
362
+ function suiteName(file, root) {
363
+ let rel = String(file || 'suite').replace(/\\/g, '/');
364
+ if (path.isAbsolute(rel)) {
365
+ const r = root ? path.relative(root, rel).replace(/\\/g, '/') : '';
366
+ rel = r && !r.startsWith('..') ? r : rel.split('/').pop();
367
+ }
368
+ return rel.replace(/\.(test|spec)\.[jt]sx?$/, '');
369
+ }
370
+ // The configured test folder for a category (testOutput in testforge.config.json), absolute.
371
+ function testRoot(root, category) {
372
+ const out = loadConfig(root).testOutput;
373
+ const d = out && out[category];
374
+ return d ? path.resolve(root, d) : undefined;
375
+ }
376
+
377
+ // The defect review's verdict for each failed test (test-artifacts/defect-review/*.json),
378
+ // so the dashboard can say whether a failure is a product defect, a test fault or the
379
+ // environment. Keyed by absolute file + test title; an absent or broken file means none.
380
+ function reviewVerdicts(root) {
381
+ const map = new Map();
382
+ const dir = path.join(root || process.cwd(), 'test-artifacts', 'defect-review');
383
+ let files = [];
384
+ try { files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); } catch { /* no reviews yet */ }
385
+ for (const f of files) {
386
+ try {
387
+ for (const r of JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) || []) {
388
+ for (const t of r.tests || []) {
389
+ map.set(`${path.resolve(String(t.file))}|${t.title}`, { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) });
390
+ }
391
+ }
392
+ } catch { /* a half-written review is ignored */ }
393
+ }
394
+ return (file, title) => (file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined);
395
+ }
396
+
397
+ module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts };
@@ -1,16 +1,18 @@
1
1
  // TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to jest.config.js:
3
3
  // reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
4
- const { cleanError, sig, publish, testChecks, expectation, failureDetail } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgeJestReporter {
7
7
  constructor(_globalConfig, options) {
8
8
  this.category = (options && options.category) || 'api';
9
+ this.testRoot = testRoot(findRoot(process.cwd()), this.category);
10
+ this.verdictOf = reviewVerdicts(findRoot(process.cwd()));
9
11
  }
10
12
  async onRunComplete(_contexts, results) {
11
13
  const suites = {};
12
14
  for (const f of results.testResults || []) {
13
- const name = String(f.testFilePath || f.name || 'suite').split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
15
+ const name = suiteName(f.testFilePath || f.name || 'suite', this.testRoot);
14
16
  const su = suites[name] || (suites[name] = { name, category: this.category, tests: [] });
15
17
  for (const a of (f.testResults || f.assertionResults || [])) {
16
18
  const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
@@ -26,6 +28,7 @@ module.exports = class TestForgeJestReporter {
26
28
  checks: testChecks(f.testFilePath, a.location && a.location.line),
27
29
  ...(status === 'failed' ? expectation(raw) : {}),
28
30
  detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
31
+ ...(status === 'failed' ? this.verdictOf(f.testFilePath, a.fullName || a.title) : {}),
29
32
  });
30
33
  }
31
34
  }
@@ -1,7 +1,7 @@
1
1
  // TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to playwright.config.js reporter array:
3
3
  // ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
4
- const { cleanError, sig, publish, testChecks, failureDetail, findRoot } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgePlaywrightReporter {
7
7
  constructor(options) {
@@ -9,6 +9,13 @@ module.exports = class TestForgePlaywrightReporter {
9
9
  this.suites = {};
10
10
  this.start = Date.now();
11
11
  this.root = findRoot(process.cwd());
12
+ this.testRoot = testRoot(this.root, this.category);
13
+ this.verdictOf = reviewVerdicts(this.root);
14
+ }
15
+ onBegin(config) {
16
+ // Playwright's own testDir is the authority when it has one.
17
+ const td = config && config.projects && config.projects[0] && config.projects[0].testDir;
18
+ if (td) this.testRoot = td;
12
19
  }
13
20
  // Everything the runner knows about a failure beyond its message: where it
14
21
  // happened, the browser, the test's timeout and the artifacts it left.
@@ -26,7 +33,7 @@ module.exports = class TestForgePlaywrightReporter {
26
33
  }
27
34
  onTestEnd(test, result) {
28
35
  const file = (test.location && test.location.file) || 'suite';
29
- const name = String(file).split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
36
+ const name = suiteName(file, this.testRoot);
30
37
  const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
31
38
  const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
32
39
  const raw = result.error && result.error.message;
@@ -41,6 +48,7 @@ module.exports = class TestForgePlaywrightReporter {
41
48
  checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
42
49
  || testChecks(test.location && test.location.file, test.location && test.location.line),
43
50
  ...(status === 'failed' ? this.failed(test, result) : {}),
51
+ ...(status === 'failed' ? this.verdictOf(test.location && test.location.file, test.title) : {}),
44
52
  });
45
53
  }
46
54
  failed(test, result) {
@@ -0,0 +1,66 @@
1
+ // TestForge run data — one id per test run, shared by every worker, and a record
2
+ // of what the run created so it can be traced on the dashboard and cleaned later.
3
+ //
4
+ // The generated runner config calls runId() at load, so the main process sets
5
+ // TESTFORGE_RUN_ID before any worker starts and every worker inherits it.
6
+ const fs = require('fs');
7
+ const path = require('path');
8
+ const crypto = require('crypto');
9
+ const { findRoot } = require('./_shared.cjs');
10
+
11
+ const dir = (root) => path.join(root || findRoot(process.cwd()), 'test-artifacts', 'run-data');
12
+
13
+ /** The current run's id: <yyyymmdd>-<6 hex>, created once and kept in the env. */
14
+ function runId() {
15
+ if (!process.env.TESTFORGE_RUN_ID) {
16
+ process.env.TESTFORGE_RUN_ID = `${new Date().toISOString().slice(0, 10).replace(/-/g, '')}-${crypto.randomBytes(3).toString('hex')}`;
17
+ }
18
+ return process.env.TESTFORGE_RUN_ID;
19
+ }
20
+
21
+ /** A value no other run produces: label + the run id's last 6 characters (letters and digits only). */
22
+ function unique(label = '') {
23
+ return `${label}${runId().slice(-6)}`;
24
+ }
25
+
26
+ /**
27
+ * Record one thing the run created. entry: { kind, id, role?, cleanup?: { method, url } }
28
+ * — cleanup is the request that removes or cancels it, when the product has one.
29
+ * Appending one JSON line is safe from several workers at once.
30
+ */
31
+ function record(entry) {
32
+ const d = dir();
33
+ fs.mkdirSync(d, { recursive: true });
34
+ // The testkit that recorded it, so `testforge clean` signs in through the same one.
35
+ const kit = callerFile();
36
+ fs.appendFileSync(path.join(d, `${runId()}.jsonl`), JSON.stringify({ at: new Date().toISOString(), runId: runId(), ...(kit ? { kit } : {}), ...entry }) + '\n');
37
+ }
38
+
39
+ function callerFile() {
40
+ const lines = String(new Error().stack || '').split('\n').slice(1);
41
+ for (const l of lines) {
42
+ const m = /\(?((?:file:\/\/)?\/[^():]+):\d+:\d+\)?\s*$/.exec(l);
43
+ if (m && !m[1].endsWith('run-data.cjs')) return m[1].replace(/^file:\/\//, '');
44
+ }
45
+ return undefined;
46
+ }
47
+
48
+ /** Every record of one run. */
49
+ function readRun(id, root) {
50
+ try {
51
+ return fs.readFileSync(path.join(dir(root), `${id}.jsonl`), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
52
+ } catch {
53
+ return [];
54
+ }
55
+ }
56
+
57
+ /** Runs that recorded data, oldest first: { runId, at, count }. */
58
+ function listRuns(root) {
59
+ let files = [];
60
+ try { files = fs.readdirSync(dir(root)).filter((f) => f.endsWith('.jsonl')); } catch { return []; }
61
+ return files
62
+ .map((f) => { const id = f.slice(0, -6); const rows = readRun(id, root); return { runId: id, at: rows[0] ? rows[0].at : '', count: rows.length }; })
63
+ .sort((a, b) => (a.at < b.at ? -1 : 1));
64
+ }
65
+
66
+ module.exports = { runId, unique, record, readRun, listRuns };
@@ -0,0 +1,56 @@
1
+ // TestForge UI helpers for Playwright specs — shipped once, re-exported by every
2
+ // generated testkit, so no spec redefines (and diverges on) the same helper.
3
+ const settle = (ms) => new Promise((r) => setTimeout(r, ms));
4
+
5
+ /**
6
+ * Click a control once its layout has stopped moving. A pop-up whose content
7
+ * grows (a payment form opening extra fields, a list loading) scrolls the
8
+ * control while Playwright's own click is in flight, and the click lands on
9
+ * whatever is underneath. Scroll it into view, wait until its box is the same
10
+ * twice in a row, then click.
11
+ */
12
+ async function settledClick(locator, { settleMs = 300, tries = 10 } = {}) {
13
+ await locator.scrollIntoViewIfNeeded();
14
+ let prev = JSON.stringify(await locator.boundingBox());
15
+ for (let i = 0; i < tries; i++) {
16
+ await settle(settleMs);
17
+ const cur = JSON.stringify(await locator.boundingBox());
18
+ if (cur === prev) break;
19
+ prev = cur;
20
+ }
21
+ await locator.click();
22
+ }
23
+
24
+ /**
25
+ * Fill named inputs inside an iframe found by its name prefix (an embedded
26
+ * payment or address element). values: { inputName: value }. An input the
27
+ * frame does not show is skipped, so one call fits every variant of the form.
28
+ * Returns the names it filled.
29
+ */
30
+ async function fillFrameInputs(page, framePrefix, values) {
31
+ const frame = page.frameLocator(`iframe[name^="${framePrefix}"]`).first();
32
+ const filled = [];
33
+ for (const [name, value] of Object.entries(values)) {
34
+ const input = frame.locator(`input[name="${name}"]`);
35
+ if (await input.isVisible().catch(() => false)) {
36
+ await input.fill(String(value));
37
+ filled.push(name);
38
+ }
39
+ }
40
+ return filled;
41
+ }
42
+
43
+ /**
44
+ * Accept every browser dialog (alert / confirm) and keep its message, in order.
45
+ * Call before the action that raises it. Returns the live array of messages.
46
+ */
47
+ function watchDialogs(page) {
48
+ const messages = [];
49
+ page.on('dialog', (d) => {
50
+ messages.push(d.message());
51
+ d.accept().catch(() => {});
52
+ });
53
+ return messages;
54
+ }
55
+
56
+ module.exports = { settledClick, fillFrameInputs, watchDialogs };