@activate-core/test-forge 2.5.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -75,8 +75,9 @@ export function mapPlaywright(json, category = 'web') {
75
75
  for (const t of spec.tests || []) {
76
76
  const r = (t.results || [])[(t.results || []).length - 1] || {};
77
77
  const st = r.status || t.status;
78
- const status = st === 'passed' ? 'passed' : st === 'skipped' ? 'skipped' : 'failed';
79
78
  const raw = r.error && r.error.message;
79
+ const notProduct = st !== 'passed' && st !== 'skipped' ? shared.notTheProduct(raw) : undefined;
80
+ const status = st === 'passed' ? 'passed' : st === 'skipped' || notProduct ? 'skipped' : 'failed';
80
81
  const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
81
82
  const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
82
83
  const err = r.error || {};
@@ -95,8 +96,8 @@ export function mapPlaywright(json, category = 'web') {
95
96
  ...(detail ? { detail } : {}),
96
97
  error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
97
98
  sig: status === 'failed' ? shared.sig(raw) : undefined,
98
- note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
99
- ...(status === 'failed' ? verdictOf(loc, spec.title) : {}),
99
+ note: status === 'skipped' ? notProduct || (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
100
+ ...(status === 'failed' ? verdictOf(loc, spec.title, raw) : {}),
100
101
  });
101
102
  }
102
103
  }
@@ -114,14 +115,16 @@ export function mapJest(json, category = 'api', testDir) {
114
115
  const name = shared.suiteName(f.name || f.testFilePath || 'suite', testDir);
115
116
  const su = (suites[name] ||= { name, category, tests: [] });
116
117
  for (const a of f.assertionResults || []) {
117
- const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
118
118
  const raw = a.failureMessages && a.failureMessages[0];
119
+ const notProduct = a.status === 'failed' ? shared.notTheProduct(raw) : undefined;
120
+ const status = a.status === 'passed' ? 'passed' : a.status === 'failed' && !notProduct ? 'failed' : 'skipped';
119
121
  const err = strip(raw || '').split('\n').find((l) => l.trim()) || '';
120
122
  su.tests.push({
121
123
  name: a.title,
122
124
  status,
123
125
  error: status === 'failed' ? err.slice(0, 200) : undefined,
124
126
  sig: status === 'failed' ? shared.sig(raw) : undefined,
127
+ ...(notProduct ? { note: notProduct } : {}),
125
128
  });
126
129
  }
127
130
  }
@@ -12,8 +12,8 @@
12
12
  * It runs from the area's nearest package root (where package.json /
13
13
  * playwright.config live), with config.envFile merged into the env.
14
14
  */
15
- import { testFilesIn } from './readiness.js';
16
- import { scanNames } from './name-scan.js';
15
+ import { PENDING_STAMP, testFilesIn } from './readiness.js';
16
+ import { recheckFilled, scanNames } from './name-scan.js';
17
17
  import { INTERFACE_PATH, blankNames, neededNames } from '../generate/interface-file.js';
18
18
  import { harnessInfo } from '../generate/harness.js';
19
19
  import { missingPackages } from '../generate/verify.js';
@@ -111,6 +111,11 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
111
111
  export function runTests(opts = {}) {
112
112
  const config = loadConfig();
113
113
  const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
114
+ // An env file named on the command line that is not there is a mistake, not a default.
115
+ if (opts.envFile && !existsSync(opts.envFile)) {
116
+ console.log(`The env file "${opts.envFile}" does not exist — nothing was run.`);
117
+ return 1;
118
+ }
114
119
  const envVars = loadEnvFile(envFile);
115
120
  if (Object.keys(envVars).length > 0) {
116
121
  console.log(`Loaded env from ${envFile} (${Object.keys(envVars).length} vars).`);
@@ -156,8 +161,31 @@ export function runTests(opts = {}) {
156
161
  // says so. Before the run, the names those tests wait for are looked for
157
162
  // in the code — only where that kind of name lives, only when those lines
158
163
  // changed — and filled; the tests themselves are never touched.
159
- const targets = undefined;
164
+ let targets = undefined;
165
+ // A feature the doc says is not live yet: its files are held back, not run
166
+ // to fail. They run once the doc's Status line is removed and they are generated again.
167
+ {
168
+ const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
169
+ const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
170
+ const held = files.filter((f) => { try {
171
+ return PENDING_STAMP.test(readFileSync(f, 'utf-8'));
172
+ }
173
+ catch {
174
+ return false;
175
+ } });
176
+ if (held.length) {
177
+ const run = files.filter((f) => !held.includes(f));
178
+ console.log(` held back (feature not live yet): ${held.length} file(s)`);
179
+ if (!run.length) {
180
+ console.log(`[skip] ${area}: every test file waits for a feature that is not live yet`);
181
+ continue;
182
+ }
183
+ targets = run.map((f) => resolve(f));
184
+ }
185
+ }
160
186
  if (!opts.noScan) {
187
+ for (const n of recheckFilled(root))
188
+ console.log(` no longer in the code: ${n} — blanked; its tests wait until the new name is found`);
161
189
  const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
162
190
  const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
163
191
  const blank = blankNames([...new Set(files.flatMap((f) => neededNames(readFileSync(f, 'utf-8'))))]);
@@ -176,8 +204,16 @@ export function runTests(opts = {}) {
176
204
  ranAnything = true;
177
205
  console.log(`\n=== ${area} (${framework}) ===`);
178
206
  console.log(`$ ${cmd} [cwd: ${root}]`);
207
+ const started = Date.now();
179
208
  const res = spawnSync(cmd, { shell: true, cwd: root, env, stdio: 'inherit' });
180
- if (res.status !== 0)
209
+ // The reporters' counts: a run whose only "failures" are tests waiting for
210
+ // the product or stopped by the environment has no product failure.
211
+ const sum = runSummary(root, started);
212
+ if (sum) {
213
+ const ran = sum.passed + sum.failed;
214
+ console.log(`\n${area}: ${ran} of ${sum.total} test(s) ran — ${sum.passed} passed, ${sum.failed} failed; ${sum.skipped} did not run (waiting for the product, environment, or skipped)`);
215
+ }
216
+ if (res.status !== 0 && !(sum && sum.failed === 0 && sum.total > 0))
181
217
  anyFail = true;
182
218
  }
183
219
  if (!ranAnything) {
@@ -190,3 +226,16 @@ export function runTests(opts = {}) {
190
226
  console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
191
227
  return anyFail ? 1 : 0;
192
228
  }
229
+ /** Counts the reporters wrote for runs that started at or after `since`, summed. */
230
+ function runSummary(root, since) {
231
+ try {
232
+ const all = JSON.parse(readFileSync(join(root, 'testforge', 'last-run.json'), 'utf-8'));
233
+ const fresh = Object.values(all).filter((r) => Date.parse(r.at) >= since - 1000);
234
+ if (!fresh.length)
235
+ return undefined;
236
+ return fresh.reduce((a, r) => ({ total: a.total + r.total, passed: a.passed + r.passed, failed: a.failed + r.failed, skipped: a.skipped + r.skipped }), { total: 0, passed: 0, failed: 0, skipped: 0 });
237
+ }
238
+ catch {
239
+ return undefined;
240
+ }
241
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.5.0",
3
+ "version": "2.6.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -15,7 +15,8 @@
15
15
  "scripts": {
16
16
  "build": "rm -rf dist && tsc",
17
17
  "dev": "tsc --watch",
18
- "test": "tsx --test test/*.test.ts"
18
+ "test": "tsx --test test/*.test.ts",
19
+ "prepublishOnly": "npm run build && npm test"
19
20
  },
20
21
  "license": "MIT",
21
22
  "publishConfig": {
@@ -31,6 +31,7 @@ function loadEnv(root) {
31
31
  const k = s.slice(0, i).trim();
32
32
  let v = s.slice(i + 1).trim();
33
33
  if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
34
+ else v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
34
35
  if (k && process.env[k] === undefined) process.env[k] = v;
35
36
  }
36
37
  }
@@ -395,12 +396,19 @@ function reviewVerdicts(root) {
395
396
  try {
396
397
  for (const r of JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) || []) {
397
398
  for (const t of r.tests || []) {
398
- map.set(`${path.resolve(String(t.file))}|${t.title}`, { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) });
399
+ map.set(`${path.resolve(String(t.file))}|${t.title}`, { at: sig(t.detail), v: { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) } });
399
400
  }
400
401
  }
401
402
  } catch { /* a half-written review is ignored */ }
402
403
  }
403
- return (file, title) => (file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined);
404
+ // A verdict is about one failure: the same test failing differently later is a
405
+ // new failure, and an old "test fault" must not hide a real regression.
406
+ return (file, title, raw) => {
407
+ const hit = file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined;
408
+ if (!hit) return undefined;
409
+ if (raw !== undefined && hit.at && sig(raw) !== hit.at) return undefined;
410
+ return hit.v;
411
+ };
404
412
  }
405
413
 
406
414
  /** The name a test is still waiting for (need() threw for a blank interface name), else undefined. */
@@ -431,4 +439,49 @@ function notTheProduct(raw) {
431
439
  return environmentFor(raw);
432
440
  }
433
441
 
434
- module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
442
+ /**
443
+ * Remember the tests that passed (testforge/passed.json): generate never removes
444
+ * or rewrites a test that has passed. Add-only — a later failure is a regression
445
+ * to report, not a reason to forget that the test once checked the product.
446
+ */
447
+ function recordPassed(root, passed) {
448
+ if (!passed.length) return;
449
+ const file = path.join(root, 'testforge', 'passed.json');
450
+ let all = {};
451
+ try { all = JSON.parse(fs.readFileSync(file, 'utf-8')) || {}; } catch { all = {}; }
452
+ let changed = false;
453
+ for (const t of passed) {
454
+ if (!t.file || !t.title) continue;
455
+ const rel = path.relative(root, path.resolve(t.file)).split(path.sep).join('/');
456
+ const list = Array.isArray(all[rel]) ? all[rel] : (all[rel] = []);
457
+ if (!list.includes(t.title)) { list.push(t.title); changed = true; }
458
+ }
459
+ if (!changed) return;
460
+ try {
461
+ fs.mkdirSync(path.dirname(file), { recursive: true });
462
+ fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n');
463
+ } catch (e) {
464
+ console.log(`[testforge] could not record passed tests: ${e.message}`);
465
+ }
466
+ }
467
+
468
+ /**
469
+ * The run's counts, for `testforge run` to read: how many ran of how many, and
470
+ * whether anything failed for a product reason. Waiting and environment stops
471
+ * are "skipped" here, as on the dashboard — never a failure of the run.
472
+ */
473
+ function recordSummary(root, category, suites) {
474
+ const tests = suites.flatMap((s) => s.tests || []);
475
+ const n = (st) => tests.filter((t) => t.status === st).length;
476
+ const out = { category, at: new Date().toISOString(), total: tests.length, passed: n('passed'), failed: n('failed'), skipped: n('skipped') };
477
+ try {
478
+ const file = path.join(root, 'testforge', 'last-run.json');
479
+ let all = {};
480
+ try { all = JSON.parse(fs.readFileSync(file, 'utf-8')) || {}; } catch { all = {}; }
481
+ all[category] = out;
482
+ fs.mkdirSync(path.dirname(file), { recursive: true });
483
+ fs.writeFileSync(file, JSON.stringify(all, null, 2) + '\n');
484
+ } catch { /* the run's own output still has the counts */ }
485
+ }
486
+
487
+ module.exports = { recordSummary, recordPassed, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
@@ -134,6 +134,10 @@ function blockTestsThatUse(message, name) {
134
134
  try { fullName = globalThis.expect.getState().currentTestName; } catch { /* unknown */ }
135
135
  if (needs(undefined, fullName)) throw new Error(message);
136
136
  });
137
+ // Used outside the tests (a hook or a helper): every test waits, so the
138
+ // file's own hooks must not run either, or a beforeAll/afterEach would
139
+ // fail on the placeholder and hide the "waiting" reason.
140
+ if (!users) quietHooks();
137
141
  return true;
138
142
  }
139
143
  require('@playwright/test').test.beforeEach(async ({}, testInfo) => { // Playwright
@@ -145,6 +149,16 @@ function blockTestsThatUse(message, name) {
145
149
  }
146
150
  }
147
151
 
152
+ /**
153
+ * Jest: the hooks this test file registers from here on do nothing. Each test
154
+ * file has its own globals, so no other file is touched.
155
+ */
156
+ function quietHooks() {
157
+ for (const h of ['beforeAll', 'afterAll', 'beforeEach', 'afterEach']) {
158
+ if (typeof globalThis[h] === 'function') globalThis[h] = () => {};
159
+ }
160
+ }
161
+
148
162
  /** The product's value for `name`, or an error naming what to fill. */
149
163
  function need(name) {
150
164
  const v = values()[name];
@@ -1,13 +1,14 @@
1
1
  // TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to jest.config.js:
3
3
  // reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
4
- const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle, recordPassed, recordSummary } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgeJestReporter {
7
7
  constructor(_globalConfig, options) {
8
8
  this.category = (options && options.category) || 'api';
9
9
  this.testRoot = testRoot(findRoot(process.cwd()), this.category);
10
- this.verdictOf = reviewVerdicts(findRoot(process.cwd()));
10
+ this.root = findRoot(process.cwd());
11
+ this.verdictOf = reviewVerdicts(this.root);
11
12
  }
12
13
  async onRunComplete(_contexts, results) {
13
14
  const suites = {};
@@ -31,12 +32,15 @@ module.exports = class TestForgeJestReporter {
31
32
  checks: testChecks(f.testFilePath, a.location && a.location.line),
32
33
  ...(status === 'failed' ? expectation(raw) : {}),
33
34
  detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
34
- ...(status === 'failed' ? this.verdictOf(f.testFilePath, a.fullName || a.title) : {}),
35
+ ...(status === 'failed' ? this.verdictOf(f.testFilePath, a.fullName || a.title, raw) : {}),
35
36
  });
36
37
  }
37
38
  }
39
+ recordPassed(this.root, (results.testResults || []).flatMap((f) =>
40
+ (f.testResults || f.assertionResults || []).filter((a) => a.status === 'passed').map((a) => ({ file: f.testFilePath, title: a.title }))));
38
41
  const durationSec = Math.round((results.startTime ? Date.now() - results.startTime : 0) / 1000);
39
42
  const list = Object.values(suites);
43
+ recordSummary(this.root, this.category, list);
40
44
  // Same reasoning as the Playwright reporter: no suites means nothing ran,
41
45
  // and a published 0/0/0 run reads as a clean result.
42
46
  if (!list.length) {
@@ -1,12 +1,13 @@
1
1
  // TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to playwright.config.js reporter array:
3
3
  // ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
4
- const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle, recordPassed, recordSummary } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgePlaywrightReporter {
7
7
  constructor(options) {
8
8
  this.category = (options && options.category) || 'web';
9
9
  this.suites = {};
10
+ this.passed = [];
10
11
  this.start = Date.now();
11
12
  this.root = findRoot(process.cwd());
12
13
  this.testRoot = testRoot(this.root, this.category);
@@ -41,6 +42,7 @@ module.exports = class TestForgePlaywrightReporter {
41
42
  const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' || notProduct ? 'skipped' : 'failed';
42
43
  const pend = (result.annotations || []).concat(test.annotations || []).filter((x) => x.type === 'skip' || x.type === 'fixme');
43
44
  const skipAnno = pend.find((x) => x.description) || pend[0];
45
+ if (status === 'passed') this.passed.push({ file, title: test.title });
44
46
  su.tests.push({
45
47
  name: plainTitle(test.title),
46
48
  status,
@@ -50,7 +52,7 @@ module.exports = class TestForgePlaywrightReporter {
50
52
  checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
51
53
  || testChecks(test.location && test.location.file, test.location && test.location.line),
52
54
  ...(status === 'failed' ? this.failed(test, result) : {}),
53
- ...(status === 'failed' ? this.verdictOf(test.location && test.location.file, test.title) : {}),
55
+ ...(status === 'failed' ? this.verdictOf(test.location && test.location.file, test.title, raw) : {}),
54
56
  });
55
57
  }
56
58
  failed(test, result) {
@@ -63,7 +65,9 @@ module.exports = class TestForgePlaywrightReporter {
63
65
  }
64
66
  async onEnd(result) {
65
67
  const durationSec = Math.round(((result && result.duration) || (Date.now() - this.start)) / 1000);
68
+ recordPassed(this.root, this.passed);
66
69
  const suites = Object.values(this.suites);
70
+ recordSummary(this.root, this.category, suites);
67
71
  // A run that collected no suites did not test anything: global setup failed,
68
72
  // or the filter matched nothing. Publishing it records a 0/0/0 result that
69
73
  // reads as "the suite ran and found nothing wrong" — the most misleading
@@ -19,8 +19,13 @@ function runId() {
19
19
  }
20
20
 
21
21
  /** A value no other run produces: label + the run id's last 6 characters (letters and digits only). */
22
+ const uses = new Map();
22
23
  function unique(label = '') {
23
- return `${label}${runId().slice(-6)}`;
24
+ // The same label asked for again in one test file gets a new value: two
25
+ // rows built from one label (an email, a name) must not collide.
26
+ const n = (uses.get(label) || 0) + 1;
27
+ uses.set(label, n);
28
+ return `${label}${runId().slice(-6)}${n > 1 ? n : ''}`;
24
29
  }
25
30
 
26
31
  /**