@activate-core/test-forge 1.0.4 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,12 +12,14 @@
12
12
  * is the repair prompt: the runner's own error is far better evidence than
13
13
  * anything we could infer.
14
14
  */
15
- import { spawnSync } from 'node:child_process';
16
- import { existsSync, readFileSync } from 'node:fs';
15
+ import { spawnSync, spawn } from 'node:child_process';
16
+ import { existsSync, readFileSync, mkdtempSync, rmSync, openSync, closeSync } from 'node:fs';
17
+ import { tmpdir } from 'node:os';
17
18
  import { createRequire } from 'node:module';
18
- import { join, relative } from 'node:path';
19
+ import { join, relative, dirname, resolve } from 'node:path';
19
20
  import { loadEnvFile, testDirForType, resolveFramework, } from '../project/config.js';
20
21
  import { runnerCommand, packageRoot } from '../run/run-tests.js';
22
+ import { DEFECT_MARK } from './ai-generator.js';
21
23
  /** How much runner output to carry into a repair prompt. */
22
24
  const MAX_OUTPUT = 12_000;
23
25
  /**
@@ -30,21 +32,6 @@ function trimOutput(s) {
30
32
  ? clean.trim()
31
33
  : `…(${clean.length - MAX_OUTPUT} chars trimmed)…\n${clean.slice(-MAX_OUTPUT).trim()}`;
32
34
  }
33
- /**
34
- * What the runner actually executed. Both Jest and Playwright print these
35
- * counts; anything we cannot read comes back null and is not guessed at.
36
- */
37
- function countsFrom(output) {
38
- const n = (re) => {
39
- const m = output.match(re);
40
- return m ? Number(m[1]) : null;
41
- };
42
- const passed = n(/(\d+)\s+passed/i) ?? n(/Tests:.*?(\d+)\s+passed/i);
43
- const skipped = n(/(\d+)\s+skipped/i) ?? n(/(\d+)\s+todo/i) ?? n(/(\d+)\s+pending/i);
44
- if (passed === null && skipped === null)
45
- return null;
46
- return { passed: passed ?? 0, skipped: skipped ?? 0 };
47
- }
48
35
  /**
49
36
  * Failures no test edit can fix. Sending one to the repair prompt burns AI
50
37
  * calls rewriting correct tests against an error they cannot cause (one
@@ -78,70 +65,206 @@ export function browserPreflight(root) {
78
65
  const output = `${res.stdout ?? ''}\n${res.stderr ?? ''}`;
79
66
  return infraFailure(output) ?? (output.trim().split('\n').find((l) => l.trim()) || 'the test browser could not start');
80
67
  }
68
+ const stripAnsi = (s) => s.replace(/\u001b\[[0-9;]*m/g, '');
69
+ /** Results of a Playwright JSON report (`--reporter=json`). */
70
+ export function fromPlaywright(json) {
71
+ const out = { ran: true, passed: 0, passes: [], failures: [], skips: [] };
72
+ // Spec files are recorded relative to the config's test directory.
73
+ const at = (f) => resolve(json.config?.rootDir ?? '.', f ?? '');
74
+ let specs = 0;
75
+ const walk = (suite) => {
76
+ for (const spec of suite.specs ?? []) {
77
+ for (const t of spec.tests ?? []) {
78
+ specs += 1;
79
+ const r = (t.results ?? [])[(t.results ?? []).length - 1] ?? {};
80
+ const status = r.status ?? t.status;
81
+ if (status === 'passed') {
82
+ out.passed += 1;
83
+ out.passes.push({ file: at(spec.file), line: spec.line, title: spec.title });
84
+ }
85
+ else if (status === 'skipped') {
86
+ const anns = [...(r.annotations ?? []), ...(t.annotations ?? [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
87
+ // A declared pending test's annotation points at the declaration
88
+ // itself; a skip the test decided while running points at the call
89
+ // inside its body. Only the second kind says anything about this run.
90
+ const runtime = anns.find((a) => a.location && (a.location.line !== spec.line || a.location.column !== spec.column));
91
+ if (runtime || !anns.length) {
92
+ out.skips.push({
93
+ file: at(spec.file),
94
+ line: spec.line,
95
+ title: spec.title,
96
+ detail: runtime?.description || (anns.length ? '' : 'not run — an earlier test in a serial block failed'),
97
+ });
98
+ }
99
+ }
100
+ else {
101
+ const msg = stripAnsi(r.error?.message ?? (r.errors ?? []).map((e) => e.message).join('\n'));
102
+ out.failures.push({ file: at(spec.file), line: spec.line, title: spec.title, detail: msg.split('\n').slice(0, 40).join('\n') });
103
+ }
104
+ }
105
+ }
106
+ for (const s of suite.suites ?? [])
107
+ walk(s);
108
+ };
109
+ for (const s of json.suites ?? [])
110
+ walk(s);
111
+ if (!specs) {
112
+ const errors = (json.errors ?? []).map((e) => stripAnsi(e.message ?? '')).join('\n').trim();
113
+ if (!errors)
114
+ return null;
115
+ return { ran: false, reason: `the suite could not start: ${errors.split('\n')[0]}`, passed: 0, passes: [], failures: [], skips: [], infra: infraFailure(errors) ?? undefined };
116
+ }
117
+ return out;
118
+ }
119
+ /** Results of a Jest JSON report (`--json --testLocationInResults`). */
120
+ function fromJest(json) {
121
+ const out = { ran: true, passed: 0, passes: [], failures: [], skips: [] };
122
+ const files = json.testResults ?? [];
123
+ if (!files.length)
124
+ return null;
125
+ for (const f of files) {
126
+ if (!(f.assertionResults ?? []).length && f.message) {
127
+ return { ran: false, reason: `the suite could not start: ${stripAnsi(f.message).trim().split('\n')[0]}`, passed: 0, passes: [], failures: [], skips: [], infra: infraFailure(f.message) ?? undefined };
128
+ }
129
+ for (const a of f.assertionResults ?? []) {
130
+ if (a.status === 'passed') {
131
+ out.passed += 1;
132
+ out.passes.push({ file: f.name, line: a.location?.line ?? 0, title: a.fullName ?? a.title });
133
+ }
134
+ else if (a.status === 'failed') {
135
+ out.failures.push({
136
+ file: f.name,
137
+ line: a.location?.line ?? 0,
138
+ title: a.fullName ?? a.title,
139
+ detail: stripAnsi((a.failureMessages ?? []).join('\n')).split('\n').slice(0, 40).join('\n'),
140
+ });
141
+ }
142
+ // pending / todo are declared in the file, not decided by this run
143
+ }
144
+ }
145
+ return out;
146
+ }
81
147
  /**
82
- * Run exactly the given generated files. Never throws: a verify step that
83
- * breaks generation would be worse than no verify step at all.
148
+ * Run ONE generated file and read its results from the runner's own JSON
149
+ * report — per test, not per journey. Never throws.
150
+ *
151
+ * Per file because a journey can hold hundreds of tests: one run for all of
152
+ * them outlived any sane timeout, the step reported "not verified" once, and
153
+ * a whole suite reached the project without a single test having run. Per
154
+ * test because a journey's repair must touch only the files that failed, and
155
+ * must be told which failures they were.
84
156
  */
85
- export function verifyFiles(config, area, files) {
86
- if (!files.length)
87
- return { ran: false, ok: false, output: '', reason: 'no files to verify' };
157
+ export function verifyFile(config, area, file) {
158
+ return verifyRun(config, area, [file]);
159
+ }
160
+ /**
161
+ * Run several files in ONE runner invocation, the way a team or CI runs a
162
+ * suite. Files that pass one at a time can still break each other when they
163
+ * share a run — one signs in as the shared account and ends the session
164
+ * another relies on — and only a run together shows it. Never throws.
165
+ */
166
+ export async function verifyRun(config, area, files, timeoutMs) {
167
+ const none = (reason, infra) => ({ ran: false, reason, infra, passed: 0, passes: [], failures: [], skips: [] });
168
+ const present = files.filter((f) => existsSync(f));
169
+ if (!present.length)
170
+ return none('the file is not on disk');
88
171
  const testDir = testDirForType(config, area);
89
172
  const root = packageRoot(testDir);
90
173
  const rel = relative(root, testDir) || '.';
91
174
  const framework = resolveFramework(config, area, area);
92
- const present = files.filter((f) => existsSync(f));
93
- if (!present.length) {
94
- return { ran: false, ok: false, output: '', reason: 'generated files are not on disk' };
95
- }
96
- const cmd = runnerCommand(config, area, framework, root, rel, undefined, present);
97
175
  const envFile = config.envFile ?? '.env.testing';
98
176
  const envVars = loadEnvFile(envFile);
99
177
  if (!Object.keys(envVars).length && !existsSync(envFile)) {
100
- return {
101
- ran: false,
102
- ok: false,
103
- output: '',
104
- reason: `no ${envFile} — cannot reach the app to verify`,
105
- };
178
+ return none(`no ${envFile} — cannot reach the app to verify`);
106
179
  }
107
- const res = spawnSync(cmd, {
108
- shell: true,
109
- cwd: root,
110
- env: { ...process.env, ...envVars },
111
- encoding: 'utf-8',
112
- timeout: config.verifyTimeoutMs ?? 600_000,
113
- maxBuffer: 32 * 1024 * 1024,
114
- });
115
- if (res.error) {
116
- return { ran: false, ok: false, output: '', reason: res.error.message };
180
+ // The project's own testCommand is for whole runs; here we name the files
181
+ // and ask for a machine-readable report (which also keeps a verify run from
182
+ // being published to the dashboard as if it were a real one).
183
+ const base = runnerCommand({ ...config, testCommand: undefined }, area, framework, root, rel, undefined, present);
184
+ const report = join(mkdtempSync(join(tmpdir(), 'tf-verify-')), 'report.json');
185
+ const timeout = timeoutMs ?? config.verifyTimeoutMs ?? 600_000;
186
+ // The runner enforces the limit itself where it can: Playwright stops at its
187
+ // global timeout and still writes the report for the tests that ran. The
188
+ // process limit below is only the backstop — and it must end the WHOLE tree:
189
+ // killing the shell alone left the runner and its browser alive, and a
190
+ // verify waited on them for hours.
191
+ const out = join(dirname(report), 'output.txt');
192
+ const cmd = framework === 'playwright'
193
+ ? `${base} --reporter=json --global-timeout=${timeout}`
194
+ : `${base} --json --testLocationInResults --forceExit --outputFile=${report}`;
195
+ const res = await runTree(cmd, out, { cwd: root, env: { ...process.env, ...envVars, PLAYWRIGHT_JSON_OUTPUT_NAME: report } }, timeout + 120_000);
196
+ let raw = '';
197
+ try {
198
+ raw = readFileSync(out, 'utf-8');
117
199
  }
118
- const output = trimOutput(`${res.stdout ?? ''}\n${res.stderr ?? ''}`);
119
- // A runner that could not start at all (no binary, bad config) is not a
120
- // failing test — reporting it as one would send the AI chasing a ghost.
121
- if (res.status === null) {
122
- return { ran: false, ok: false, output, reason: 'runner timed out' };
200
+ catch {
201
+ // no output at all
202
+ }
203
+ const output = trimOutput(raw);
204
+ if (res.timedOut) {
205
+ return none(`timed out after ${Math.round(timeout / 60000)} min (raise verifyTimeoutMs)`);
123
206
  }
124
207
  if (/command not found|Cannot find module '@playwright|is not recognized/i.test(output)) {
125
- return { ran: false, ok: false, output, reason: 'test runner is not installed' };
208
+ return none('test runner is not installed', 'test runner is not installed');
126
209
  }
127
- const infra = infraFailure(output);
128
- if (infra)
129
- return { ran: false, infra: true, ok: false, output, reason: infra };
130
- if (res.status !== 0)
131
- return { ran: true, ok: false, output };
132
- // A suite where every test skipped ALSO exits 0. Treating that as verified
133
- // is the exact dishonesty this step exists to remove: the seed could not be
134
- // established, or a precondition was not met, and nothing was checked at
135
- // all. Report it as a failure, with the reason the runner already printed.
136
- const counts = countsFrom(output);
137
- if (counts && counts.passed === 0 && counts.skipped > 0) {
138
- return {
139
- ran: true,
140
- ok: false,
141
- output: `Every test was SKIPPED (${counts.skipped} skipped, 0 passed) — nothing was actually verified.\nThe precondition or seed below could not be established; fix that so the tests run.\n\n${output}`,
142
- };
210
+ let parsed = null;
211
+ try {
212
+ const json = JSON.parse(readFileSync(report, 'utf-8'));
213
+ parsed = framework === 'playwright' ? fromPlaywright(json) : fromJest(json);
214
+ }
215
+ catch {
216
+ parsed = null;
217
+ }
218
+ finally {
219
+ rmSync(dirname(report), { recursive: true, force: true });
143
220
  }
144
- return { ran: true, ok: true, output };
221
+ if (!parsed) {
222
+ const infra = infraFailure(output);
223
+ return none(infra ?? (output.split('\n').find((l) => l.trim()) || 'the runner produced no results'), infra ?? undefined);
224
+ }
225
+ // Environment failures are judged per test: one test that stubs a refused
226
+ // connection on purpose must not make the whole file look like a dead app.
227
+ if (parsed.failures.length && parsed.failures.every((f) => infraFailure(f.detail))) {
228
+ parsed.infra = infraFailure(parsed.failures[0].detail) ?? undefined;
229
+ }
230
+ return parsed;
231
+ }
232
+ /**
233
+ * Run a shell command in its own process group, output to a file, and kill the
234
+ * whole group if it outlives the limit — the runner, its workers and browsers
235
+ * included, not just the shell that started them.
236
+ */
237
+ export function runTree(cmd, outFile, opts, limitMs) {
238
+ return new Promise((done) => {
239
+ const fd = openSync(outFile, 'w');
240
+ const child = spawn('sh', ['-c', cmd], { cwd: opts.cwd, env: opts.env, detached: true, stdio: ['ignore', fd, fd] });
241
+ let timedOut = false;
242
+ const timer = setTimeout(() => {
243
+ timedOut = true;
244
+ try {
245
+ if (child.pid)
246
+ process.kill(-child.pid, 'SIGKILL');
247
+ }
248
+ catch {
249
+ // already gone
250
+ }
251
+ }, limitMs);
252
+ const finish = (code) => {
253
+ clearTimeout(timer);
254
+ closeSync(fd);
255
+ done({ code, timedOut });
256
+ };
257
+ child.on('error', () => finish(null));
258
+ child.on('close', (code) => finish(code));
259
+ });
260
+ }
261
+ /**
262
+ * A skip that says only that the environment lacks something — an env var,
263
+ * an opt-in flag — is the environment's to fix. Any other runtime skip is the
264
+ * test reporting something it observed, and its own setup is a suspect.
265
+ */
266
+ export function environmentSkip(reason) {
267
+ return /\b[A-Z][A-Z0-9_]{2,}\b[^.]*\b(not set|is not set|unset|missing|absent|empty|not configured)\b|\b(set|provide|export)\s+[A-Z][A-Z0-9_]{2,}\b|TESTFORGE_ALLOW_/.test(reason);
145
268
  }
146
269
  /**
147
270
  * Packages the generated harness declares it needs (REQUIRED_PACKAGES) that
@@ -170,3 +293,40 @@ export function missingPackages(harnessFile, root) {
170
293
  }
171
294
  });
172
295
  }
296
+ /**
297
+ * Declaration lines (1-based) of tests a repair judged to be failing on the
298
+ * application, with the claim it wrote — read from the comment directly above.
299
+ */
300
+ export function claimedDefects(content) {
301
+ const lines = content.split('\n');
302
+ const claims = new Map();
303
+ for (let i = 0; i < lines.length; i++) {
304
+ if (!/\b(?:test|it)(?:\.\w+)?\s*\(/.test(lines[i]))
305
+ continue;
306
+ for (let j = i - 1; j >= 0 && lines[j].trim().startsWith('//'); j--) {
307
+ const at = lines[j].indexOf(DEFECT_MARK);
308
+ if (at !== -1)
309
+ claims.set(i + 1, lines[j].slice(at + DEFECT_MARK.length).trim());
310
+ }
311
+ }
312
+ return claims;
313
+ }
314
+ const titles = (s) => new Set([...s.matchAll(/\b(?:test|it)(?:\.(?:only|fixme|skip|fail|slow))?\s*\(\s*(['"`])((?:\\.|(?!\1)[\s\S])*?)\1/g)].map((m) => m[2]));
315
+ const count = (s, re) => (s.match(re) || []).length;
316
+ /**
317
+ * What a repair may not do, checked mechanically rather than trusted: drop or
318
+ * rename a test, add a skip, or lose an assertion. Each of those turns a
319
+ * caught defect into a silent one. Returns why the repair is refused, or null.
320
+ */
321
+ export function repairGuard(before, after) {
322
+ const gone = [...titles(before)].filter((t) => !titles(after).has(t));
323
+ if (gone.length)
324
+ return `it removed or renamed ${gone.length} test(s), e.g. "${gone[0].slice(0, 60)}"`;
325
+ const skips = /\b(?:test|it)\.(?:skip|fixme)\s*\(/g;
326
+ if (count(after, skips) > count(before, skips))
327
+ return 'it added a skip';
328
+ const expects = /\bexpect(?:\.soft)?\s*\(/g;
329
+ if (count(after, expects) < count(before, expects))
330
+ return 'it removed an assertion';
331
+ return null;
332
+ }
@@ -27,7 +27,9 @@ function extractEnvVars(text) {
27
27
  vars.add(m[1]);
28
28
  }
29
29
  // A generated suite often reads env through a small helper: env('TEST_X', default).
30
- for (const m of text.matchAll(/\benv\(\s*['"]([A-Z][A-Z0-9_]+)['"]/g)) {
30
+ // The helper's name varies (env, envVal, readEnv…), so take any call whose
31
+ // first argument is an env-shaped name.
32
+ for (const m of text.matchAll(/\b[A-Za-z_$][\w$]*\(\s*['"]([A-Z][A-Z0-9]*_[A-Z0-9_]+)['"]/g)) {
31
33
  vars.add(m[1]);
32
34
  }
33
35
  // backtick-wrapped ALL_CAPS names in "env var" lines
@@ -7,8 +7,13 @@
7
7
  */
8
8
  import fs from 'node:fs';
9
9
  import path from 'node:path';
10
+ import { createRequire } from 'node:module';
11
+ import { packageRoot } from './run-tests.js';
10
12
  import os from 'node:os';
11
13
  import { execSync } from 'node:child_process';
14
+ // The reporters' helpers, so a report published from a results file reads the
15
+ // same as one the reporter published during the run.
16
+ const shared = createRequire(import.meta.url)('../../../reporters/_shared.cjs');
12
17
  /**
13
18
  * Load TESTFORGE_KEY (and any other vars) from a gitignored `.testforge.env`
14
19
  * (then `.env`) at the project root, so a developer can keep the key in a file
@@ -16,7 +21,14 @@ import { execSync } from 'node:child_process';
16
21
  * environment (a real `export` still wins). Add `.testforge.env` to .gitignore.
17
22
  */
18
23
  export function loadDotenv(cwd = process.cwd()) {
19
- for (const file of ['.testforge.env', '.env']) {
24
+ let envFile = '';
25
+ try {
26
+ envFile = JSON.parse(fs.readFileSync(path.join(cwd, 'testforge.config.json'), 'utf8')).envFile || '';
27
+ }
28
+ catch {
29
+ // no config — the fixed names below still apply
30
+ }
31
+ for (const file of [envFile, '.testforge.env', '.env'].filter(Boolean)) {
20
32
  const p = path.join(cwd, file);
21
33
  if (!fs.existsSync(p))
22
34
  continue;
@@ -50,20 +62,39 @@ export function loadReportConfig(cwd = process.cwd()) {
50
62
  return {};
51
63
  }
52
64
  const strip = (s) => (s || '').replace(/\x1b\[[0-9;]*m/g, '');
53
- /** Bucket a failure message into a root-cause signature the dashboard clusters on. */
54
- function signature(msg) {
55
- const m = strip(msg || '').replace(/\n/g, ' ');
56
- if (/403|whoami|invalid_grant|refresh token/i.test(m))
57
- return 'IM auth (403)';
58
- if (/\$gt|\$exists|injection/i.test(m))
59
- return 'NoSQL injection accepted';
60
- if (/timeout|timed out|ETIMEDOUT/i.test(m))
61
- return 'Timeout';
62
- if (/\b500\b|Internal Server/i.test(m))
63
- return 'Server 500';
64
- if (/expect|toBe|toEqual|Received|Expected/i.test(m))
65
- return 'Assertion mismatch';
66
- return 'Uncategorised';
65
+ /** Map a Playwright `--reporter=json` report to the dashboard run contract. */
66
+ export function mapPlaywright(json, category = 'web') {
67
+ const suites = {};
68
+ // Spec files are relative to the config's testDir; the report records the root it used.
69
+ const rootDir = (json.config && json.config.rootDir) || process.cwd();
70
+ const walk = (suite) => {
71
+ for (const spec of suite.specs || []) {
72
+ const name = String(spec.file || suite.file || 'suite').split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
73
+ const su = (suites[name] ||= { name, category, tests: [] });
74
+ for (const t of spec.tests || []) {
75
+ const r = (t.results || [])[(t.results || []).length - 1] || {};
76
+ const st = r.status || t.status;
77
+ const status = st === 'passed' ? 'passed' : st === 'skipped' ? 'skipped' : 'failed';
78
+ const raw = r.error && r.error.message;
79
+ const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
80
+ const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
81
+ su.tests.push({
82
+ name: spec.title,
83
+ status,
84
+ checks: shared.testChecks(loc, spec.line),
85
+ ...(status === 'failed' ? shared.expectation(raw) : {}),
86
+ error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
87
+ sig: status === 'failed' ? shared.sig(raw) : undefined,
88
+ note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
89
+ });
90
+ }
91
+ }
92
+ for (const c of suite.suites || [])
93
+ walk(c);
94
+ };
95
+ for (const s of json.suites || [])
96
+ walk(s);
97
+ return { suites: Object.values(suites), durationSec: Math.round(((json.stats && json.stats.duration) || 0) / 1000) };
67
98
  }
68
99
  /** Map a Jest `--json` report to the dashboard run contract. */
69
100
  export function mapJest(json, category = 'api') {
@@ -81,7 +112,7 @@ export function mapJest(json, category = 'api') {
81
112
  name: a.title,
82
113
  status,
83
114
  error: status === 'failed' ? err.slice(0, 200) : undefined,
84
- sig: status === 'failed' ? signature(raw) : undefined,
115
+ sig: status === 'failed' ? shared.sig(raw) : undefined,
85
116
  });
86
117
  }
87
118
  }
@@ -90,8 +121,11 @@ export function mapJest(json, category = 'api') {
90
121
  }
91
122
  export async function publishReport(opts) {
92
123
  const json = JSON.parse(fs.readFileSync(opts.resultsPath, 'utf8'));
93
- const { suites, durationSec } = mapJest(json, opts.category || 'api');
94
- const body = { project: opts.project, env: opts.env || 'dev', durationSec, suites };
124
+ // Jest's report has testResults; Playwright's has suites.
125
+ const { suites, durationSec } = Array.isArray(json.testResults)
126
+ ? mapJest(json, opts.category || 'api')
127
+ : mapPlaywright(json, opts.category || 'web');
128
+ const body = { project: opts.project, env: opts.env || process.env.TESTFORGE_ENV || 'local', durationSec, suites };
95
129
  const res = await fetch(opts.dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
96
130
  method: 'POST',
97
131
  headers: { Authorization: `Bearer ${opts.apiKey}`, 'Content-Type': 'application/json' },
@@ -320,8 +354,9 @@ export function scanInventory(cwd, testOutput, ext, framework = {}) {
320
354
  if (!fs.existsSync(dir))
321
355
  continue;
322
356
  if (framework[category] === 'playwright') {
323
- // package root (holds playwright.config) is the parent of the tests dir
324
- if (playwrightList(path.dirname(dir), category, suites)) {
357
+ // The config sits at the package root (or, beside a project's own
358
+ // config, in the tests' parent folder) — list from wherever it is.
359
+ if (playwrightList(packageRoot(dir), category, suites) || playwrightList(path.dirname(dir), category, suites)) {
325
360
  attachPending(dir, ext, suites);
326
361
  continue;
327
362
  }
@@ -35,8 +35,17 @@ export function packageRoot(dir) {
35
35
  * journey); without it the whole area runs.
36
36
  */
37
37
  export function runnerCommand(config, area, framework, root, rel, journey, targets) {
38
- const explicit = config.testCommand?.[area];
38
+ let explicit = config.testCommand?.[area];
39
39
  const paths = (targets ?? []).map((t) => relative(root, t) || t);
40
+ if (explicit && (paths.length || journey)) {
41
+ // A command that already names the whole test folder ("npx playwright
42
+ // test tests/e2e") would still run all of it next to a narrower path:
43
+ // the runner treats every path argument as one more filter to include.
44
+ explicit = explicit
45
+ .split(/\s+/)
46
+ .filter((tok) => tok.replace(/^\.\//, '').replace(/\/$/, '') !== rel)
47
+ .join(' ');
48
+ }
40
49
  if (explicit) {
41
50
  if (paths.length)
42
51
  return `${explicit} ${paths.join(' ')}`;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "1.0.4",
3
+ "version": "1.2.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -17,9 +17,10 @@ function findRoot(cwd) {
17
17
  return cwd;
18
18
  }
19
19
 
20
- // Load TESTFORGE_KEY from a gitignored .testforge.env / .env (never overrides a real env var).
20
+ // Load TESTFORGE_KEY from the project's test env file (envFile in testforge.config.json),
21
+ // or a gitignored .testforge.env / .env. Never overrides a real env var.
21
22
  function loadEnv(root) {
22
- for (const f of ['.testforge.env', '.env']) {
23
+ for (const f of [loadConfig(root).envFile, '.testforge.env', '.env'].filter(Boolean)) {
23
24
  const p = path.join(root, f);
24
25
  if (!fs.existsSync(p)) continue;
25
26
  for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
@@ -86,6 +87,56 @@ function cleanError(raw) {
86
87
  return (lead + (rest ? '\n\n' + rest : '')).slice(0, 700);
87
88
  }
88
89
 
90
+ // What a test checks, in its own words: the requirement each assertion cites in an
91
+ // "// oracle: …" comment (every TestForge test carries them), read from the test's
92
+ // body in its source file. Runners report the declaration line, so read from there
93
+ // to the end of that test. Undefined when the file has no such comments.
94
+ const fileLines = new Map();
95
+ function testChecks(file, line) {
96
+ if (!file || !line) return undefined;
97
+ let lines = fileLines.get(file);
98
+ if (!lines) {
99
+ try { lines = fs.readFileSync(file, 'utf8').split('\n'); } catch { lines = []; }
100
+ fileLines.set(file, lines);
101
+ }
102
+ const decl = lines[line - 1];
103
+ if (decl === undefined) return undefined;
104
+ const indent = decl.match(/^\s*/)[0].length;
105
+ const found = [];
106
+ let cur = null;
107
+ for (let i = line; i < lines.length; i++) {
108
+ const l = lines[i];
109
+ const t = l.trim();
110
+ // the test ends at the first line back at (or left of) its own indentation
111
+ if (t && l.match(/^\s*/)[0].length <= indent) break;
112
+ const c = t.match(/^\/\/\s?(.*)$/);
113
+ if (c && /^oracle\s*:/i.test(c[1])) { cur = c[1].replace(/^oracle\s*:\s*/i, ''); found.push(cur); continue; }
114
+ if (c && cur !== null) { found[found.length - 1] += ' ' + c[1].trim(); continue; }
115
+ cur = null;
116
+ }
117
+ const uniq = [...new Set(found.map((x) => x.replace(/\s+/g, ' ').trim()).filter(Boolean))];
118
+ return uniq.length ? uniq.join('\n').slice(0, 1200) : undefined;
119
+ }
120
+
121
+ // For a failed check: what it wanted and what it got, from the runner's own message.
122
+ // A test's own failure message ("no visible reason, only a red field") says what the
123
+ // app did better than a bare true/false, so it wins over Received when present.
124
+ function expectation(raw) {
125
+ const s = strip(raw || '');
126
+ if (!s) return {};
127
+ const pick = (re) => { const m = s.match(re); return m ? m[1].trim() : undefined; };
128
+ const expected = pick(/^\s*Expected(?: [a-z ]+)?:\s*(.+)$/m);
129
+ let actual = pick(/^\s*Received(?: [a-z ]+)?:\s*(.+)$/m);
130
+ if (/element\(s\) not found/.test(s)) actual = 'the element was not found on the page';
131
+ const first = (s.split('\n').find((l) => l.trim()) || '').replace(/^\s*Error:\s*/, '').trim();
132
+ const own = first && !/^expect[.(]|^Test timeout|^locator\./i.test(first) ? first : undefined;
133
+ if (/^Test timeout of/i.test(first)) return { expected: plain(expected), actual: first };
134
+ // A bare true/false says nothing once the test's own message explains it.
135
+ return { expected: own && /^(true|false)$/.test(expected || '') ? undefined : plain(expected), actual: own || plain(actual) };
136
+ }
137
+ const PLAIN = { visible: 'shown on screen', hidden: 'not shown on screen (hidden)', enabled: 'enabled', disabled: 'disabled' };
138
+ const plain = (v) => (v && PLAIN[v]) || v;
139
+
89
140
  function sh(cmd, cwd) {
90
141
  try { return (execSync(cmd, { cwd, stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim()) || undefined; }
91
142
  catch { return undefined; }
@@ -166,4 +217,4 @@ async function publish(category, suites, durationSec) {
166
217
  }
167
218
  }
168
219
 
169
- module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv };
220
+ module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation };
@@ -1,7 +1,7 @@
1
1
  // TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to jest.config.js:
3
3
  // reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
4
- const { cleanError, sig, publish } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgeJestReporter {
7
7
  constructor(_globalConfig, options) {
@@ -23,6 +23,8 @@ module.exports = class TestForgeJestReporter {
23
23
  note: status === 'skipped'
24
24
  ? (a.status === 'todo' ? 'Marked to-do — this scenario isn’t implemented yet' : 'Skipped — not run in this execution')
25
25
  : undefined,
26
+ checks: testChecks(f.testFilePath, a.location && a.location.line),
27
+ ...(status === 'failed' ? expectation(raw) : {}),
26
28
  });
27
29
  }
28
30
  }
@@ -1,7 +1,7 @@
1
1
  // TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to playwright.config.js reporter array:
3
3
  // ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
4
- const { cleanError, sig, publish } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgePlaywrightReporter {
7
7
  constructor(options) {
@@ -15,13 +15,17 @@ module.exports = class TestForgePlaywrightReporter {
15
15
  const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
16
16
  const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
17
17
  const raw = result.error && result.error.message;
18
- const skipAnno = (test.annotations || []).find((x) => x.type === 'skip' || x.type === 'fixme');
18
+ const pend = (result.annotations || []).concat(test.annotations || []).filter((x) => x.type === 'skip' || x.type === 'fixme');
19
+ const skipAnno = pend.find((x) => x.description) || pend[0];
19
20
  su.tests.push({
20
21
  name: test.title,
21
22
  status,
22
23
  error: status === 'failed' ? cleanError(raw) : undefined,
23
24
  sig: status === 'failed' ? sig(raw) : undefined,
24
25
  note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
26
+ checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
27
+ || testChecks(test.location && test.location.file, test.location && test.location.line),
28
+ ...(status === 'failed' ? expectation(raw) : {}),
25
29
  });
26
30
  }
27
31
  async onEnd(result) {