@activate-core/test-forge 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -105,7 +105,7 @@ never touched.
105
105
  | `testforge map` | Rebuild the project map (`testforge.md`) only. |
106
106
  | `testforge discover` | Redraft the per-journey docs in `docs/` only. |
107
107
  | `testforge env` | Rebuild `.env.testing.example` from the docs only. |
108
- | `testforge run` | Run the generated tests (`--journey` / `--area` / `--env-file`). Files whose feature is not in the code yet are held back as pending (`--all` runs them). |
108
+ | `testforge run` | Run the generated tests (`--journey` / `--area` / `--env-file`). First fills the names tests wait for from the code (only where each kind of name lives, only when those lines changed); `--no-scan` skips that. |
109
109
  | `testforge bind` | After a feature ships: which test files the code is ready for, which wait for the feature, which use a name the code spells differently. `--accept "doc=code"` takes the code's wording. No AI. |
110
110
  | `testforge review` | Ask an AI reviewer whether the last failures are defects, test mistakes or the environment. On demand only. |
111
111
  | `testforge report` | Publish results to the dashboard — `--results` after a run, `--inventory` to register every generated check. |
@@ -119,7 +119,7 @@ never touched.
119
119
  | `--force` | generate | Regenerate even when the doc has not changed. |
120
120
  | `--plan-only` | generate | Plan and check each journey (`testforge/memory/<area>/<id>/plan-check.md`), write no tests. |
121
121
  | `--refresh` | discover | Fetch the reference again instead of the copy kept from the last run. |
122
- | `--all` | run | Also run files whose feature is not in the code yet. |
122
+ | `--no-scan` | run | Do not look for blank interface names in the code before the run. |
123
123
  | `--refresh-docs` | update | Re-draft the docs even when nothing detectably changed — see below. |
124
124
  | `--reference <file\|url>` | setup | Base the docs on a spec instead of the code map. |
125
125
  | `--source map` | setup | Base the docs on `testforge.md`, no prompt. |
@@ -410,16 +410,26 @@ route, a button's label, a command, a table or field. When the code already
410
410
  has one, the test uses it. When it does not (yet), the test asks for it by a
411
411
  plain name — `need('order summary URL')` from
412
412
  `@activate-core/test-forge/reporters/interface.cjs` — and the name is listed
413
- in `testforge/interface.json` with an empty value. The developer fills it once
414
- it exists; `testforge run` and `bind` hold a file back, and list what it waits
415
- for, while one of its names is empty. TestForge never overwrites a value.
416
-
417
- Every run ends with its AI cost per step (write, grounding, seed, harness …).
418
-
419
- **A feature that is not live yet.** Put `Status: not built` or
420
- `Status: behind flag <name>` in the journey doc's header. Every file of the
421
- journey is stamped `// testforge-pending: …`, and `testforge run` (and `bind`)
422
- hold it back until the line is removed; `--all` runs it anyway. No seed is written for it: its tests make their own data.
413
+ in `testforge/interface.json` with an empty value. TestForge never overwrites
414
+ a value somebody wrote.
415
+
416
+ **Every run runs everything.** Before it starts, `testforge run` looks for the
417
+ blank names in the code — a URL only in the route files, a command only in the
418
+ console commands, a table only in the migrations and models, a field only in
419
+ the resources and controllers, and only on lines that share two of the name's
420
+ words. Nothing found, or the same lines as last time: no AI call. New lines:
421
+ one small call without tools picks each value, kept only if the code really
422
+ has it, and printed with what was found. Tests, docs and plans are never
423
+ touched. A test whose name is still blank is reported as "waiting for the
424
+ product", not as a defect. `--no-scan` skips the look-up.
425
+
426
+ Every run of `generate` ends with its AI cost per step (write, grounding, seed, harness …).
427
+
428
+ **A feature that is not live yet.** Write `Status: not built` or
429
+ `Status: behind flag <name>` in the journey doc's header if you like: no seed is
430
+ written for it (its tests make their own data), and its files carry a
431
+ `// testforge-pending: …` note. Its tests still run; until the feature is there
432
+ they fail or wait for its names, and the report says which.
423
433
 
424
434
  `generate` runs nothing. `testforge run` runs the suite; `testforge review`
425
435
  judges failures when you ask for it.
package/dist/cli/index.js CHANGED
@@ -224,6 +224,7 @@ async function main() {
224
224
  envFile: flagValue('--env-file'),
225
225
  passThrough,
226
226
  all: process.argv.includes('--all'),
227
+ noScan: process.argv.includes('--no-scan'),
227
228
  });
228
229
  process.exitCode = code;
229
230
  }
@@ -39,7 +39,7 @@ const NOT_A_REASON = [
39
39
  /\b(?:second|two|distinct|another|different) (?:\w+ )?(?:customer|account|user|people|person)s?\b/i,
40
40
  ];
41
41
  /** The only reasons a rule cannot be tested: an outside service the test cannot drive. */
42
- const OUTSIDE = /\b(?:stripe|payment (?:provider|processor|gateway)|card (?:network|issuer|authori[sz]ation)|3-?d ?secure|third[- ]party|external (?:service|system|provider|site)|e-?mail|inbox|sms|text message|push notification|google|routing provider|maps? (?:api|provider)|mapbox|tomtom|another app|mobile app|washer app|operator app|native app|another tab|other tab|second tab|two tabs|iframe|captcha|clock|real time|time passing|wait(?:ing)? (?:\d+|hours|days))\b/i;
42
+ const OUTSIDE = /\b(?:stripe|payment (?:provider|processor|gateway)|card (?:network|issuer|authori[sz]ation)|3-?d ?secure|third[- ]party|external (?:service|system|provider|site)|e-?mail|inbox|sms|text message|push notification|google|routing provider|maps? (?:api|provider)|mapbox|tomtom|another app|mobile app|native app|another tab|other tab|second tab|two tabs|iframe|captcha|clock|real time|time passing|wait(?:ing)? (?:\d+|hours|days))\b/i;
43
43
  export const acceptedReason = (why) => OUTSIDE.test(why) && !NOT_A_REASON.some((re) => re.test(why));
44
44
  export function ruleCoverage(content, planned) {
45
45
  const tested = new Set(findTestBlocks(content).flatMap((b) => b.rules));
@@ -0,0 +1,235 @@
1
+ /**
2
+ * testforge — fill the names tests wait for, from the code, at run time.
3
+ *
4
+ * A test written from the doc asks the product for a name it did not have
5
+ * yet: need('assignment mode switch URL'). Before a run, each blank name is
6
+ * looked for only where that kind of name lives — a URL in the route files, a
7
+ * command in the console commands, a table in the migrations, a field in the
8
+ * resources and controllers — and only on the lines that share a word with
9
+ * it. No line, no call. Lines unchanged since the last look, no call. When
10
+ * there are new lines, one small call without tools picks the value, and it
11
+ * is kept only if the code really contains it. Tests are never touched.
12
+ */
13
+ import { createHash } from 'node:crypto';
14
+ import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
15
+ import { dirname, join, relative } from 'node:path';
16
+ import { STATE_DIR } from '../paths.js';
17
+ import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
18
+ import { runAiSync } from '../generate/ai-usage.js';
19
+ import { cleanOutput } from '../generate/ai-generator.js';
20
+ import { readInterface, recordInterface } from '../generate/interface-file.js';
21
+ const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
22
+ const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
23
+ const CODE = /\.(php|ts|tsx|js|jsx|mjs|cjs|py|rb|go|vue|svelte|prisma|sql)$/;
24
+ const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls']);
25
+ /** Where a name of this kind lives, judged from how the name ends. */
26
+ export function kindOf(name) {
27
+ const n = name.trim().toLowerCase();
28
+ if (/\b(?:url|urls|path|prefix|endpoint|route)$/.test(n))
29
+ return 'route';
30
+ if (/\bcommand$/.test(n))
31
+ return 'command';
32
+ if (/\b(?:table|column|model)$/.test(n))
33
+ return 'schema';
34
+ if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|setting)$/.test(n))
35
+ return 'field';
36
+ if (/\blabel$/.test(n))
37
+ return 'label';
38
+ return undefined;
39
+ }
40
+ /** Files of a kind: a route file, a console command, a migration or model, a resource or controller, a component. */
41
+ function fileMatches(kind, rel) {
42
+ switch (kind) {
43
+ case 'route': return /(^|\/)(routes?|router|urls)(\/|\.[a-z]+$)|\.routes?\.[a-z]+$|\/routes?\.[a-z]+$|(^|\/)app\/.*\/(page|route)\.[jt]sx?$/i.test(rel);
44
+ case 'command': return /(^|\/)(console|commands?|cli|bin)(\/|$)|command/i.test(rel);
45
+ case 'schema': return /(^|\/)(migrations?|models?|entities|schema)(\/|\.)|\.prisma$|\.sql$/i.test(rel);
46
+ case 'field': return /(^|\/)(resources?|transformers?|serializers?|controllers?|dto|types?|api)(\/|\.)|resource|serializer|controller/i.test(rel);
47
+ case 'label': return /\.(tsx|jsx|vue|svelte)$/.test(rel);
48
+ }
49
+ }
50
+ /** What a line that DEFINES a name of each kind looks like. */
51
+ const SHAPE = {
52
+ route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]/,
53
+ command: /\$signature\s*=|\$name\s*=\s*['"]|->command\(\s*['"]|\.command\(\s*['"]|add_parser\(|@click\.command/,
54
+ schema: /Schema::(?:create|table)\(|\$table->\w+\(\s*['"]|create_table|\bclass\s+\w+\s+extends\s+\w*Model\b|protected\s+\$(?:table|fillable)|\bmodel\s+\w+\s*\{|@(?:Entity|Column|Table)\b/,
55
+ field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]/,
56
+ label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]/,
57
+ };
58
+ /** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
59
+ function candidateLines(name, kind, files, root) {
60
+ const words = [...new Set(name.toLowerCase().match(/[a-z]{3,}/g) ?? [])]
61
+ .filter((w) => !GENERIC.has(w))
62
+ .map((w) => (w.length > 5 ? w.replace(/(?:ings?|ed|es|s)$/, '') : w));
63
+ if (!words.length)
64
+ return [];
65
+ // One shared common word ("add", "error") points at nothing: a line must
66
+ // carry two of the name's words (or its only one).
67
+ const need = Math.min(2, words.length);
68
+ const hits = [];
69
+ const prefixes = [];
70
+ for (const f of files) {
71
+ const rel = relative(root, f);
72
+ if (!fileMatches(kind, rel))
73
+ continue;
74
+ let lines;
75
+ try {
76
+ lines = readFileSync(f, 'utf-8').split('\n');
77
+ }
78
+ catch {
79
+ continue;
80
+ }
81
+ lines.forEach((l, i) => {
82
+ if (/^\s*(?:\/\/|#|\*)/.test(l) || l.length > 300)
83
+ return;
84
+ const low = l.toLowerCase();
85
+ if (kind === 'route' && /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>/.test(l))
86
+ prefixes.push(`${rel}:${i + 1}: ${l.trim()}`);
87
+ if (!SHAPE[kind].test(l))
88
+ return;
89
+ const score = words.filter((w) => low.includes(w)).length;
90
+ if (score >= need)
91
+ hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score });
92
+ });
93
+ }
94
+ if (!hits.length)
95
+ return [];
96
+ const best = hits.sort((a, b) => b.score - a.score).slice(0, 8).map((h) => h.line);
97
+ // A route's full path needs the prefix of the group it sits in: those of its own files ride along.
98
+ const files_ = new Set(best.map((l) => l.split(':')[0]));
99
+ return [...best, ...prefixes.filter((p) => files_.has(p.split(':')[0])).slice(0, 6)];
100
+ }
101
+ function codeFiles(root) {
102
+ const out = [];
103
+ const walk = (d) => {
104
+ let entries;
105
+ try {
106
+ entries = readdirSync(d);
107
+ }
108
+ catch {
109
+ return;
110
+ }
111
+ for (const e of entries) {
112
+ const p = join(d, e);
113
+ let st;
114
+ try {
115
+ st = statSync(p);
116
+ }
117
+ catch {
118
+ continue;
119
+ }
120
+ if (st.isDirectory()) {
121
+ if (!SKIP.test(e))
122
+ walk(p);
123
+ }
124
+ else if (CODE.test(e) && st.size < 400_000)
125
+ out.push(p);
126
+ }
127
+ };
128
+ walk(root);
129
+ return out;
130
+ }
131
+ /** A value is kept only when the code really has it: each part of a path, or the value itself. */
132
+ export function foundInCode(value, lines) {
133
+ const text = lines.join('\n');
134
+ const v = value.trim();
135
+ if (!v)
136
+ return false;
137
+ if (/^\/|^[\w-]+\//.test(v)) {
138
+ const parts = v.split('/').filter((p) => p && p !== 'api' && !/^[{:<]/.test(p));
139
+ return parts.length > 0 && parts.every((p) => text.includes(p));
140
+ }
141
+ return text.includes(v);
142
+ }
143
+ /**
144
+ * The blank names worth one lookup call, each with its candidate lines; the
145
+ * memo records what each name's lines looked like, so unchanged ones are not
146
+ * asked again. No AI.
147
+ */
148
+ export function namesToAsk(blank, root, memo) {
149
+ const files = blank.length ? codeFiles(root) : [];
150
+ const ask = [];
151
+ for (const name of blank) {
152
+ const kind = kindOf(name);
153
+ const lines = kind ? candidateLines(name, kind, files, root) : [];
154
+ const print = createHash('sha1').update(lines.join('\n')).digest('hex').slice(0, 16);
155
+ // Nothing that could be it, or nothing new since the last look: no call.
156
+ if (!lines.length || memo[name] === print) {
157
+ memo[name] = print;
158
+ continue;
159
+ }
160
+ memo[name] = print;
161
+ ask.push({ name, lines });
162
+ }
163
+ return ask;
164
+ }
165
+ /**
166
+ * Look for the blank names in the code and fill what is found. `blank` are
167
+ * the names the tests to run still wait for. Returns what was filled.
168
+ */
169
+ export function scanNames(blank, root = '.') {
170
+ const memo = (() => {
171
+ try {
172
+ return JSON.parse(readFileSync(SCAN_PATH, 'utf-8'));
173
+ }
174
+ catch {
175
+ return {};
176
+ }
177
+ })();
178
+ const all = namesToAsk(blank, root, memo);
179
+ // One call stays small: names past the budget are asked on the next run.
180
+ const ask = [];
181
+ let size = 0;
182
+ for (const a of all) {
183
+ const n = a.name.length + a.lines.join('\n').length;
184
+ if (ask.length && size + n > 60_000) {
185
+ delete memo[a.name];
186
+ continue;
187
+ }
188
+ ask.push(a);
189
+ size += n;
190
+ }
191
+ const filled = [];
192
+ let calls = 0;
193
+ if (ask.length) {
194
+ const known = Object.entries(readInterface()).filter(([, v]) => String(v).trim()).slice(0, 8);
195
+ const prompt = [
196
+ 'You match names a test needs to values in code. For each NAME below, pick its value from ITS lines only, or null.',
197
+ 'Answer with ONE JSON object {"<name>": "<value>" | null} and nothing else.',
198
+ 'A value is copied from the lines, never invented. A URL is the path a client calls, with "/" first and every route prefix the lines show',
199
+ '(a route inside a prefix group gets the prefix). A command is the command name a person types. A field, key or column is the exact key.',
200
+ 'If no line clearly is that thing, the value is null.',
201
+ ...(known.length ? ['', 'Values already filled, for their format:', ...known.map(([k, v]) => `- ${k}: ${v}`)] : []),
202
+ '',
203
+ ...ask.flatMap((a) => [`NAME: ${a.name}`, ...a.lines.map((l) => ` ${l}`), '']),
204
+ ].join('\n');
205
+ try {
206
+ const { program, args } = withNoTools(splitCommand(loadConfig().aiCommand), 'Answer only with the JSON object asked for.');
207
+ calls = 1;
208
+ const raw = cleanOutput(runAiSync(program, args, { input: prompt, encoding: 'utf-8', maxBuffer: 4 * 1024 * 1024, timeout: 120_000 }));
209
+ const json = JSON.parse(raw.slice(raw.indexOf('{'), raw.lastIndexOf('}') + 1));
210
+ const keep = {};
211
+ for (const a of ask) {
212
+ const v = json[a.name];
213
+ if (typeof v === 'string' && foundInCode(v, a.lines))
214
+ keep[a.name] = v.trim();
215
+ }
216
+ recordInterface(keep);
217
+ for (const [name, value] of Object.entries(keep))
218
+ filled.push({ name, value });
219
+ }
220
+ catch (e) {
221
+ console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
222
+ for (const a of ask)
223
+ delete memo[a.name]; // look again next time
224
+ }
225
+ }
226
+ try {
227
+ mkdirSync(dirname(SCAN_PATH), { recursive: true });
228
+ writeFileSync(SCAN_PATH, `${JSON.stringify(memo, null, 2)}\n`, 'utf-8');
229
+ }
230
+ catch {
231
+ // best-effort memo
232
+ }
233
+ const done = new Set(filled.map((f) => f.name));
234
+ return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
235
+ }
@@ -12,10 +12,11 @@
12
12
  * It runs from the area's nearest package root (where package.json /
13
13
  * playwright.config live), with config.envFile merged into the env.
14
14
  */
15
- import { checkReadiness, testFilesIn } from './readiness.js';
16
- import { sourceRoot, sourceText } from '../docs/excerpts.js';
15
+ import { testFilesIn } from './readiness.js';
16
+ import { scanNames } from './name-scan.js';
17
+ import { INTERFACE_PATH, blankNames, neededNames } from '../generate/interface-file.js';
17
18
  import { spawnSync } from 'node:child_process';
18
- import { existsSync } from 'node:fs';
19
+ import { existsSync, readFileSync } from 'node:fs';
19
20
  import { join, dirname, resolve, relative } from 'node:path';
20
21
  import { loadConfig, loadEnvFile, testDirForType, resolveFramework, } from '../project/config.js';
21
22
  /** Nearest ancestor dir containing a package.json (the runner's cwd). */
@@ -53,7 +54,9 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
53
54
  return `${explicit} ${paths.join(' ')}`;
54
55
  // A custom command still honours --journey: narrow it to that journey's
55
56
  // tests, or `run --journey x` silently runs the whole suite.
56
- return journey ? `${explicit} ${framework === 'playwright' ? `${rel}/${journey}` : `--testPathPattern "${journey}"`}` : explicit;
57
+ // A path argument, not --testPathPattern: Jest 30 renamed that flag, and a
58
+ // path narrows the run the same way in every version of both runners.
59
+ return journey ? `${explicit} ${rel}/${journey}` : explicit;
57
60
  }
58
61
  if (framework === 'playwright') {
59
62
  const target = paths.length
@@ -100,7 +103,7 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
100
103
  if (paths.length)
101
104
  return `npx jest${jestCfg} ${paths.join(' ')}`;
102
105
  return journey
103
- ? `npx jest${jestCfg} --testPathPattern "${journey}"`
106
+ ? `npx jest${jestCfg} ${rel}/${journey}`
104
107
  : `npx jest${jestCfg} ${rel}`;
105
108
  }
106
109
  export function runTests(opts = {}) {
@@ -130,23 +133,21 @@ export function runTests(opts = {}) {
130
133
  const framework = resolveFramework(config, area, area);
131
134
  const root = packageRoot(testDir);
132
135
  const rel = relative(root, testDir) || '.';
133
- // Tests written before their feature: hold back the files whose names and
134
- // pages the code does not have yet, instead of running them to fail.
135
- let targets;
136
- if (framework === 'playwright' && !opts.all) {
136
+ // Every test runs: a feature that is not there yet fails, and its report
137
+ // says so. Before the run, the names those tests wait for are looked for
138
+ // in the code — only where that kind of name lives, only when those lines
139
+ // changed — and filled; the tests themselves are never touched.
140
+ const targets = undefined;
141
+ if (!opts.noScan) {
137
142
  const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
138
143
  const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
139
- const roots = [sourceRoot(), ...(config.relatedDirs ?? [])];
140
- const states = checkReadiness(files, sourceText(roots), () => [], [root]);
141
- const held = states.filter((s) => s.state === 'not-built');
142
- if (held.length) {
143
- console.log(`\n${held.length} file(s) pending — their feature is not live yet (\`testforge bind\` shows why; --all runs them anyway):`);
144
- for (const h of held.slice(0, 15))
145
- console.log(` not built ${relative(root, h.file)}: ${h.missing.slice(0, 3).map((m) => `"${m}"`).join(', ')}`);
146
- targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
147
- if (!targets.length) {
148
- console.log(`[skip] ${area}: nothing to run yet`);
149
- continue;
144
+ const blank = blankNames([...new Set(files.flatMap((f) => neededNames(readFileSync(f, 'utf-8'))))]);
145
+ if (blank.length) {
146
+ const res = scanNames(blank, root);
147
+ for (const f of res.filled)
148
+ console.log(` found in the code: ${f.name} = ${f.value}`);
149
+ if (res.stillBlank.length) {
150
+ console.log(` ${res.stillBlank.length} name(s) not in the code yet — their tests report "waiting for the product" (${INTERFACE_PATH}): ${res.stillBlank.slice(0, 6).join(', ')}${res.stillBlank.length > 6 ? ', …' : ''}`);
150
151
  }
151
152
  }
152
153
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.2.0",
3
+ "version": "2.3.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -394,4 +394,10 @@ function reviewVerdicts(root) {
394
394
  return (file, title) => (file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined);
395
395
  }
396
396
 
397
- module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts };
397
+ /** The name a test is still waiting for (need() threw for a blank interface name), else undefined. */
398
+ function waitingFor(raw) {
399
+ const m = /\[testforge:waiting\] "([^"]+)"/.exec(String(raw || ''));
400
+ return m ? m[1] : undefined;
401
+ }
402
+
403
+ module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor };
@@ -40,7 +40,8 @@ function values() {
40
40
  function need(name) {
41
41
  const v = values()[name];
42
42
  if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
43
- throw new Error(`"${name}" is not filled in testforge/interface.json: the product does not have it yet, or nobody has written it down. Fill it, then run again.`);
43
+ // The prefix lets the reporters file this as "waiting for the product", not as a defect.
44
+ throw new Error(`[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json): this test runs for real once it is there.`);
44
45
  }
45
46
 
46
47
  module.exports = { need };
@@ -1,7 +1,7 @@
1
1
  // TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to jest.config.js:
3
3
  // reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
4
- const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgeJestReporter {
7
7
  constructor(_globalConfig, options) {
@@ -15,15 +15,17 @@ module.exports = class TestForgeJestReporter {
15
15
  const name = suiteName(f.testFilePath || f.name || 'suite', this.testRoot);
16
16
  const su = suites[name] || (suites[name] = { name, category: this.category, tests: [] });
17
17
  for (const a of (f.testResults || f.assertionResults || [])) {
18
- const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
19
18
  const raw = (a.failureMessages || [])[0];
19
+ // A test waiting for a name the product does not have yet is not a defect.
20
+ const waiting = a.status === 'failed' ? waitingFor(raw) : undefined;
21
+ const status = a.status === 'passed' ? 'passed' : a.status === 'failed' && !waiting ? 'failed' : 'skipped';
20
22
  su.tests.push({
21
23
  name: a.fullName || a.title,
22
24
  status,
23
25
  error: status === 'failed' ? cleanError(raw) : undefined,
24
26
  sig: status === 'failed' ? sig(raw) : undefined,
25
27
  note: status === 'skipped'
26
- ? (a.status === 'todo' ? 'Marked to-do — this scenario isn’t implemented yet' : 'Skipped — not run in this execution')
28
+ ? (waiting ? `Waiting for the product: "${waiting}" is not in it yet` : a.status === 'todo' ? 'Marked to-do — this scenario isn’t implemented yet' : 'Skipped — not run in this execution')
27
29
  : undefined,
28
30
  checks: testChecks(f.testFilePath, a.location && a.location.line),
29
31
  ...(status === 'failed' ? expectation(raw) : {}),
@@ -1,7 +1,7 @@
1
1
  // TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to playwright.config.js reporter array:
3
3
  // ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
4
- const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgePlaywrightReporter {
7
7
  constructor(options) {
@@ -35,8 +35,10 @@ module.exports = class TestForgePlaywrightReporter {
35
35
  const file = (test.location && test.location.file) || 'suite';
36
36
  const name = suiteName(file, this.testRoot);
37
37
  const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
38
- const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
39
38
  const raw = result.error && result.error.message;
39
+ // A test waiting for a name the product does not have yet is not a defect.
40
+ const waiting = result.status !== 'passed' && result.status !== 'skipped' ? waitingFor(raw) : undefined;
41
+ const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' || waiting ? 'skipped' : 'failed';
40
42
  const pend = (result.annotations || []).concat(test.annotations || []).filter((x) => x.type === 'skip' || x.type === 'fixme');
41
43
  const skipAnno = pend.find((x) => x.description) || pend[0];
42
44
  su.tests.push({
@@ -44,7 +46,7 @@ module.exports = class TestForgePlaywrightReporter {
44
46
  status,
45
47
  error: status === 'failed' ? cleanError(raw) : undefined,
46
48
  sig: status === 'failed' ? sig(raw) : undefined,
47
- note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
49
+ note: status === 'skipped' ? (waiting ? `Waiting for the product: "${waiting}" is not in it yet` : (skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
48
50
  checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
49
51
  || testChecks(test.location && test.location.file, test.location && test.location.line),
50
52
  ...(status === 'failed' ? this.failed(test, result) : {}),