@activate-core/test-forge 1.1.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -87,6 +87,10 @@ export const DEEP_DOC_GUIDANCE = [
87
87
  ` middleware that trims, normalises or converts empty strings, and the`,
88
88
  ` input mutators the handler applies. A row whose verdict is wrong`,
89
89
  ` becomes a test that fails on a correct product.`,
90
+ ` An [observed] criterion is about what the customer sees or what the`,
91
+ ` server accepts or refuses — never an implementation detail (a storage`,
92
+ ` key name, a CSS class, an internal flag, a request field name) unless the`,
93
+ ` source itself is about it.`,
90
94
  ` Only what a test can drive is a criterion. Behaviour no screen or`,
91
95
  ` endpoint can reach ("not tested through the screen") is not tagged —`,
92
96
  ` leave it out, or mention it in Test setup if the generator needs to know.`,
@@ -100,8 +104,11 @@ export const DEEP_DOC_GUIDANCE = [
100
104
  ` [baseline] item that fits to EACH row, not just to the main form.`,
101
105
  `2. COMBINATIONS. When the source states a rule across alternatives and`,
102
106
  ` conditions ("for login and sign-up, also on a slow connection and on`,
103
- ` a retry"), write one criterion per combination (login × slow, login ×`,
104
- ` retry, sign-up × slow, sign-up × retry), never just one of them.`,
107
+ ` a retry"), check in the code which of them take a different path. Write`,
108
+ ` one criterion per combination that takes its own path; alternatives`,
109
+ ` handled by the same code share ONE criterion that names them all. A`,
110
+ ` criterion per opener, per page or per input that all reach the same`,
111
+ ` code becomes one test per item — volume, not coverage.`,
105
112
  `3. CODE BRANCHES. Read the code the journey runs — screens, components,`,
106
113
  ` client API calls, and the server handlers they reach (related repos`,
107
114
  ` included) — and walk EVERY validation rule, error response and`,
@@ -114,9 +121,12 @@ export const DEEP_DOC_GUIDANCE = [
114
121
  `4. QUOTE [spec]. End every [spec] criterion with the source sentence it`,
115
122
  ` rests on, quoted: (source: "..."). A criterion you cannot quote is`,
116
123
  ` NOT [spec] — make it [baseline] or [observed], or a Question.`,
117
- `5. SHARED RULES. A rule shared by several journeys (password, redirects,`,
118
- ` attack payloads, accepted formats) uses the same cases and values in`,
119
- ` every doc.`,
124
+ `5. SHARED RULES AND SHARED CONTROLS. A rule or control shared by several`,
125
+ ` journeys (a password rule, a code-entry step, the login form, sign-out)`,
126
+ ` is specified in full ONCE, in the journey whose subject it is. Other`,
127
+ ` journeys that pass through it state only their own part: how the`,
128
+ ` customer reaches it from there and the outcome specific to that journey,`,
129
+ ` and name the journey that holds the full rule.`,
120
130
  `6. COVERAGE. After the numbered sections, add "## Coverage": one row per`,
121
131
  ` control from step 1, naming the section that covers each [baseline]`,
122
132
  ` item that fits it. A blank cell is a gap — fill it before finishing.`,
@@ -17,10 +17,10 @@
17
17
  * false-rejecting valid TypeScript.
18
18
  */
19
19
  import { execFileSync } from 'node:child_process';
20
- import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
21
- import { join } from 'node:path';
20
+ import { writeFileSync, mkdtempSync, rmSync, existsSync } from 'node:fs';
21
+ import { join, dirname, resolve } from 'node:path';
22
22
  import { tmpdir } from 'node:os';
23
- import { reasonlessPending } from '../generate/pending.js';
23
+ import { reasonlessPending, pendingDeclarations } from '../generate/pending.js';
24
24
  /**
25
25
  * Third-party network SDKs common enough to be worth gating everywhere:
26
26
  * a backend test that imports one without mocking it is talking to a real
@@ -164,6 +164,18 @@ export function checkTestGates(content, _ext, ctx) {
164
164
  'too; scope the stub to one of those, or name the ones you ruled out');
165
165
  }
166
166
  }
167
+ // F16 — no placeholders in a freshly written file: a placeholder never runs,
168
+ // and a suite full of them reads as coverage it does not have.
169
+ if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
170
+ return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
171
+ }
172
+ // F17 — relative imports resolve from where the file is written.
173
+ if (ctx.filePath) {
174
+ const missing = unresolvedImports(content, ctx.filePath);
175
+ if (missing.length) {
176
+ return `relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`;
177
+ }
178
+ }
167
179
  // F15 — a pending test says why. A reasonless fixme/skip reaches the report
168
180
  // as a bare "fixme", indistinguishable from the tool giving up. Runs on
169
181
  // content normalisePending() has already given every reason it could find.
@@ -381,3 +393,19 @@ function looksStructurallyClean(content) {
381
393
  count(content, '(') === count(content, ')');
382
394
  return balanced;
383
395
  }
396
+ const RESOLVE_EXTS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json'];
397
+ /** Relative module specifiers (import / require / dynamic import) that reach no file from `filePath`. */
398
+ export function unresolvedImports(content, filePath) {
399
+ const specs = new Set();
400
+ const re = /(?:\bfrom\s*|\brequire\s*\(\s*|\bimport\s*\(\s*|^\s*import\s+)(['"])(\.{1,2}\/[^'"]*)\1/gm;
401
+ for (const m of content.matchAll(re))
402
+ specs.add(m[2]);
403
+ const base = dirname(resolve(filePath));
404
+ return [...specs].filter((spec) => {
405
+ const target = resolve(base, spec);
406
+ // A compiled .js specifier may point at the .ts source.
407
+ const stem = target.replace(/\.(c|m)?js$/, '');
408
+ return !RESOLVE_EXTS.some((e) => existsSync(target + e) || existsSync(stem + e))
409
+ && !RESOLVE_EXTS.slice(1).some((e) => existsSync(join(target, 'index' + e)));
410
+ });
411
+ }
@@ -77,11 +77,6 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
77
77
  const framework = isApi
78
78
  ? 'Jest for an API/backend test'
79
79
  : 'Playwright for a web/UI test';
80
- // Pending-test syntax differs by runner: Jest has test.todo/it.skip;
81
- // Playwright has NEITHER (test.todo throws) — it uses test.fixme/skip.
82
- const pending = isApi
83
- ? 'test.todo(...) or it.skip(...)'
84
- : "test.fixme(title, { annotation: { type: 'fixme', description: '<why it is pending>' } }, async () => {}) — the reason goes in the annotation, where the report shows it; Playwright has NO test.todo/it.*";
85
80
  const ext = config.testExtension ?? '.test.ts';
86
81
  const isJs = ext.endsWith('.js');
87
82
  const language = isJs
@@ -134,13 +129,14 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
134
129
  ` - Assert the real end state (the response body and status, the`,
135
130
  ` persisted record, the screen) — not merely that nothing threw, and`,
136
131
  ` never a value your own stub hard-coded.`,
137
- ` - No line, no assertion: write the case pending (${pending}). An item under`,
132
+ ` - No line, no assertion: a case the document fixes no outcome for gets`,
133
+ ` NO test — not a placeholder either. Neither does an item under`,
138
134
  ` "Questions for product (not asserted)" or "Suggested", or one the`,
139
- ` document says is not tested through the screen, gets NO test at all —`,
140
- ` not even a pending one: it is the product's to answer.`,
135
+ ` document says is not tested through the screen: it is the product's`,
136
+ ` to answer.`,
141
137
  ` - A security probe asserts the SECURE outcome (refused, rejected,`,
142
138
  ` idempotent) unless the document records the weakness as accepted; when`,
143
- ` you cannot tell whether a defence exists, the case is pending.`,
139
+ ` you cannot tell whether a defence exists, the case is left out.`,
144
140
  `E2. WHAT YOU TOUCH — evidence: this project's source.`,
145
141
  ` - Every locator, route, request field, response field, storage key and id`,
146
142
  ` is something you found in THIS project's code (or the CODE FACTS), never`,
@@ -227,8 +223,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
227
223
  ` - A stub intercepts exactly the dependency — a different host; the same`,
228
224
  ` host on another port; a path prefix; or a predicate on the request —`,
229
225
  ` never a URL substring that also matches the app's own pages; then`,
230
- ` confirm the page under test still loads. Only a dependency you truly`,
231
- ` cannot separate stays pending, naming what you checked.`,
226
+ ` confirm the page under test still loads. A dependency you truly cannot`,
227
+ ` separate is left untested; say which of these you checked in a comment.`,
232
228
  ` - Real side effects (a message, an email, a charge) sit behind the opt-in`,
233
229
  ` gate the sibling files use, with one identity per rate-limited send.`,
234
230
  ` - A login elsewhere can end a cached session: get tokens through the`,
@@ -238,10 +234,32 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
238
234
  ` (change decoded bytes, not an encoded character), and the valid control`,
239
235
  ` is asserted too where you can.`,
240
236
  ``,
237
+ `COVER EACH OUTCOME ONCE. A suite is judged by the defects it can catch, not`,
238
+ `by its size; every redundant test costs run time and buries the ones that`,
239
+ `matter.`,
240
+ ` - One test per distinct outcome: each criterion, and each branch or`,
241
+ ` boundary the code takes for it (just inside, just outside). Inputs that`,
242
+ ` reach the same branch and give the same outcome are ONE case: thirty`,
243
+ ` attack strings against a field that only checks "not empty" prove`,
244
+ ` nothing the first did not. A parametrised test lists only rows that`,
245
+ ` reach different branches, each with a short comment naming its branch.`,
246
+ ` The same goes for repeating one check per field, per page, per opener`,
247
+ ` or per error code when the same code handles them all.`,
248
+ ` - A behaviour the "covers" line of this file says another journey owns is`,
249
+ ` not re-tested here: check only how the customer reaches it from this`,
250
+ ` journey, and the outcome specific to this journey.`,
251
+ ` - Do not assert implementation details — storage key names, CSS classes,`,
252
+ ` internal flags, request field names — unless the criterion itself is`,
253
+ ` about them (for example, a token must never appear in a URL).`,
254
+ ` - Never write a placeholder: no test.fixme / test.skip / test.todo`,
255
+ ` declarations. A precondition the environment lacks is a run-time skip`,
256
+ ` inside a real test, stating what was observed.`,
257
+ ``,
241
258
  `SELF-CHECK BEFORE YOU OUTPUT. Go through every test you wrote and name, for`,
242
- `each line, its evidence under E1–E5. A line with none is fixed, or its test`,
243
- `becomes pending with the reason. When in doubt, report LESS: a pending test`,
244
- `costs a line in the report; a false failure costs the team a release.`,
259
+ `each line, its evidence under E1–E5, and the distinct outcome each test`,
260
+ `adds. A line with no evidence is fixed, or its test is removed; a test that`,
261
+ `adds no new outcome is removed. When in doubt, report LESS: a missing test`,
262
+ `costs one line of coverage; a false failure costs the team a release.`,
245
263
  ``,
246
264
  ...(grounding
247
265
  ? [
@@ -276,9 +294,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
276
294
  `and let the test FAIL — a real bug surfaced is the goal, never`,
277
295
  `something to paper over. NEVER reshape an assertion to match observed`,
278
296
  `code behaviour just to make it green; that rubber-stamps the bug. If`,
279
- `the document does not state the expected outcome for a case, make it`,
280
- `pending (${pending}) — do not invent the`,
281
- `expectation from the code.`,
297
+ `the document does not state the expected outcome for a case, write no`,
298
+ `test for it — do not invent the expectation from the code.`,
282
299
  `REQUEST URLS: build every request path as the FULL mounted route the`,
283
300
  `router actually registers — INCLUDING the version/area prefix (e.g.`,
284
301
  `/v{VERSION}/<area>/<action>, reading the real version from its env var).`,
@@ -292,12 +309,12 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
292
309
  `fallback overrides the token for an AUTHENTICATED call: the testkit`,
293
310
  `seeds the owner fixtures under the authenticated identity, so expect`,
294
311
  `the token owner's rows. A second, non-owner identity (for IDOR/scope`,
295
- `tests) needs a distinct token; if unavailable, make the case pending.`,
312
+ `tests) needs a distinct token; if unavailable, the test skips at run time`,
313
+ `naming the missing identity.`,
296
314
  `Do NOT write or edit any file; output only the test code to stdout.`,
297
- `EVERY runnable test MUST contain at least one real assertion. If you`,
298
- `genuinely cannot assert an outcome, make the case pending (${pending})`,
299
- `— never write a test body with no assertion, which passes without`,
300
- `testing anything.`,
315
+ `EVERY test MUST contain at least one real assertion. If you genuinely`,
316
+ `cannot assert an outcome, write no test for that case — never a test body`,
317
+ `with no assertion, which passes without testing anything.`,
301
318
  ``,
302
319
  isApi
303
320
  ? [
@@ -346,9 +363,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
346
363
  `ground on role/label/text/name instead of inventing a test-id.`,
347
364
  `NEVER invent a data-testid/data-cy: an unverified test-id is`,
348
365
  `REJECTED by the selector-grounding gate and wastes the whole file.`,
349
- `Only when an element cannot be identified by ANY real handle,`,
350
- `defer that single case with ${pending} naming the control you could`,
351
- `not identify — do not default to pending, never guess.`,
366
+ `Only when an element cannot be identified by ANY real handle, leave`,
367
+ `that single case out and name the control in a comment — never guess.`,
352
368
  `NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
353
369
  `never a description/comment, and never a route-file placeholder`,
354
370
  `like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
@@ -358,8 +374,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
358
374
  `CRITICAL Playwright API: import { test, expect } from`,
359
375
  `'@playwright/test'. There is NO test.todo / it.* / describe.skip`,
360
376
  `in Playwright — using them throws at collection and breaks the`,
361
- `WHOLE file. For a pending case use ${pending}; group with`,
362
- `test.describe(...).`,
377
+ `WHOLE file. Group with test.describe(...).`,
363
378
  `CRITICAL API-request API: the \`request\` FIXTURE (test args`,
364
379
  `\`async ({ request }) => …\`) IS already an APIRequestContext —`,
365
380
  `call it DIRECTLY: \`await request.post(url, { data })\`,`,
@@ -416,7 +431,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
416
431
  `- Do NOT wrap the code in markdown fences.`,
417
432
  `- Tag the top with a comment: // @generated by testforge`,
418
433
  `- Never write "TODO" or "FIXME" in the file. Something you cannot`,
419
- ` establish is a pending test with its reason, never a note to finish later.`,
434
+ ` establish is left out, never a note to finish later.`,
420
435
  ``,
421
436
  `--- PROJECT MAP (data layer) ---`,
422
437
  map,
@@ -20,6 +20,7 @@ import { existsSync, readFileSync } from 'node:fs';
20
20
  import { relative } from 'node:path';
21
21
  import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
22
22
  import { runWithStdin } from './ai-generator.js';
23
+ import { aiUnavailableReason, noteAiFailure, noteAiSuccess, waitForAi } from './ai-session.js';
23
24
  const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'unproven'];
24
25
  /**
25
26
  * Group failing tests by the claim marked above them. A parametrised test
@@ -149,24 +150,47 @@ export function splitReview(group, parsed) {
149
150
  out.unshift({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence });
150
151
  return out;
151
152
  }
152
- /** Review every claim group, a few at a time. Never throws. */
153
- export async function reviewDefects(journey, groups, passes) {
153
+ /**
154
+ * Review every claim group, a few at a time. Never throws.
155
+ *
156
+ * A review that failed because the AI is gone (usage limit, logged out) is
157
+ * not a verdict. On the CleanMe booking-summary run the limit landed exactly
158
+ * here: all 18 claims came back "unproven — the review could not be read",
159
+ * the journey-level wait only noticed at the next AI call, and the summary
160
+ * reported 0 defects. So the group goes back on the queue, every worker
161
+ * shares one wait, and the review is asked again once the AI answers.
162
+ */
163
+ export async function reviewDefects(journey, groups, passes, io) {
154
164
  const config = loadConfig();
155
165
  const decisions = decisionsSource(config.decisions);
156
166
  // A linked list (an internal Notion page) needs the reference tools to be read.
157
167
  const tools = decisions.link ? withReferenceTools : withReadOnlyTools;
158
168
  const { program, args } = tools(splitCommand(config.aiCommand));
169
+ const ask = io?.ask ?? ((prompt) => runWithStdin(program, args, prompt, config.aiTimeoutMs));
170
+ const wait = io?.wait ?? (() => ((config.waitForLimit ?? true) ? waitForAi(config) : Promise.resolve(false)));
159
171
  const out = [];
160
172
  const queue = [...groups];
173
+ let waiting = null;
174
+ let gaveUp = false; // one failed wait is enough; the rest are recorded as unproven
161
175
  const worker = async () => {
162
176
  for (let g = queue.shift(); g; g = queue.shift()) {
163
177
  const files = new Set(g.tests.map((t) => t.file));
164
178
  const siblings = passes.filter((p) => files.has(p.file));
165
179
  let raw = '';
166
180
  try {
167
- raw = await runWithStdin(program, args, buildDefectReviewPrompt(journey, g, siblings, decisions), config.aiTimeoutMs);
181
+ raw = await ask(buildDefectReviewPrompt(journey, g, siblings, decisions));
182
+ noteAiSuccess();
168
183
  }
169
- catch {
184
+ catch (e) {
185
+ noteAiFailure(e);
186
+ if (aiUnavailableReason() && !gaveUp) {
187
+ waiting ??= wait().finally(() => (waiting = null));
188
+ if (await waiting) {
189
+ queue.unshift(g);
190
+ continue;
191
+ }
192
+ gaveUp = true;
193
+ }
170
194
  // an unanswered review proves nothing
171
195
  }
172
196
  out.push(...splitReview(g, parseReview(raw)));
@@ -116,3 +116,7 @@ export function reasonlessPending(content) {
116
116
  .filter((d) => !d.hasDetails)
117
117
  .map((d) => content.slice(d.open + 1, d.titleEnd + 1).trim().slice(1, 61));
118
118
  }
119
+ /** How many declaration-form pending tests (placeholders) the content holds. */
120
+ export function pendingDeclarations(content) {
121
+ return pendingDecls(content).length + (content.match(/\btest\.todo\s*\(/g) || []).length;
122
+ }
@@ -406,6 +406,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
406
406
  knownSelectors,
407
407
  externalModules: config.externalModules,
408
408
  });
409
+ // A freshly written file also may not hold placeholders; a repair is held to
410
+ // the repair guard instead (it may not add a skip), so old files stay repairable.
411
+ const genGate = (content, filePath) => testGate(content) ??
412
+ checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath });
409
413
  // Skip if unchanged — but NOT if the journey still has deferred stubs
410
414
  // to retry (a resume pass fills those in without touching real files).
411
415
  if (!force &&
@@ -513,7 +517,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
513
517
  let status;
514
518
  if (useAI) {
515
519
  const facts = await grounding();
516
- const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, testGate);
520
+ const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, (c) => genGate(c, filePath));
517
521
  if (ai !== null) {
518
522
  content = framework === 'playwright' ? normalisePending(ai) : ai;
519
523
  status = 'ai';
@@ -15,11 +15,48 @@
15
15
  * per-file generator turns each entry into an actual focused test.
16
16
  */
17
17
  import { execFileSync } from 'node:child_process';
18
- import { readFileSync, existsSync } from 'node:fs';
18
+ import { readFileSync, existsSync, readdirSync } from 'node:fs';
19
+ import { join } from 'node:path';
19
20
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
21
+ import { PLANS_DIR } from '../paths.js';
20
22
  const MAP_PATH = 'testforge.md';
23
+ /**
24
+ * The other journeys in the same area — their names, and the files already
25
+ * planned for them. A planner that sees only its own journey re-tests every
26
+ * shared form, rule and component it touches; on one real suite that was
27
+ * well over half of all tests. With the siblings in view, each behaviour is
28
+ * planned once, in the journey that owns it.
29
+ */
30
+ export function siblingJourneys(journey, docsDir = 'docs') {
31
+ const lines = [];
32
+ const planned = new Set();
33
+ const dir = join(PLANS_DIR, journey.area);
34
+ for (const f of existsSync(dir) ? readdirSync(dir) : []) {
35
+ const id = f.replace(/\.json$/, '');
36
+ if (!f.endsWith('.json') || id === journey.id)
37
+ continue;
38
+ try {
39
+ const plan = JSON.parse(readFileSync(join(dir, f), 'utf-8'));
40
+ lines.push(`- ${id} (planned):`, ...(plan.files ?? []).map((x) => ` - ${x.title ?? ''}: ${x.covers ?? ''}`));
41
+ planned.add(id);
42
+ }
43
+ catch {
44
+ // unreadable plan — the doc name below still names the journey
45
+ }
46
+ }
47
+ const area = join(docsDir, journey.area);
48
+ for (const f of existsSync(area) ? readdirSync(area) : []) {
49
+ if (!f.endsWith('.md'))
50
+ continue;
51
+ const name = (readFileSync(join(area, f), 'utf-8').match(/^# Journey:\s*(.+)$/m) ?? [])[1];
52
+ if (name && name.trim() !== journey.name && !lines.some((l) => l.includes(name.trim()))) {
53
+ lines.push(`- ${name.trim()} (not planned yet)`);
54
+ }
55
+ }
56
+ return lines.join('\n');
57
+ }
21
58
  /** Build the planning prompt for one journey. */
22
- function buildPlanPrompt(journey, map) {
59
+ export function buildPlanPrompt(journey, map, siblings = '') {
23
60
  return [
24
61
  `You are a senior test architect. Plan how to split ONE user journey`,
25
62
  `into a set of FOCUSED test files — one concern per file — the way a`,
@@ -43,7 +80,19 @@ function buildPlanPrompt(journey, map) {
43
80
  `- "## Controls" and "## Coverage" are indexes, "## Test setup" is what`,
44
81
  ` every file needs to run, and "## Questions for product (not`,
45
82
  ` asserted)" is not tested: plan no file for any of them.`,
83
+ `- ONE OWNER PER BEHAVIOUR. Every behaviour is tested in exactly one`,
84
+ ` file of exactly one journey. A form, component or rule this journey`,
85
+ ` shares with another journey (a login form, a code-entry step, a`,
86
+ ` password rule, a sign-out) belongs to the journey whose subject it is;`,
87
+ ` this journey tests only what is its OWN: how the customer reaches it`,
88
+ ` from here, and the outcome specific to this journey. Say so in`,
89
+ ` "covers" ("opening the login form from X and landing on X after`,
90
+ ` login — the form's own validation is owned by <journey>").`,
91
+ `- Within this journey, no two files test the same behaviour.`,
46
92
  ``,
93
+ ...(siblings
94
+ ? [`OTHER JOURNEYS IN THIS AREA (what they own or will own):`, siblings, ``]
95
+ : []),
47
96
  `Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
48
97
  ` [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,
49
98
  ` "covers": "<one or two sentences: exactly what this file tests>" }]`,
@@ -99,7 +148,7 @@ function safeSlug(raw) {
99
148
  */
100
149
  export function planSuite(journey) {
101
150
  const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
102
- const prompt = buildPlanPrompt(journey, map);
151
+ const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey));
103
152
  const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
104
153
  let raw = '';
105
154
  try {
@@ -65,6 +65,18 @@ export async function verifyJourney(ctx) {
65
65
  let outcome = await run();
66
66
  for (let attempt = 0;; attempt += 1) {
67
67
  if (!usable(outcome)) {
68
+ // The file itself does not load (a bad import, a syntax the runner
69
+ // rejects): that is the file's fault, so it is repaired like a
70
+ // failing test. Only an environment problem (infra) is left alone.
71
+ const loadError = !outcome.ran && !outcome.infra;
72
+ if (loadError && attempt < ctx.attempts) {
73
+ console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
74
+ const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
75
+ if (await repairOnce(target, [issue], [])) {
76
+ outcome = await run();
77
+ continue;
78
+ }
79
+ }
68
80
  notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
69
81
  break;
70
82
  }
@@ -78,11 +78,20 @@ export function mapPlaywright(json, category = 'web') {
78
78
  const raw = r.error && r.error.message;
79
79
  const pend = [...(r.annotations || []), ...(t.annotations || [])].filter((a) => a.type === 'skip' || a.type === 'fixme');
80
80
  const loc = spec.file ? path.resolve(rootDir, spec.file) : undefined;
81
+ const err = r.error || {};
82
+ const detail = status === 'failed'
83
+ ? shared.failureDetail(raw, {
84
+ file: err.location && err.location.file, line: err.location && err.location.line, column: err.location && err.location.column,
85
+ snippet: err.snippet, timeoutMs: undefined, browser: t.projectName,
86
+ attachments: (r.attachments || []).map((a) => ({ name: a.name, path: a.path })), root: shared.findRoot(rootDir),
87
+ })
88
+ : undefined;
81
89
  su.tests.push({
82
90
  name: spec.title,
83
91
  status,
84
92
  checks: shared.testChecks(loc, spec.line),
85
93
  ...(status === 'failed' ? shared.expectation(raw) : {}),
94
+ ...(detail ? { detail } : {}),
86
95
  error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
87
96
  sig: status === 'failed' ? shared.sig(raw) : undefined,
88
97
  note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "1.1.0",
3
+ "version": "1.2.1",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -118,21 +118,156 @@ function testChecks(file, line) {
118
118
  return uniq.length ? uniq.join('\n').slice(0, 1200) : undefined;
119
119
  }
120
120
 
121
- // For a failed check: what it wanted and what it got, from the runner's own message.
122
- // A test's own failure message ("no visible reason, only a red field") says what the
123
- // app did better than a bare true/false, so it wins over Received when present.
124
- function expectation(raw) {
125
- const s = strip(raw || '');
126
- if (!s) return {};
121
+ // ---- Failure detail -------------------------------------------------------
122
+ // The runner's message holds four different things that a report must keep
123
+ // apart: the test's OWN message (`expect(x, 'a visible reason under Password')`
124
+ // — what the check wanted, in the test's words), the matcher and its
125
+ // Expected/Received values, the locator and what it resolved to (the
126
+ // evidence of the element's real state), and the call log. The old mapping
127
+ // put the test's own message in "actual", so a report read "got: a visible
128
+ // reason under Password" — the expectation, dressed up as the observation.
129
+
130
+ const STATE_MATCHERS = /^toBe(Visible|Hidden|Enabled|Disabled|Checked|Focused|Editable|Empty|Attached|InViewport)$/;
131
+ const SUBJECT_OF = {
132
+ toHaveText: 'Text', toContainText: 'Text', toHaveURL: 'Page URL', toHaveTitle: 'Page title',
133
+ toHaveCount: 'Element count', toHaveValue: 'Field value', toHaveValues: 'Field values',
134
+ toHaveAttribute: 'Attribute', toHaveClass: 'Class', toHaveCSS: 'Style', toHaveId: 'Id',
135
+ toHaveScreenshot: 'Screenshot', toBeOK: 'Response', toHaveLength: 'Length',
136
+ };
137
+ const KIND_LABEL = {
138
+ hidden: 'Element hidden', 'not-found': 'Element not found', ambiguous: 'Ambiguous selector',
139
+ timeout: 'Test timeout', state: 'Wrong element state', mismatch: 'Value mismatch',
140
+ assertion: 'Condition false', error: 'Error thrown',
141
+ };
142
+ const short = (v, n = 90) => { v = String(v == null ? '' : v).trim(); return v.length > n ? v.slice(0, n - 1) + '…' : v; };
143
+ const cap = (v) => (v ? v[0].toUpperCase() + v.slice(1) : v);
144
+ const secs = (ms) => (ms % 1000 === 0 ? ms / 1000 + 's' : ms + 'ms');
145
+ const relTo = (root, p) => (p && root && p.startsWith(root + path.sep) ? p.slice(root.length + 1) : p);
146
+
147
+ // Inner text of the element Playwright printed ("<span class="x">Password format is incorrect.</span>").
148
+ function elementText(html) {
149
+ const m = (html || '').match(/^<([a-zA-Z0-9-]+)[^>]*>([\s\S]*?)<\/\1>\s*$/) || (html || '').match(/^<([a-zA-Z0-9-]+)[^>]*>([\s\S]*)$/);
150
+ if (!m) return { tag: ((html || '').match(/^<([a-zA-Z0-9-]+)/) || [])[1] };
151
+ const text = m[2].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim().replace(/…$/, '');
152
+ return { tag: m[1], text: text || undefined };
153
+ }
154
+
155
+ /**
156
+ * Structured detail of one failure, from the runner's message plus what the
157
+ * reporter knows (location, browser, timeout, artifacts). Every field comes
158
+ * from evidence; nothing is inferred from the wording of a message.
159
+ *
160
+ * ctx: { file, line, column, snippet, browser, timeoutMs, attachments: [{name, path}], root }
161
+ */
162
+ function failureDetail(raw, ctx) {
163
+ ctx = ctx || {};
164
+ const s = strip(raw || '').replace(/\r/g, '');
165
+ if (!s.trim()) return undefined;
166
+ const lines = s.split('\n');
127
167
  const pick = (re) => { const m = s.match(re); return m ? m[1].trim() : undefined; };
168
+ const first = (lines.find((l) => l.trim()) || '').trim().replace(/^Error:\s*/, '');
169
+
170
+ const assertionLine = lines.map((l) => l.trim()).find((l) => /^(Error:\s*)?expect\(.*\)\.\w+\(/.test(l));
171
+ const assertion = assertionLine ? assertionLine.replace(/^Error:\s*/, '').replace(/\s*(failed|\/\/.*)$/, '').trim() : undefined;
172
+ const matcher = assertion ? (assertion.match(/\.(\w+)\(/) || [])[1] : undefined;
173
+ const locator = pick(/^\s*Locator:\s*(.+)$/m);
128
174
  const expected = pick(/^\s*Expected(?: [a-z ]+)?:\s*(.+)$/m);
129
- let actual = pick(/^\s*Received(?: [a-z ]+)?:\s*(.+)$/m);
130
- if (/element\(s\) not found/.test(s)) actual = 'the element was not found on the page';
131
- const first = (s.split('\n').find((l) => l.trim()) || '').replace(/^\s*Error:\s*/, '').trim();
132
- const own = first && !/^expect[.(]|^Test timeout|^locator\./i.test(first) ? first : undefined;
133
- if (/^Test timeout of/i.test(first)) return { expected: plain(expected), actual: first };
134
- // A bare true/false says nothing once the test's own message explains it.
135
- return { expected: own && /^(true|false)$/.test(expected || '') ? undefined : plain(expected), actual: own || plain(actual) };
175
+ const received = pick(/^\s*Received(?: [a-z ]+)?:\s*(.+)$/m);
176
+ const testTimeout = s.match(/Test timeout of (\d+) ?ms exceeded/i);
177
+ const timeoutMs = testTimeout ? Number(testTimeout[1])
178
+ : Number(pick(/^\s*Timeout:?\s*(\d+)\s*ms/m) || (s.match(/Timeout (\d+) ?ms exceeded/) || [])[1] || ctx.timeoutMs || 0) || undefined;
179
+ const resolved = s.match(/(\d+) × locator resolved to (.+)$/m);
180
+ const strict = s.match(/strict mode violation: (.+?) resolved to (\d+) elements/);
181
+ const callLog = (s.split(/\n\s*Call log:\s*\n/i)[1] || '').replace(/\n\s*at\s.*$/gm, '').trim() || undefined;
182
+
183
+ // The test's own message: the first line when it is not the runner's own wording.
184
+ const runnerWording = /^(expect[.(]|Test timeout|Timeout|locator\.|page\.|strict mode violation|element\(s\) not found|Target (page|context|browser)|Error: )/i;
185
+ const subject = first && first !== assertion && !runnerWording.test(first) ? first : undefined;
186
+ const element = resolved ? elementText(resolved[2]) : {};
187
+
188
+ let kind;
189
+ if (testTimeout) kind = 'timeout';
190
+ else if (strict) kind = 'ambiguous';
191
+ else if (/element\(s\) not found/.test(s) || (locator && /waiting for/.test(s) && !resolved && matcher && STATE_MATCHERS.test(matcher))) kind = 'not-found';
192
+ else if (matcher === 'toBeVisible' && received === 'hidden') kind = 'hidden';
193
+ else if (matcher && STATE_MATCHERS.test(matcher)) kind = 'state';
194
+ else if (/^(true|false)$/.test(expected || '') && /^(true|false)$/.test(received || '')) kind = 'assertion';
195
+ else if (matcher && (expected !== undefined || received !== undefined)) kind = 'mismatch';
196
+ else if (matcher) kind = 'assertion';
197
+ else kind = 'error';
198
+
199
+ const what = subject || (locator ? `the element ${short(locator, 60)}` : undefined);
200
+ const waited = timeoutMs ? ` Waited ${secs(timeoutMs)}.` : '';
201
+ let headline, explain, exp, act;
202
+ switch (kind) {
203
+ case 'timeout':
204
+ headline = `Test timed out after ${secs(timeoutMs)}`;
205
+ explain = `The runner stopped the test after ${secs(timeoutMs)}, before it finished.`;
206
+ act = `the test did not finish within ${secs(timeoutMs)}`;
207
+ break;
208
+ case 'ambiguous':
209
+ headline = `${cap(subject || `Selector ${short(strict[1], 60)}`)} matched ${strict[2]} elements`;
210
+ explain = `The selector matched ${strict[2]} elements; Playwright strict mode needs exactly one.`;
211
+ exp = 'exactly one matching element'; act = `${strict[2]} elements matched`;
212
+ break;
213
+ case 'not-found':
214
+ headline = `${cap(what || 'The element')} was not found on the page`;
215
+ explain = `No element matched the selector: either it was never rendered, or the selector does not match the page.${waited}`;
216
+ exp = plain(expected) || 'present on the page'; act = 'not found on the page';
217
+ break;
218
+ case 'hidden':
219
+ headline = `${cap(what || 'The element')} is on the page but hidden`;
220
+ explain = `The element exists${element.text ? ` and reads "${short(element.text, 120)}"` : ''}, but it is not displayed, so the customer cannot see it.${waited}`;
221
+ exp = 'shown on screen'; act = `on the page but hidden${element.text ? ` ("${short(element.text, 80)}")` : ''}`;
222
+ break;
223
+ case 'state':
224
+ headline = `${cap(what || 'The element')}: expected ${plain(expected)}, got ${plain(received)}`;
225
+ explain = `Expected the element to be ${plain(expected)}; it was ${plain(received)}.${waited}`;
226
+ exp = plain(expected); act = plain(received);
227
+ break;
228
+ case 'mismatch': {
229
+ const noun = subject || SUBJECT_OF[matcher] || 'Value';
230
+ const e = expected === 'predicate to succeed' ? "a value matching the test's pattern" : expected;
231
+ headline = `${cap(noun)}: expected ${short(e, 60)}, got ${short(received, 60)}`;
232
+ explain = `Expected ${e}; the app gave ${received}.${waited}`;
233
+ exp = e; act = received;
234
+ break;
235
+ }
236
+ case 'assertion':
237
+ headline = subject ? cap(subject) : `Check failed: ${assertion}`;
238
+ explain = 'The test\'s condition was false. The runner recorded no element or value for it, so the exact on-screen state was not captured.';
239
+ exp = 'the check to pass'; act = 'not captured (the condition was false)';
240
+ break;
241
+ default:
242
+ headline = short(first, 140);
243
+ explain = 'The test stopped with this error; no expected or actual value was recorded.';
244
+ }
245
+
246
+ const root = ctx.root;
247
+ const d = {
248
+ kind, kindLabel: KIND_LABEL[kind], headline, explain,
249
+ subject, assertion, matcher, locator,
250
+ expected: exp, actual: act, rawExpected: expected, rawReceived: received,
251
+ element: element.tag ? { tag: element.tag, text: element.text, html: short(resolved[2], 300), count: Number(resolved[1]) } : undefined,
252
+ timeoutMs, callLog: callLog && callLog.slice(0, 3000), message: s.slice(0, 4000),
253
+ file: relTo(root, ctx.file), line: ctx.line, column: ctx.column, snippet: ctx.snippet && strip(ctx.snippet).slice(0, 1200),
254
+ browser: ctx.browser || undefined,
255
+ artifacts: (ctx.attachments || []).filter((a) => a && a.path).map((a) => ({ name: a.name, path: relTo(root, a.path) })),
256
+ };
257
+ for (const k of Object.keys(d)) if (d[k] === undefined || (Array.isArray(d[k]) && !d[k].length)) delete d[k];
258
+ return d;
259
+ }
260
+
261
+ // For a failed check: what it wanted and what it got, as two short strings.
262
+ // Kept for the older report fields; the values come from failureDetail, so
263
+ // "actual" is always the observed state, never the test's own message.
264
+ function expectation(raw) {
265
+ const d = failureDetail(raw);
266
+ if (!d) return {};
267
+ const out = {};
268
+ if (d.expected) out.expected = d.expected;
269
+ if (d.actual) out.actual = d.actual;
270
+ return out;
136
271
  }
137
272
  const PLAIN = { visible: 'shown on screen', hidden: 'not shown on screen (hidden)', enabled: 'enabled', disabled: 'disabled' };
138
273
  const plain = (v) => (v && PLAIN[v]) || v;
@@ -217,4 +352,4 @@ async function publish(category, suites, durationSec) {
217
352
  }
218
353
  }
219
354
 
220
- module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation };
355
+ module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot };
@@ -1,7 +1,7 @@
1
1
  // TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to jest.config.js:
3
3
  // reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
4
- const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, expectation, failureDetail } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgeJestReporter {
7
7
  constructor(_globalConfig, options) {
@@ -25,6 +25,7 @@ module.exports = class TestForgeJestReporter {
25
25
  : undefined,
26
26
  checks: testChecks(f.testFilePath, a.location && a.location.line),
27
27
  ...(status === 'failed' ? expectation(raw) : {}),
28
+ detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
28
29
  });
29
30
  }
30
31
  }
@@ -1,13 +1,28 @@
1
1
  // TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to playwright.config.js reporter array:
3
3
  // ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
4
- const { cleanError, sig, publish, testChecks, expectation } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, failureDetail, findRoot } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgePlaywrightReporter {
7
7
  constructor(options) {
8
8
  this.category = (options && options.category) || 'web';
9
9
  this.suites = {};
10
10
  this.start = Date.now();
11
+ this.root = findRoot(process.cwd());
12
+ }
13
+ // Everything the runner knows about a failure beyond its message: where it
14
+ // happened, the browser, the test's timeout and the artifacts it left.
15
+ detail(test, result) {
16
+ const err = result.error || {};
17
+ const loc = err.location || test.location || {};
18
+ const project = test.parent && typeof test.parent.project === 'function' ? test.parent.project() : undefined;
19
+ const d = failureDetail(err.message, {
20
+ file: loc.file, line: loc.line, column: loc.column, snippet: err.snippet,
21
+ browser: project && project.name, timeoutMs: test.timeout,
22
+ attachments: (result.attachments || []).map((a) => ({ name: a.name, path: a.path })),
23
+ root: this.root,
24
+ });
25
+ return d || undefined;
11
26
  }
12
27
  onTestEnd(test, result) {
13
28
  const file = (test.location && test.location.file) || 'suite';
@@ -25,9 +40,17 @@ module.exports = class TestForgePlaywrightReporter {
25
40
  note: status === 'skipped' ? ((skipAnno && skipAnno.description) || 'Skipped — not run in this execution') : undefined,
26
41
  checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
27
42
  || testChecks(test.location && test.location.file, test.location && test.location.line),
28
- ...(status === 'failed' ? expectation(raw) : {}),
43
+ ...(status === 'failed' ? this.failed(test, result) : {}),
29
44
  });
30
45
  }
46
+ failed(test, result) {
47
+ const detail = this.detail(test, result);
48
+ if (!detail) return {};
49
+ const out = { detail };
50
+ if (detail.expected) out.expected = detail.expected;
51
+ if (detail.actual) out.actual = detail.actual;
52
+ return out;
53
+ }
31
54
  async onEnd(result) {
32
55
  const durationSec = Math.round(((result && result.duration) || (Date.now() - this.start)) / 1000);
33
56
  const suites = Object.values(this.suites);