@activate-core/test-forge 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +61 -23
- package/dist/core/docs/excerpts.js +245 -0
- package/dist/core/docs/grounding.js +49 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +202 -104
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +139 -0
- package/dist/core/generate/harness.js +3 -31
- package/dist/core/generate/journey-memory.js +325 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +169 -78
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +115 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +119 -0
- package/dist/core/run/run-tests.js +23 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
|
@@ -169,6 +169,21 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
169
169
|
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
170
170
|
return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
|
|
171
171
|
}
|
|
172
|
+
// F22 — a fresh file exists to test its rules: one with no test at all says
|
|
173
|
+
// nothing, and passes. It goes back once, told to stub what it cannot create.
|
|
174
|
+
if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
|
|
175
|
+
return 'the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out';
|
|
176
|
+
}
|
|
177
|
+
// F21 — a run-time skip is for what the ENVIRONMENT lacks (an account, a
|
|
178
|
+
// variable, seeded data). Skipping because an element is not on the page
|
|
179
|
+
// hides exactly what a test written from the doc exists to report: the
|
|
180
|
+
// element the doc names is missing, and that is a failure.
|
|
181
|
+
if (ctx.forbidPlaceholders) {
|
|
182
|
+
const why = elementSkips(content);
|
|
183
|
+
if (why.length) {
|
|
184
|
+
return `a test skips itself because an element is not on the page (${why[0]}): a missing element the doc names is a failure — assert it instead. Skip only for an account, a variable or data the environment lacks`;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
172
187
|
// F17 — relative imports resolve from where the file is written.
|
|
173
188
|
if (ctx.filePath) {
|
|
174
189
|
const missing = unresolvedImports(content, ctx.filePath);
|
|
@@ -234,9 +249,59 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
234
249
|
}
|
|
235
250
|
}
|
|
236
251
|
}
|
|
252
|
+
// (c) A name, label or placeholder the app never renders is a guess too.
|
|
253
|
+
if (ctx.knownTexts) {
|
|
254
|
+
const missing = usedLabels(active).filter((l) => !ctx.knownTexts.includes(l.toLowerCase()));
|
|
255
|
+
if (missing.length) {
|
|
256
|
+
return `uses ${missing.map((m) => `"${m}"`).slice(0, 3).join(', ')} as an accessible name, label or placeholder, but that text appears nowhere in the app's source or the journey doc — take every name from the code facts or the rules`;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
237
259
|
}
|
|
238
260
|
return null;
|
|
239
261
|
}
|
|
262
|
+
/**
|
|
263
|
+
* The run-time skips whose condition looks at the page: test.skip(true, …),
|
|
264
|
+
* or a condition that counts, checks the visibility of, or locates an element.
|
|
265
|
+
* Returns each condition, shortened.
|
|
266
|
+
*/
|
|
267
|
+
export function elementSkips(content) {
|
|
268
|
+
const out = [];
|
|
269
|
+
for (const m of content.matchAll(/\btest\.skip\s*\(/g)) {
|
|
270
|
+
const open = m.index + m[0].length - 1;
|
|
271
|
+
const close = matchParen(content, open);
|
|
272
|
+
if (close === -1)
|
|
273
|
+
continue;
|
|
274
|
+
const args = content.slice(open + 1, close);
|
|
275
|
+
// A declaration skip (test.skip('title', fn)) is the placeholder gate's job.
|
|
276
|
+
if (/^\s*['"`]/.test(args))
|
|
277
|
+
continue;
|
|
278
|
+
const cond = args.split(/,\s*['"`]/)[0];
|
|
279
|
+
const looksAtPage = /^\s*true\s*$/.test(cond) || /\.count\(|\.isVisible\(|\.isHidden\(|getBy\w+\(|\bfind\(|\.locator\(|\$\(/.test(cond);
|
|
280
|
+
// …or a flag set from the page a few lines above (const burger = await …isVisible()).
|
|
281
|
+
const flag = /^\s*!?\s*([A-Za-z_$][\w$]*)\s*$/.exec(cond)?.[1];
|
|
282
|
+
const flagFromPage = flag && new RegExp(`\\b(?:const|let|var)\\s+${flag}\\s*=[^;\\n]*(?:\\.count\\(|\\.isVisible\\(|getBy\\w+\\(|\\bfind\\(|\\.locator\\()`).test(content);
|
|
283
|
+
if (looksAtPage || flagFromPage)
|
|
284
|
+
out.push(cond.replace(/\s+/g, ' ').trim().slice(0, 80));
|
|
285
|
+
}
|
|
286
|
+
return out;
|
|
287
|
+
}
|
|
288
|
+
/** Literal names a test locates by: getByRole(…, { name }), getByLabel, getByPlaceholder. */
|
|
289
|
+
export function usedLabels(content) {
|
|
290
|
+
const out = new Set();
|
|
291
|
+
const res = [
|
|
292
|
+
/getByRole\(\s*['"`][\w-]+['"`]\s*,\s*\{[^}]*?\bname\s*:\s*(['"`])((?:(?!\1).)+)\1/g,
|
|
293
|
+
/getBy(?:Label|Placeholder)\(\s*(['"`])((?:(?!\1).)+)\1/g,
|
|
294
|
+
/\bfind\(\s*\w+\s*,\s*['"`][\w-]+['"`]\s*,\s*(['"`])((?:(?!\1).)+)\1/g,
|
|
295
|
+
];
|
|
296
|
+
for (const re of res) {
|
|
297
|
+
for (const m of content.matchAll(re)) {
|
|
298
|
+
const name = m[2].replace(/\s+/g, ' ').trim();
|
|
299
|
+
if (name.length >= 3 && !name.includes('${') && !name.includes('\\'))
|
|
300
|
+
out.add(name);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return [...out];
|
|
304
|
+
}
|
|
240
305
|
/**
|
|
241
306
|
* Remove pending test blocks (`it/test/describe.skip|fixme|todo(...)`) so the
|
|
242
307
|
* F9 selector check only ever inspects RUNNABLE code. A guessed selector
|
|
@@ -410,10 +475,104 @@ function looksStructurallyClean(content) {
|
|
|
410
475
|
t.startsWith('"use strict"');
|
|
411
476
|
if (!startsWithCode)
|
|
412
477
|
return false;
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
478
|
+
return unbalancedAt(content) === null;
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Where the brackets of a TS file stop matching, or null when they all do.
|
|
482
|
+
* Strings, template literals (with their \${…}), comments and regex literals
|
|
483
|
+
* are skipped: a "(" inside a test's text is not code. Counting them was what
|
|
484
|
+
* refused correct files as "not valid syntax".
|
|
485
|
+
*/
|
|
486
|
+
export function unbalancedAt(content) {
|
|
487
|
+
const stack = [];
|
|
488
|
+
const pairs = { ')': '(', '}': '{', ']': '[' };
|
|
489
|
+
let line = 1;
|
|
490
|
+
let prev = ''; // last significant character, to tell a regex from a division
|
|
491
|
+
const templates = []; // brace depth at which each open template's \${ began
|
|
492
|
+
for (let i = 0; i < content.length; i++) {
|
|
493
|
+
const c = content[i];
|
|
494
|
+
if (c === '\n') {
|
|
495
|
+
line++;
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
if (/\s/.test(c))
|
|
499
|
+
continue;
|
|
500
|
+
if (c === '/' && content[i + 1] === '/') {
|
|
501
|
+
while (i < content.length && content[i] !== '\n')
|
|
502
|
+
i++;
|
|
503
|
+
line++;
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
if (c === '/' && content[i + 1] === '*') {
|
|
507
|
+
i += 2;
|
|
508
|
+
while (i < content.length && !(content[i] === '*' && content[i + 1] === '/')) {
|
|
509
|
+
if (content[i] === '\n')
|
|
510
|
+
line++;
|
|
511
|
+
i++;
|
|
512
|
+
}
|
|
513
|
+
i++;
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
if (c === "'" || c === '"') {
|
|
517
|
+
for (i++; i < content.length && content[i] !== c && content[i] !== '\n'; i++)
|
|
518
|
+
if (content[i] === '\\')
|
|
519
|
+
i++;
|
|
520
|
+
prev = c;
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
if (c === '`' || (c === '}' && templates.length && templates[templates.length - 1] === stack.length)) {
|
|
524
|
+
if (c === '}')
|
|
525
|
+
templates.pop();
|
|
526
|
+
// Inside a template: up to its end, or to a \${ that opens code.
|
|
527
|
+
for (i++; i < content.length; i++) {
|
|
528
|
+
if (content[i] === '\\') {
|
|
529
|
+
i++;
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
if (content[i] === '\n')
|
|
533
|
+
line++;
|
|
534
|
+
if (content[i] === '`')
|
|
535
|
+
break;
|
|
536
|
+
if (content[i] === '$' && content[i + 1] === '{') {
|
|
537
|
+
i++;
|
|
538
|
+
templates.push(stack.length);
|
|
539
|
+
break;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
prev = '`';
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
if (c === '/' && (!prev || /[(,=:[!&|?{};+\-*%<>~^]/.test(prev))) {
|
|
546
|
+
// A regex literal: skip to its closing slash, minding classes and escapes.
|
|
547
|
+
let inClass = false;
|
|
548
|
+
for (i++; i < content.length && content[i] !== '\n'; i++) {
|
|
549
|
+
if (content[i] === '\\') {
|
|
550
|
+
i++;
|
|
551
|
+
continue;
|
|
552
|
+
}
|
|
553
|
+
if (content[i] === '[')
|
|
554
|
+
inClass = true;
|
|
555
|
+
else if (content[i] === ']')
|
|
556
|
+
inClass = false;
|
|
557
|
+
else if (content[i] === '/' && !inClass)
|
|
558
|
+
break;
|
|
559
|
+
}
|
|
560
|
+
while (/[a-z]/i.test(content[i + 1] ?? ''))
|
|
561
|
+
i++;
|
|
562
|
+
prev = 'r';
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
if (c === '(' || c === '{' || c === '[')
|
|
566
|
+
stack.push({ ch: c, line });
|
|
567
|
+
else if (c in pairs) {
|
|
568
|
+
const open = stack.pop();
|
|
569
|
+
if (!open || open.ch !== pairs[c])
|
|
570
|
+
return `line ${line}: "${c}" does not close ${open ? `the "${open.ch}" from line ${open.line}` : 'anything'}`;
|
|
571
|
+
}
|
|
572
|
+
prev = c;
|
|
573
|
+
}
|
|
574
|
+
const open = stack.pop();
|
|
575
|
+
return open ? `the "${open.ch}" from line ${open.line} is never closed` : null;
|
|
417
576
|
}
|
|
418
577
|
const RESOLVE_EXTS = ['', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json'];
|
|
419
578
|
/** Relative module specifiers (import / require / dynamic import) that reach no file from `filePath`. */
|
|
@@ -491,13 +650,18 @@ function testCriteria(content) {
|
|
|
491
650
|
starts.push(i); });
|
|
492
651
|
return starts.map((start, n) => {
|
|
493
652
|
const end = starts[n + 1] ?? lines.length;
|
|
494
|
-
|
|
653
|
+
// The oracle comment sits right above the test or inside it.
|
|
654
|
+
const above = [];
|
|
655
|
+
for (let i = start - 1; i > (starts[n - 1] ?? -1) && /^\s*\/\//.test(lines[i] ?? ''); i--)
|
|
656
|
+
above.push(i);
|
|
657
|
+
for (const i of [...above, ...Array.from({ length: end - start }, (_, k) => start + k)]) {
|
|
495
658
|
const m = /\/\/\s*oracle:\s*(.+)$/.exec(lines[i]);
|
|
496
659
|
if (m) {
|
|
497
660
|
const tier = (/\[(spec|baseline|observed)\]/i.exec(m[1])?.[1]?.toLowerCase() ?? 'untagged');
|
|
498
|
-
// The criterion is the
|
|
661
|
+
// The criterion is the rule id when the oracle names one, else the quoted rule, else the oracle text.
|
|
662
|
+
const id = /^\s*(R\d+)\b/.exec(m[1])?.[1];
|
|
499
663
|
const quoted = /["“]([^"”]{12,})["”]/.exec(m[1]);
|
|
500
|
-
return { criterion: (quoted ? quoted[1] : m[1]).trim().toLowerCase(), tier };
|
|
664
|
+
return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
|
|
501
665
|
}
|
|
502
666
|
}
|
|
503
667
|
return { criterion: '', tier: 'untagged' };
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { spawn } from 'node:child_process';
|
|
14
14
|
import { readFileSync, existsSync } from 'node:fs';
|
|
15
|
-
import { loadConfig, splitCommand, withReadOnlyTools, resolveFramework, } from '../project/config.js';
|
|
16
|
-
import { runAiSync, noteAiCall } from './ai-usage.js';
|
|
15
|
+
import { loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
|
|
16
|
+
import { runAiSync, noteAiCall, traceArgs, traceCall } from './ai-usage.js';
|
|
17
17
|
const MAP_PATH = 'testforge.md';
|
|
18
18
|
/**
|
|
19
19
|
* Spawn a command, feed `input` on stdin, resolve with its stdout.
|
|
@@ -23,7 +23,7 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
|
|
|
23
23
|
const started = Date.now();
|
|
24
24
|
let produced = '';
|
|
25
25
|
return new Promise((resolve, reject) => {
|
|
26
|
-
const child = spawn(program, args, {
|
|
26
|
+
const child = spawn(program, traceArgs(program, args), {
|
|
27
27
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
28
28
|
});
|
|
29
29
|
let out = '';
|
|
@@ -40,6 +40,8 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
|
|
|
40
40
|
// otherwise never close — kill it and surface a clear, resumable error.
|
|
41
41
|
const timer = setTimeout(() => {
|
|
42
42
|
child.kill('SIGKILL');
|
|
43
|
+
if (!done)
|
|
44
|
+
traceCall(input, out, started, `timed out after ${timeoutMs}ms`);
|
|
43
45
|
finish(() => reject(new Error(`${program} timed out after ${timeoutMs}ms — the AI CLI did not respond. ` +
|
|
44
46
|
`Check it runs non-interactively (auth set up, no TTY prompt), ` +
|
|
45
47
|
`or raise aiTimeoutMs in testforge.config.json.`)));
|
|
@@ -52,8 +54,9 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
|
|
|
52
54
|
finish(() => {
|
|
53
55
|
produced = out;
|
|
54
56
|
if (code === 0)
|
|
55
|
-
resolve(out);
|
|
57
|
+
resolve(traceCall(input, out, started));
|
|
56
58
|
else {
|
|
59
|
+
traceCall(input, out, started, `exited ${code}`);
|
|
57
60
|
// The limit message is not always on stderr — some CLI versions
|
|
58
61
|
// print it to stdout, some print nothing at all. Say what we have,
|
|
59
62
|
// and say plainly when that is nothing, so a silent exit is never
|
|
@@ -67,7 +70,7 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
|
|
|
67
70
|
}).finally(() => noteAiCall(input.length, produced.length, Date.now() - started));
|
|
68
71
|
}
|
|
69
72
|
/** Build the prompt we send to the AI for one journey. */
|
|
70
|
-
export function buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding) {
|
|
73
|
+
export function buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding, memory) {
|
|
71
74
|
const config = loadConfig();
|
|
72
75
|
// Opinionated: testforge only generates Jest (api/backend) or Playwright
|
|
73
76
|
// (web/UI) tests — any other framework (e.g. a project's Cypress setup)
|
|
@@ -86,7 +89,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
86
89
|
const language = isJs
|
|
87
90
|
? 'plain JavaScript (CommonJS: use require/module.exports, NOT import; NO TypeScript type annotations)'
|
|
88
91
|
: 'TypeScript';
|
|
89
|
-
|
|
92
|
+
const prompt = [
|
|
90
93
|
`You are a senior QA engineer. Generate a complete ${framework}`,
|
|
91
94
|
`for the user journey described below. (The specific file to write is`,
|
|
92
95
|
`stated at the very end, after the journey — read all the context first.)`,
|
|
@@ -297,32 +300,43 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
297
300
|
`adds no new outcome is removed. When in doubt, report LESS: a missing test`,
|
|
298
301
|
`costs one line of coverage; a false failure costs the team a release.`,
|
|
299
302
|
``,
|
|
300
|
-
...(
|
|
303
|
+
...(memory
|
|
301
304
|
? [
|
|
302
|
-
`
|
|
303
|
-
`below
|
|
304
|
-
`
|
|
305
|
-
`
|
|
306
|
-
`
|
|
307
|
-
`
|
|
308
|
-
`
|
|
309
|
-
`
|
|
310
|
-
`the CODE FACTS section marks "unknown" or genuinely does not`,
|
|
311
|
-
`cover. Those facts are the MECHANICS: HOW to reach and`,
|
|
312
|
-
`drive the behaviour. But the EXPECTED OUTCOME you`,
|
|
305
|
+
`EVERYTHING YOU NEED IS IN THIS MESSAGE. You have no tools and cannot`,
|
|
306
|
+
`open files. The CODE FACTS below were read from this project's source`,
|
|
307
|
+
`once for the whole journey, the TESTKIT list is the harness's real`,
|
|
308
|
+
`exports, and the JOURNEY block holds exactly the rules this file tests`,
|
|
309
|
+
`with their context. Those are the MECHANICS: HOW to reach and drive`,
|
|
310
|
+
`the behaviour. Never invent a selector, route, field or helper that`,
|
|
311
|
+
`appears nowhere below; a case you cannot reach with what is here is`,
|
|
312
|
+
`left out. But the EXPECTED OUTCOME you`,
|
|
313
313
|
]
|
|
314
|
-
:
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
314
|
+
: grounding
|
|
315
|
+
? [
|
|
316
|
+
`THE MECHANICS ARE ALREADY ESTABLISHED. The CODE FACTS section`,
|
|
317
|
+
`below was read from this project's real source for this journey —`,
|
|
318
|
+
`routes with their full mounted paths, auth behaviour, request and`,
|
|
319
|
+
`response shapes, models, externals, and the selectors the screens`,
|
|
320
|
+
`genuinely expose. Use it. Do NOT re-derive it: opening the same`,
|
|
321
|
+
`controllers and components again for every file in a journey is`,
|
|
322
|
+
`what makes generation take an hour instead of minutes, and it`,
|
|
323
|
+
`arrives at the same answer. Read source only for a specific detail`,
|
|
324
|
+
`the CODE FACTS section marks "unknown" or genuinely does not`,
|
|
325
|
+
`cover. Those facts are the MECHANICS: HOW to reach and`,
|
|
326
|
+
`drive the behaviour. But the EXPECTED OUTCOME you`,
|
|
327
|
+
]
|
|
328
|
+
: [
|
|
329
|
+
`GROUND EVERYTHING IN THE REAL SOURCE. You are running inside the`,
|
|
330
|
+
`project's repository and have read-only tools (Read, Grep, Glob, LS).`,
|
|
331
|
+
`BEFORE writing tests, look up the actual code for this journey — use`,
|
|
332
|
+
`the route/endpoint named below to grep for the router, then read the`,
|
|
333
|
+
`route file, its controller/handler, the service it calls, and the`,
|
|
334
|
+
`models it touches. Confirm the REAL endpoint paths, HTTP methods,`,
|
|
335
|
+
`request/response shapes, status codes, auth requirements, and function`,
|
|
336
|
+
`signatures from the code — that is the MECHANICS: HOW to reach and`,
|
|
337
|
+
`drive the behaviour (paths, methods, payload/response shape,`,
|
|
338
|
+
`selectors, model/collection names). But the EXPECTED OUTCOME you`,
|
|
339
|
+
]),
|
|
326
340
|
`assert on is the ORACLE, and its source of truth is the JOURNEY`,
|
|
327
341
|
`DOCUMENT below — NOT what the code currently does. The code tells you`,
|
|
328
342
|
`HOW; the document tells you WHAT is correct. If the code's real`,
|
|
@@ -394,6 +408,23 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
394
408
|
` 3. getByText(...) — real on-screen text`,
|
|
395
409
|
` 4. locator('[name="…"]') / by id — a real input name/id attribute`,
|
|
396
410
|
` 5. getByTestId(...) / [data-cy=…] — ONLY if that hook truly exists`,
|
|
411
|
+
`FIND EVERY ELEMENT WITH kit.find(page, kind, name): kind is what the doc`,
|
|
412
|
+
`says it is (button, link, input, dropdown, checkbox, radio, toggle, tab,`,
|
|
413
|
+
`menuitem, dialog, heading, alert, toast, image, list, table …), name is`,
|
|
414
|
+
`the text the doc gives it. It tries the proper role first and falls back`,
|
|
415
|
+
`to the label, placeholder, aria-label, alt text and visible text, adding`,
|
|
416
|
+
`a finding when the element is not built as what it is — so a test written`,
|
|
417
|
+
`before the feature still finds it once it ships. Use page.getBy* only for`,
|
|
418
|
+
`what kit.find cannot express. CHECK THE OUTCOME, NOT THE BUILD: the URL`,
|
|
419
|
+
`the app lands on, the request it sends and its data, the text and state`,
|
|
420
|
+
`the user sees after, what survives a refresh — never a tag, a class or`,
|
|
421
|
+
`the page's structure. A missing element is a failure, never a skip:`,
|
|
422
|
+
`skip only when the ENVIRONMENT lacks something (an account, a variable).`,
|
|
423
|
+
`A STATE THE TEST CANNOT CREATE (a disputed or cancelled booking, an`,
|
|
424
|
+
`expired code, a banned account, an empty list) is STUBBED, not skipped:`,
|
|
425
|
+
`answer the API call that returns it with page.route(), shaped exactly as`,
|
|
426
|
+
`the response type in the CODE FACTS, with only the fields that state needs`,
|
|
427
|
+
`changed, and put "(stubbed state)" in the test title. Then test the screen.`,
|
|
397
428
|
`The first four are present in almost any real DOM without special`,
|
|
398
429
|
`hooks, so a page with zero data-testids is still fully groundable —`,
|
|
399
430
|
`ground on role/label/text/name instead of inventing a test-id.`,
|
|
@@ -469,9 +500,10 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
469
500
|
`- Never write "TODO" or "FIXME" in the file. Something you cannot`,
|
|
470
501
|
` establish is left out, never a note to finish later.`,
|
|
471
502
|
``,
|
|
472
|
-
|
|
473
|
-
map,
|
|
474
|
-
|
|
503
|
+
// A web file's slice already names its screens; the data-layer map is for API files.
|
|
504
|
+
...(memory && !isApi ? [] : [`--- PROJECT MAP (data layer) ---`, map, ``]),
|
|
505
|
+
// The same for every file of the journey, so it sits in the cached shared part.
|
|
506
|
+
...(memory ? [`--- TESTKIT (the shared harness's exports) ---`, memory.helpers, ``] : []),
|
|
475
507
|
// Read from the project's real source once per journey, before any file is
|
|
476
508
|
// written, and handed unchanged to every file in it. Sits with the map and
|
|
477
509
|
// the journey so the whole block stays a stable prefix across the journey.
|
|
@@ -489,12 +521,19 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
489
521
|
``,
|
|
490
522
|
]
|
|
491
523
|
: []),
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
524
|
+
...(memory
|
|
525
|
+
? [
|
|
526
|
+
`--- JOURNEY (only the part this file tests; other files test the rest) ---`,
|
|
527
|
+
memory.journey,
|
|
528
|
+
]
|
|
529
|
+
: [
|
|
530
|
+
`--- JOURNEY ---`,
|
|
531
|
+
`It may include sections beyond fields — actions, expected results,`,
|
|
532
|
+
`negative cases, race conditions, webhooks, security notes, and a`,
|
|
533
|
+
`Data section. Use ALL of it.`,
|
|
534
|
+
``,
|
|
535
|
+
journey.rawMarkdown,
|
|
536
|
+
]),
|
|
498
537
|
// ---- PER-FILE TASK (kept LAST so the big context above stays a stable,
|
|
499
538
|
// cache-reusable prefix across every file in this journey, and so the
|
|
500
539
|
// concrete task is the most recent thing the model reads) ----
|
|
@@ -510,6 +549,13 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
510
549
|
`cover those. Go DEEP on this one concern (happy path + its own`,
|
|
511
550
|
`edge/negative/security cases) rather than broad-but-shallow.`,
|
|
512
551
|
`Use this exact describe() title: "${focus.title}".`,
|
|
552
|
+
...(memory?.rules.length
|
|
553
|
+
? [
|
|
554
|
+
`Test exactly these rules: ${memory.rules.join(', ')}. Name the rule id AND its`,
|
|
555
|
+
`tag in each test's oracle comment, e.g. "// oracle: ${memory.rules[0]} [spec] …":`,
|
|
556
|
+
`the tag is how a failure is later graded as a defect or a finding.`,
|
|
557
|
+
]
|
|
558
|
+
: []),
|
|
513
559
|
]
|
|
514
560
|
: []),
|
|
515
561
|
...(testPath
|
|
@@ -523,9 +569,63 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
523
569
|
]
|
|
524
570
|
: []),
|
|
525
571
|
``,
|
|
572
|
+
...(memory
|
|
573
|
+
? [
|
|
574
|
+
`YOUR FILE IS REFUSED, AND WRITTEN AGAIN, IF IT:`,
|
|
575
|
+
`- calls page.waitForTimeout() or sleeps for a fixed time;`,
|
|
576
|
+
`- skips a test because an element is missing (test.skip(true, …), or a`,
|
|
577
|
+
` condition that counts or looks for an element) — that is a failure to assert;`,
|
|
578
|
+
`- has a test with no expect(), or more than ${maxPerRule()} tests for one rule;`,
|
|
579
|
+
`- has no test at all: every rule gets one; stub a state you cannot create;`,
|
|
580
|
+
`- names a button, link, field or label that is in neither the CODE FACTS nor the rules;`,
|
|
581
|
+
`- defines its own helper functions (use the TESTKIT) or comments anything but`,
|
|
582
|
+
` the one "// oracle: <rule id> [tag] …" line above each test.`,
|
|
583
|
+
`Keep each test short: the steps, then the expect()s. No logging.`,
|
|
584
|
+
``,
|
|
585
|
+
]
|
|
586
|
+
: []),
|
|
526
587
|
`Now output ONLY the raw ${isJs ? 'JavaScript' : 'TypeScript'} test code`,
|
|
527
588
|
`for THIS file — the first character must be the first line of code.`,
|
|
528
589
|
].join('\n');
|
|
590
|
+
return memory ? withoutFileReading(prompt) : prompt;
|
|
591
|
+
}
|
|
592
|
+
/**
|
|
593
|
+
* A tool-less call's prompt in two: the part every file of the journey shares
|
|
594
|
+
* (the rules for writing a test, the harness, the language), sent as the
|
|
595
|
+
* system prompt so it is cached once and read cheaply by every later write and
|
|
596
|
+
* repair, and the file's own part (its facts, its slice, its task) on stdin.
|
|
597
|
+
*/
|
|
598
|
+
export function splitShared(prompt) {
|
|
599
|
+
const at = ['\n--- CODE FACTS', '\n--- JOURNEY (only'].map((m) => prompt.indexOf(m)).filter((i) => i > 0);
|
|
600
|
+
if (!at.length)
|
|
601
|
+
return { system: '', message: prompt };
|
|
602
|
+
const cut = Math.min(...at);
|
|
603
|
+
return { system: prompt.slice(0, cut).trim(), message: prompt.slice(cut).trim() };
|
|
604
|
+
}
|
|
605
|
+
/** The reason the previous answer was refused, after the task, where it is read last. */
|
|
606
|
+
function withRefusal(prompt, refusal) {
|
|
607
|
+
return refusal
|
|
608
|
+
? `${prompt}\n\n--- YOUR PREVIOUS ANSWER WAS REFUSED ---\nIt was not run. Reason: ${refusal}\nWrite it again without that problem.`
|
|
609
|
+
: prompt;
|
|
610
|
+
}
|
|
611
|
+
/** Program and args for one call: tool-less with the shared part as system prompt, or the old explorer. */
|
|
612
|
+
function writerCommand(prompt, memory) {
|
|
613
|
+
const cmd = splitCommand(loadConfig().aiCommand);
|
|
614
|
+
if (!memory)
|
|
615
|
+
return { ...withReadOnlyTools(cmd), input: prompt };
|
|
616
|
+
const { system, message } = splitShared(prompt);
|
|
617
|
+
return system ? { ...withNoTools(cmd, system), input: message } : { ...withNoTools(cmd), input: prompt };
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* The general rules tell a writer with tools to go and read a component or
|
|
621
|
+
* the testkit. A writer without tools is pointed at the block that already
|
|
622
|
+
* holds it instead.
|
|
623
|
+
*/
|
|
624
|
+
function withoutFileReading(prompt) {
|
|
625
|
+
return prompt
|
|
626
|
+
.replace(/the UI\. FIRST read this page's real components: grep the webapp\nsource for the route, page name, and on-screen text, then open\nthose component files and use only locators that GENUINELY exist\./, 'the UI. Use only locators that the CODE FACTS or the JOURNEY block below\nshow GENUINELY exist.')
|
|
627
|
+
.replace('Read a harness helper before calling it, to see what it really returns.', 'Call a testkit helper exactly as its line under TESTKIT describes it.')
|
|
628
|
+
.replace(/Read that file to confirm its\nexact exports before using them\./, 'Its real exports are listed under\nTESTKIT below.');
|
|
529
629
|
}
|
|
530
630
|
/**
|
|
531
631
|
* Call the configured AI command to generate a test file for one
|
|
@@ -558,86 +658,84 @@ export function generateWithAI(journey, testPath, focus, grounding) {
|
|
|
558
658
|
*/
|
|
559
659
|
/** Marks a test whose failure the repair judged to be the application's. */
|
|
560
660
|
/** The most tests one file may spend on one criterion (config maxTestsPerRule, default 4). */
|
|
561
|
-
export const maxPerRule = () => loadConfig().maxTestsPerRule ??
|
|
661
|
+
export const maxPerRule = () => loadConfig().maxTestsPerRule ?? 2;
|
|
562
662
|
export const DEFECT_MARK = 'testforge-defect:';
|
|
563
663
|
/** A failure that is real application behaviour against a [baseline]/[observed] rule: kept failing, never a defect. */
|
|
564
664
|
export const FINDING_MARK = 'testforge-finding:';
|
|
565
665
|
/**
|
|
566
|
-
*
|
|
567
|
-
*
|
|
568
|
-
*
|
|
569
|
-
*
|
|
570
|
-
*
|
|
571
|
-
* test — because either turns a caught defect into a silent one.
|
|
666
|
+
* Tests for some rules of a file that already exists: new rules, or rules
|
|
667
|
+
* whose wording changed. The AI sees the file's top (imports, setup), the old
|
|
668
|
+
* tests of a changed rule, and the slice for those rules only, and answers
|
|
669
|
+
* with test blocks that are put into the file in place. The rest of the file
|
|
670
|
+
* is neither sent nor rewritten.
|
|
572
671
|
*/
|
|
573
|
-
export async function
|
|
574
|
-
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
575
|
-
const base = buildPrompt(journey, map, testPath, undefined, harnessPath, transport, grounding);
|
|
672
|
+
export async function writeTestsWithAI(journey, testPath, fileHead, title, oldTests, harnessPath, transport, grounding, memory, refusal) {
|
|
576
673
|
const prompt = [
|
|
577
|
-
|
|
674
|
+
buildPrompt(journey, '', testPath, { title, covers: `Only the rules ${memory.rules.join(', ')}.` }, harnessPath, transport, grounding, memory),
|
|
578
675
|
'',
|
|
579
|
-
'
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
'output is the ground truth: it tells you what the application actually',
|
|
583
|
-
'renders, routes and returns.',
|
|
676
|
+
'--- THIS FILE ALREADY EXISTS ---',
|
|
677
|
+
'Its top (imports and setup), which your tests run under:',
|
|
678
|
+
fileHead,
|
|
584
679
|
'',
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
' it wrongly: a locator that matched nothing or the wrong',
|
|
589
|
-
' element, a container that does not hold the element, a wait',
|
|
590
|
-
' for the wrong event, a response field the server does not',
|
|
591
|
-
' send, a precondition its own setup got wrong, an assertion',
|
|
592
|
-
' the journey document does not state. FIX the test.',
|
|
593
|
-
' A "strict mode violation" means the locator matched several',
|
|
594
|
-
' elements: narrow it to the one the criterion means — never',
|
|
595
|
-
' .first() on a guess.',
|
|
596
|
-
' When the app sent the test back to an EARLIER step or page,',
|
|
597
|
-
' first suspect a missing precondition the earlier steps',
|
|
598
|
-
' would have set: reach the step with kit.openStep().',
|
|
599
|
-
' APP DEFECT — the test reached the behaviour (the element or response it',
|
|
600
|
-
' checks is there) and the application contradicts a criterion',
|
|
601
|
-
' the journey document states. Change NOTHING in that test.',
|
|
602
|
-
` Put one comment line directly above its declaration:`,
|
|
603
|
-
` // ${DEFECT_MARK} <the criterion, quoted with its tag> — <what the app did instead>`,
|
|
604
|
-
` Only a [spec] criterion can be a defect. When the criterion is`,
|
|
605
|
-
` [baseline] or [observed], write the same line with`,
|
|
606
|
-
` "${FINDING_MARK}" instead: it stays failing, as a finding.`,
|
|
607
|
-
' SKIPPED — a test listed under SKIPPED skipped itself. If its own',
|
|
608
|
-
' setup looked in the wrong place (wrong field, wrong route,',
|
|
609
|
-
' wrong id), fix that so it runs. If what it needs truly does',
|
|
610
|
-
' not exist in this environment, leave it exactly as it is.',
|
|
680
|
+
oldTests
|
|
681
|
+
? `These tests were written for an earlier wording of ${memory.rules.join(', ')}. Rewrite them for the rules as they are now:\n${oldTests}`
|
|
682
|
+
: `The file already tests its other rules. Write tests only for ${memory.rules.join(', ')}.`,
|
|
611
683
|
'',
|
|
612
|
-
'
|
|
613
|
-
'
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
684
|
+
'Output ONLY the test(...) blocks, each with its "// oracle:" comment above it: no imports, no',
|
|
685
|
+
'describe, no setup. The first character of your answer is the first line of code.',
|
|
686
|
+
].join('\n');
|
|
687
|
+
const config = loadConfig();
|
|
688
|
+
const { program, args, input } = writerCommand(withRefusal(prompt, refusal), memory);
|
|
689
|
+
return cleanOutput(await runWithStdin(program, args, input, config.aiTimeoutMs));
|
|
690
|
+
}
|
|
691
|
+
/** The line that opens each file in a batch answer. */
|
|
692
|
+
const FILE_MARK = (path) => `// ===== FILE: ${path} =====`;
|
|
693
|
+
/**
|
|
694
|
+
* Several files of one journey in one call. Their shared part (the rules for
|
|
695
|
+
* writing a test, the harness, the testkit) is the cached system prompt once;
|
|
696
|
+
* each file adds only its own facts, rules and task. The answer holds every
|
|
697
|
+
* file after its marker line; a file missing from it is simply not returned.
|
|
698
|
+
*/
|
|
699
|
+
export async function writeFilesWithAI(files) {
|
|
700
|
+
const parts = files.map((f) => ({ ...f, ...splitShared(f.prompt) }));
|
|
701
|
+
const system = parts[0].system;
|
|
702
|
+
const message = [
|
|
703
|
+
`Write ${files.length} test files. Each one's facts, rules and task follow; the`,
|
|
704
|
+
`rules for writing a test are the same for all of them.`,
|
|
627
705
|
'',
|
|
628
|
-
|
|
706
|
+
...parts.flatMap((p, k) => [`=== FILE ${k + 1} of ${files.length}: ${p.path} ===`, p.message, '']),
|
|
707
|
+
`OUTPUT: all ${files.length} files, in this order. Before each, one line exactly:`,
|
|
708
|
+
...files.map((f) => ` ${FILE_MARK(f.path)}`),
|
|
709
|
+
`then that file's code. Nothing before the first line and nothing after the last file.`,
|
|
629
710
|
].join('\n');
|
|
630
711
|
const config = loadConfig();
|
|
631
|
-
const
|
|
632
|
-
const
|
|
633
|
-
return
|
|
712
|
+
const cmd = splitCommand(config.aiCommand);
|
|
713
|
+
const { program, args } = system ? withNoTools(cmd, system) : withNoTools(cmd);
|
|
714
|
+
return splitBatchAnswer(await runWithStdin(program, args, message, config.aiTimeoutMs), files.map((f) => f.path));
|
|
715
|
+
}
|
|
716
|
+
/** Each file's code from a batch answer, by its marker line. */
|
|
717
|
+
export function splitBatchAnswer(out, paths) {
|
|
718
|
+
const result = new Map();
|
|
719
|
+
const marks = paths
|
|
720
|
+
.map((path) => ({ path, at: out.indexOf(FILE_MARK(path)) }))
|
|
721
|
+
.filter((m) => m.at >= 0)
|
|
722
|
+
.sort((a, b) => a.at - b.at);
|
|
723
|
+
marks.forEach((m, k) => {
|
|
724
|
+
const end = k + 1 < marks.length ? marks[k + 1].at : out.length;
|
|
725
|
+
const code = cleanOutput(out.slice(m.at + FILE_MARK(m.path).length, end));
|
|
726
|
+
if (code.trim())
|
|
727
|
+
result.set(m.path, code);
|
|
728
|
+
});
|
|
729
|
+
return result;
|
|
634
730
|
}
|
|
635
|
-
export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport, grounding
|
|
731
|
+
export async function generateWithAIAsync(journey, testPath, focus, harnessPath, transport, grounding, memory,
|
|
732
|
+
/** Why the last answer was refused before it ran; told to the next try. */
|
|
733
|
+
refusal) {
|
|
636
734
|
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
637
|
-
const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding);
|
|
735
|
+
const prompt = buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding, memory);
|
|
638
736
|
const config = loadConfig();
|
|
639
|
-
const { program, args } =
|
|
640
|
-
const stdout = await runWithStdin(program, args,
|
|
737
|
+
const { program, args, input } = writerCommand(withRefusal(prompt, refusal), memory);
|
|
738
|
+
const stdout = await runWithStdin(program, args, input, config.aiTimeoutMs);
|
|
641
739
|
return cleanOutput(stdout);
|
|
642
740
|
}
|
|
643
741
|
/**
|