@activate-core/test-forge 2.6.0 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/cli/index.js +4 -1
- package/dist/core/docs/excerpts.js +49 -3
- package/dist/core/docs/grounding.js +4 -0
- package/dist/core/gates/validate.js +49 -23
- package/dist/core/generate/ai-generator.js +41 -0
- package/dist/core/generate/ai-usage.js +1 -0
- package/dist/core/generate/complete-file.js +46 -7
- package/dist/core/generate/run-generate.js +224 -18
- package/dist/core/generate/test-blocks.js +87 -0
- package/dist/core/project/env-diff.js +3 -1
- package/dist/core/project/setup.js +10 -2
- package/dist/core/prompts/blocks.js +13 -5
- package/dist/core/run/name-scan.js +58 -3
- package/dist/core/run/report.js +2 -0
- package/dist/core/run/run-tests.js +5 -0
- package/package.json +3 -2
- package/reporters/interface.cjs +26 -3
|
@@ -26,12 +26,46 @@ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, is
|
|
|
26
26
|
import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
|
-
import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
|
-
import { completeFile } from './complete-file.js';
|
|
31
|
-
import { tuckStrayTests } from './test-blocks.js';
|
|
29
|
+
import { rewriteSetupWithAI, buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
|
+
import { completeFile, withoutBlankNames, withoutNames } from './complete-file.js';
|
|
31
|
+
import { findSetupBlocks, findTestBlocks, tuckStrayTests, useInsertIds } from './test-blocks.js';
|
|
32
32
|
import { passedTitles } from './passed.js';
|
|
33
|
+
import { scanNames } from '../run/name-scan.js';
|
|
33
34
|
import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
|
|
34
35
|
/** Names the file asks for that testforge/interface.json still has blank. */
|
|
36
|
+
/**
|
|
37
|
+
* Is a blank name used outside every test (a hook, a helper, the file's
|
|
38
|
+
* setup)? Then the whole file waits on it, and only writing the whole file
|
|
39
|
+
* again — with the code facts — clears it; no test of it can have passed.
|
|
40
|
+
*/
|
|
41
|
+
export function blankUsedOutsideTests(content) {
|
|
42
|
+
const blocks = findTestBlocks(content);
|
|
43
|
+
const outside = blocks.reduceRight((c, b) => c.slice(0, b.start) + c.slice(b.end), content);
|
|
44
|
+
return blankNames(content).some((n) => {
|
|
45
|
+
const esc = n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
46
|
+
const v = new RegExp(`\\b(?:const|let|var)\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*need\\(\\s*(['"\`])${esc}\\2`).exec(content)?.[1];
|
|
47
|
+
if (!v)
|
|
48
|
+
return false;
|
|
49
|
+
const uses = outside.match(new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`, 'g')) ?? [];
|
|
50
|
+
return uses.length > 1; // more than its own declaration
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Where a file's blank names are used outside its tests: 'setup' when every
|
|
55
|
+
* such use sits in a hook or helper TestForge can rewrite on its own, 'file'
|
|
56
|
+
* when one sits somewhere it cannot split out (a parametrised table, a test
|
|
57
|
+
* built in a loop), undefined when none.
|
|
58
|
+
*/
|
|
59
|
+
export function blankOutsideKind(content) {
|
|
60
|
+
if (!blankUsedOutsideTests(content))
|
|
61
|
+
return undefined;
|
|
62
|
+
const vars = [...content.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]+)\2/g)]
|
|
63
|
+
.filter((m) => blankNames(`need('${m[3]}')`).length > 0).map((m) => m[1]);
|
|
64
|
+
const covered = [...findTestBlocks(content), ...findSetupBlocks(content)];
|
|
65
|
+
const rest = covered.sort((a, b) => b.start - a.start).reduce((c, b) => c.slice(0, b.start) + c.slice(b.end), content);
|
|
66
|
+
const used = vars.some((v) => (rest.match(new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`, 'g')) ?? []).length > 1);
|
|
67
|
+
return used ? 'file' : 'setup';
|
|
68
|
+
}
|
|
35
69
|
/** Does this test use a name the product has not given yet (directly, or through the constant it was put in)? */
|
|
36
70
|
function usesBlankName(content, block) {
|
|
37
71
|
const text = content.slice(block.start, block.end);
|
|
@@ -43,6 +77,20 @@ function usesBlankName(content, block) {
|
|
|
43
77
|
return !!v && new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(text);
|
|
44
78
|
});
|
|
45
79
|
}
|
|
80
|
+
/** The names a testkit exports (module.exports = { … } or export …). */
|
|
81
|
+
export function exportNames(src) {
|
|
82
|
+
const names = new Set();
|
|
83
|
+
const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(src);
|
|
84
|
+
if (cjs)
|
|
85
|
+
for (const n of cjs[1].split(',')) {
|
|
86
|
+
const m = /^\s*(\w+)/.exec(n);
|
|
87
|
+
if (m)
|
|
88
|
+
names.add(m[1]);
|
|
89
|
+
}
|
|
90
|
+
for (const m of src.matchAll(/^export\s+(?:async\s+)?(?:function\*?|const|let|class)\s+(\w+)/gm))
|
|
91
|
+
names.add(m[1]);
|
|
92
|
+
return [...names];
|
|
93
|
+
}
|
|
46
94
|
function blankNames(content) {
|
|
47
95
|
const iface = readInterface();
|
|
48
96
|
return [...content.matchAll(/\bneed\(\s*(['"`])([^'"`]+)\1\s*\)/g)].map((m) => m[2]).filter((n) => {
|
|
@@ -54,7 +102,7 @@ import { harnessSteps, ruleCoverage } from '../gates/test-quality.js';
|
|
|
54
102
|
import { normalisePending } from './pending.js';
|
|
55
103
|
import { packageRoot } from '../run/run-tests.js';
|
|
56
104
|
import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
|
|
57
|
-
import { factsCodeMoved, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
|
|
105
|
+
import { factsCodeMoved, journeyCodeHash, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
|
|
58
106
|
import { generateSeed, seedProblem } from './seed-generator.js';
|
|
59
107
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
60
108
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
@@ -64,7 +112,7 @@ import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
|
64
112
|
import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
|
|
65
113
|
import { updateJourney } from './update-journey.js';
|
|
66
114
|
import { sourceRoot, sourceText } from '../docs/excerpts.js';
|
|
67
|
-
import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
115
|
+
import { isSyntaxValid, checkTestGates, trimExtraTests, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
68
116
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
69
117
|
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
70
118
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
@@ -113,7 +161,9 @@ export async function pool(items, limit, fn) {
|
|
|
113
161
|
/** Async version of generateValidated (for concurrent per-file gen). */
|
|
114
162
|
async function generateValidatedAsync(generate, ext, gate,
|
|
115
163
|
/** Why an earlier draft of the same file (from a batch) was refused, with that draft. */
|
|
116
|
-
firstRefusal
|
|
164
|
+
firstRefusal,
|
|
165
|
+
/** Mechanical fixes applied before the gate: what code can fix costs no retry. */
|
|
166
|
+
fix = (c) => c) {
|
|
117
167
|
let content = '';
|
|
118
168
|
let refusal = firstRefusal;
|
|
119
169
|
for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
|
|
@@ -121,7 +171,7 @@ firstRefusal) {
|
|
|
121
171
|
return null; // retrying a dead CLI only burns time
|
|
122
172
|
try {
|
|
123
173
|
// A second try sees what it wrote and why that was refused; asked blind, it made the same file again.
|
|
124
|
-
content = await generate(refusal);
|
|
174
|
+
content = fix(await generate(refusal));
|
|
125
175
|
}
|
|
126
176
|
catch (e) {
|
|
127
177
|
noteAiFailure(e);
|
|
@@ -294,7 +344,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
294
344
|
// command. Test files are untouched.
|
|
295
345
|
console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
|
|
296
346
|
}
|
|
347
|
+
const old = existsSync(info.path) ? readFileSync(info.path, 'utf-8') : '';
|
|
297
348
|
const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
|
|
349
|
+
// The tests already written call the old testkit: a rebuild that fails, or
|
|
350
|
+
// drops something they use, is not taken — the old one stays.
|
|
351
|
+
const lost = old && kit !== null ? exportNames(old).filter((n) => !exportNames(kit).includes(n)) : [];
|
|
352
|
+
if (old && (kit === null || lost.length)) {
|
|
353
|
+
console.log(` kept the existing ${area} test harness: the rebuild ${kit === null ? 'failed' : `dropped ${lost.join(', ')}`}`);
|
|
354
|
+
exportFind(info.path);
|
|
355
|
+
harnessByArea.set(area, info.path);
|
|
356
|
+
return info.path;
|
|
357
|
+
}
|
|
298
358
|
if (kit === null) {
|
|
299
359
|
harnessByArea.set(area, null);
|
|
300
360
|
return undefined;
|
|
@@ -453,6 +513,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
453
513
|
const appTexts = () => framework === 'playwright'
|
|
454
514
|
? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
455
515
|
: undefined;
|
|
516
|
+
const fixTests = (content) => trimExtraTests(content, config.maxTestsPerRule ?? 2);
|
|
456
517
|
const genGate = (content, filePath) => testGate(content) ??
|
|
457
518
|
checkTestGates(content, ext, {
|
|
458
519
|
framework, forbidPlaceholders: true, filePath,
|
|
@@ -463,7 +524,61 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
463
524
|
// to retry (a resume pass fills those in without touching real files).
|
|
464
525
|
// A suite whose tests leave rules out or cannot fail is not done either:
|
|
465
526
|
// it goes on, and only those tests are written again.
|
|
527
|
+
// Rules asked for and refused before, with this doc and this code: not asked again until one moves.
|
|
528
|
+
const refusedPath = join(journeyDir(journey), 'refused.json');
|
|
529
|
+
let refusedKeyMemo;
|
|
530
|
+
const refusedKey = () => (refusedKeyMemo ??= `${hash}:${journeyCodeHash(journey, sourceRoot())}`);
|
|
531
|
+
const readRefused = () => {
|
|
532
|
+
try {
|
|
533
|
+
const r = JSON.parse(readFileSync(refusedPath, 'utf-8'));
|
|
534
|
+
return r.key === refusedKey() ? (r.rules ?? {}) : {};
|
|
535
|
+
}
|
|
536
|
+
catch {
|
|
537
|
+
return {};
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
const rememberRefused = (slug, ids) => {
|
|
541
|
+
try {
|
|
542
|
+
const all = readRefused();
|
|
543
|
+
all[slug] = [...new Set([...(all[slug] ?? []), ...ids])];
|
|
544
|
+
mkdirSync(dirname(refusedPath), { recursive: true });
|
|
545
|
+
writeFileSync(refusedPath, JSON.stringify({ key: refusedKey(), rules: all }, null, 2), 'utf-8');
|
|
546
|
+
}
|
|
547
|
+
catch { /* best effort */ }
|
|
548
|
+
};
|
|
549
|
+
/** The file with the setup blocks that use a blank name written again, or null when that did not work. */
|
|
550
|
+
const redoSetup = async (content, filePath, facts, kitApi) => {
|
|
551
|
+
const isBlank = (n) => blankNames(`need('${n}')`).length > 0;
|
|
552
|
+
const blankVars = [...content.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]+)\2/g)]
|
|
553
|
+
.filter((m) => isBlank(m[3])).map((m) => m[1]);
|
|
554
|
+
const uses = (t) => blankVars.some((v) => new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(t));
|
|
555
|
+
const blocks = findSetupBlocks(content).filter((b) => uses(b.text));
|
|
556
|
+
if (!blocks.length)
|
|
557
|
+
return null;
|
|
558
|
+
const top = withoutBlankNames(content.slice(0, Math.min(...blocks.map((b) => b.start))), isBlank).slice(-6000);
|
|
559
|
+
let got = null;
|
|
560
|
+
try {
|
|
561
|
+
got = await rewriteSetupWithAI({ filePath, top, blocks, facts, helpers: kitApi });
|
|
562
|
+
noteAiSuccess();
|
|
563
|
+
}
|
|
564
|
+
catch (e) {
|
|
565
|
+
noteAiFailure(e);
|
|
566
|
+
return null;
|
|
567
|
+
}
|
|
568
|
+
if (!got)
|
|
569
|
+
return null;
|
|
570
|
+
// Each block is still what it was (the same hook or helper), and asks for nothing.
|
|
571
|
+
if (got.some((g, i) => !g.includes(blocks[i].name) || /\bneed\(/.test(g) || uses(g)))
|
|
572
|
+
return null;
|
|
573
|
+
let out = content;
|
|
574
|
+
for (let i = blocks.length - 1; i >= 0; i--)
|
|
575
|
+
out = out.slice(0, blocks[i].start) + got[i] + out.slice(blocks[i].end);
|
|
576
|
+
out = withoutNames(out);
|
|
577
|
+
const why = isSyntaxValid(out, ext) ? genGate(out, filePath) : syntaxReason(out, ext);
|
|
578
|
+
return why ? null : out;
|
|
579
|
+
};
|
|
466
580
|
const suiteComplete = () => {
|
|
581
|
+
const refused = readRefused();
|
|
467
582
|
const plan = readPlan(journey);
|
|
468
583
|
if (!plan?.some((f) => f.rules?.length))
|
|
469
584
|
return true;
|
|
@@ -475,7 +590,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
475
590
|
return true;
|
|
476
591
|
// Rules left without a test make a suite unfinished. A check that flags an
|
|
477
592
|
// existing test does not: kept tests are reported, never rewritten.
|
|
478
|
-
return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.length;
|
|
593
|
+
return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.filter((id) => !(refused[f.slug] ?? []).includes(id)).length;
|
|
479
594
|
});
|
|
480
595
|
};
|
|
481
596
|
// Tests an earlier version appended after the outer describe go back inside
|
|
@@ -486,7 +601,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
486
601
|
continue;
|
|
487
602
|
const p = join(testDir, f);
|
|
488
603
|
const before = readFileSync(p, 'utf-8');
|
|
489
|
-
|
|
604
|
+
// Both mechanical, no AI: stray tests back inside the describe, and
|
|
605
|
+
// kit.insert's id used as the id it is (op[ID_FIELD] → op).
|
|
606
|
+
const after = useInsertIds(tuckStrayTests(before));
|
|
490
607
|
if (after !== before)
|
|
491
608
|
writeFileSync(p, after, 'utf-8');
|
|
492
609
|
}
|
|
@@ -504,6 +621,28 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
504
621
|
catch {
|
|
505
622
|
return false;
|
|
506
623
|
} }));
|
|
624
|
+
// The doc is the same and every rule has its test: new code changes no
|
|
625
|
+
// test. Its facts are read again and the names the tests wait on are
|
|
626
|
+
// filled from them; the test files are not written again (that is
|
|
627
|
+
// --rewrite-waiting, asked for by name).
|
|
628
|
+
if (!force && !opts.rewriteWaiting && waitsOnCode && cache[file] === hash && !hasDeferredStubs(testDir, ext) && suiteComplete()) {
|
|
629
|
+
const usage0 = aiUsage();
|
|
630
|
+
const facts = /^status:\s*not built/im.test(markdown) ? null : await groundJourney(journey, hash, sourceRoot(), markdown, false, opts.refreshCode);
|
|
631
|
+
const blank = [...new Set(testFiles0.flatMap((p) => { try {
|
|
632
|
+
return blankNames(readFileSync(p, 'utf-8'));
|
|
633
|
+
}
|
|
634
|
+
catch {
|
|
635
|
+
return [];
|
|
636
|
+
} }))];
|
|
637
|
+
const res = facts && blank.length ? scanNames(blank, sourceRoot(), facts) : undefined;
|
|
638
|
+
const notes = [
|
|
639
|
+
`doc unchanged: no test written again (code facts read again${res ? `, ${res.filled.length} of ${blank.length} blank name(s) filled from them` : ''})`,
|
|
640
|
+
...(res?.filled.length ? [`filled: ${res.filled.slice(0, 8).map((f) => `${f.name} = ${f.value}`).join(', ')}${res.filled.length > 8 ? ', …' : ''}`] : []),
|
|
641
|
+
...(res?.stillBlank.length ? [`${res.stillBlank.length} name(s) still blank — fill them in testforge/interface.json, or run with --rewrite-waiting to have the tests written again: ${res.stillBlank.slice(0, 8).join(', ')}${res.stillBlank.length > 8 ? ', …' : ''}`] : []),
|
|
642
|
+
];
|
|
643
|
+
results.push({ journeyName: journey.name, testDir, testFiles: [], seedPath, status: 'skipped', seedStatus: 'none', warnings: notes, usage: usageSince(usage0) });
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
507
646
|
if (!force &&
|
|
508
647
|
!waitsOnCode &&
|
|
509
648
|
cache[file] === hash &&
|
|
@@ -574,21 +713,45 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
574
713
|
// What each file really tests once written: the memory and the report come from this, not the plan.
|
|
575
714
|
const written = [];
|
|
576
715
|
// need() names asked so far (the project's and this run's): handed to every later file.
|
|
577
|
-
|
|
716
|
+
// Only names the product has given are handed on for reuse: a blank one in
|
|
717
|
+
// that list reads as "use this name", and the new test waits on it again.
|
|
718
|
+
const filledNow = readInterface();
|
|
719
|
+
const askedNames = new Set(Object.keys(filledNow).filter((n) => isThingName(n) && String(filledNow[n] ?? '').trim()));
|
|
578
720
|
const finish = async (content, pf, filePath, kept = false) => {
|
|
579
721
|
if (!memory || !pf.rules?.length)
|
|
580
722
|
return content;
|
|
581
723
|
const { mem, facts } = await fileMemory(pf);
|
|
724
|
+
// A blank name in a hook or helper makes the whole file wait. Only the
|
|
725
|
+
// setup is written again (one small call), never the tests; once per
|
|
726
|
+
// doc and code if it is refused.
|
|
727
|
+
if (kept && stale.has(filePath) && blankOutsideKind(content) === 'setup' && !(readRefused()[pf.slug] ?? []).includes('#setup')) {
|
|
728
|
+
const redone = await redoSetup(content, filePath, facts, mem?.helpers ?? helpers);
|
|
729
|
+
if (redone) {
|
|
730
|
+
warnings.push(`${pf.slug}: its setup (hooks and helpers) was written again from the code facts; its tests were kept`);
|
|
731
|
+
content = redone;
|
|
732
|
+
}
|
|
733
|
+
else {
|
|
734
|
+
warnings.push(`${pf.slug}: its setup could not be written again — the file keeps waiting on its names`);
|
|
735
|
+
rememberRefused(pf.slug, ['#setup']);
|
|
736
|
+
}
|
|
737
|
+
}
|
|
582
738
|
const done = await completeFile(content, {
|
|
583
739
|
kept, locked: passedTitles(filePath),
|
|
584
740
|
redo: kept && stale.has(filePath) ? (redoAll ? () => true : (b) => usesBlankName(content, b)) : undefined,
|
|
741
|
+
isBlank: (n) => blankNames(`need('${n}')`).length > 0,
|
|
742
|
+
skip: new Set(readRefused()[pf.slug] ?? []),
|
|
743
|
+
fix: fixTests,
|
|
585
744
|
journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
|
|
586
745
|
harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
|
|
587
746
|
check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
|
|
588
747
|
});
|
|
589
748
|
written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
|
|
749
|
+
// What was asked and still has no test is not asked again for this doc and code.
|
|
750
|
+
if (done.refusedRules.length)
|
|
751
|
+
rememberRefused(pf.slug, done.refusedRules);
|
|
590
752
|
for (const n of neededNames(done.content))
|
|
591
|
-
|
|
753
|
+
if (!blankNames(`need('${n}')`).length)
|
|
754
|
+
askedNames.add(n);
|
|
592
755
|
for (const n of done.notes)
|
|
593
756
|
warnings.push(`${pf.slug}: ${n}`);
|
|
594
757
|
return done.content;
|
|
@@ -709,7 +872,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
709
872
|
const stale = new Set();
|
|
710
873
|
// Every test of a journey first written with no code is looked at again once its code is there.
|
|
711
874
|
let redoAll = false;
|
|
712
|
-
if (useAI && !notBuilt && blindBefore && testFiles0.length && (await grounding())) {
|
|
875
|
+
if (useAI && !notBuilt && opts.rewriteWaiting && blindBefore && testFiles0.length && (await grounding())) {
|
|
713
876
|
for (const p of planned.map(pathOf))
|
|
714
877
|
if (existsSync(p))
|
|
715
878
|
stale.add(p);
|
|
@@ -717,7 +880,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
717
880
|
if (redoAll)
|
|
718
881
|
warnings.push(`its code is here now: the tests first written without it are written again from the code (tests that passed are kept)`);
|
|
719
882
|
}
|
|
720
|
-
if (useAI && !notBuilt && !redoAll) {
|
|
883
|
+
if (useAI && !notBuilt && !redoAll && opts.rewriteWaiting) {
|
|
721
884
|
const waiting = planned.map(pathOf).filter((p) => { try {
|
|
722
885
|
return existsSync(p) && blankNames(readFileSync(p, 'utf-8')).length > 0;
|
|
723
886
|
}
|
|
@@ -730,12 +893,35 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
730
893
|
if (waiting.length && (opts.refreshCode || codeMoved) && (await grounding())) {
|
|
731
894
|
for (const p of waiting)
|
|
732
895
|
stale.add(p);
|
|
733
|
-
if (stale.size)
|
|
734
|
-
|
|
896
|
+
if (stale.size) {
|
|
897
|
+
const kinds = [...stale].map((p) => { try {
|
|
898
|
+
return blankOutsideKind(readFileSync(p, 'utf-8'));
|
|
899
|
+
}
|
|
900
|
+
catch {
|
|
901
|
+
return undefined;
|
|
902
|
+
} });
|
|
903
|
+
const setup = kinds.filter((k) => k === 'setup').length;
|
|
904
|
+
const done = readRefused();
|
|
905
|
+
const whole = [...stale].filter((p, i) => kinds[i] === 'file' && !(done[planned.find((pf) => pathOf(pf) === p)?.slug ?? ''] ?? []).includes('#whole')).length;
|
|
906
|
+
warnings.push(`${stale.size} file(s) waiting on names the code now has: only their waiting tests are written again${setup ? `; the setup (hooks, helpers) of ${setup}` : ''}${whole ? `; ${whole} written again whole, once (their names sit in parametrised tables or loops)` : ''}`);
|
|
907
|
+
}
|
|
735
908
|
}
|
|
736
909
|
}
|
|
737
910
|
// A doc without rule ids has no per-test redo: its waiting files are written again whole, as before.
|
|
738
|
-
|
|
911
|
+
// Written again whole: a doc without rule ids (no per-test redo), or a file
|
|
912
|
+
// whose blank names sit where no test or setup block can be split out —
|
|
913
|
+
// once per doc and code, and never a file holding a test that passed.
|
|
914
|
+
const rewriteWhole = (p) => stale.has(p) && (!memory || (() => {
|
|
915
|
+
try {
|
|
916
|
+
const c = readFileSync(p, 'utf-8');
|
|
917
|
+
const slug = planned.find((pf) => pathOf(pf) === p)?.slug ?? '';
|
|
918
|
+
return blankOutsideKind(c) === 'file' && !(readRefused()[slug] ?? []).includes('#whole')
|
|
919
|
+
&& !findTestBlocks(c).some((b) => passedTitles(p).has(b.title));
|
|
920
|
+
}
|
|
921
|
+
catch {
|
|
922
|
+
return false;
|
|
923
|
+
}
|
|
924
|
+
})());
|
|
739
925
|
const keeps = (p) => {
|
|
740
926
|
if (rewriteWhole(p))
|
|
741
927
|
return false;
|
|
@@ -774,7 +960,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
774
960
|
const got = await writeFilesWithAI(files);
|
|
775
961
|
noteAiSuccess();
|
|
776
962
|
for (const [p, c] of got)
|
|
777
|
-
drafts.set(p, c);
|
|
963
|
+
drafts.set(p, fixTests(c));
|
|
778
964
|
}
|
|
779
965
|
catch (e) {
|
|
780
966
|
noteAiFailure(e);
|
|
@@ -821,6 +1007,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
821
1007
|
}
|
|
822
1008
|
let content;
|
|
823
1009
|
let status;
|
|
1010
|
+
if (memory && stale.has(filePath) && existsSync(filePath))
|
|
1011
|
+
rememberRefused(pf.slug, ['#whole']);
|
|
824
1012
|
if (useAI) {
|
|
825
1013
|
let draft = drafts.get(filePath);
|
|
826
1014
|
let draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
|
|
@@ -835,7 +1023,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
835
1023
|
}
|
|
836
1024
|
}
|
|
837
1025
|
const { mem, facts } = await fileMemory(pf);
|
|
838
|
-
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
|
|
1026
|
+
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined, fixTests);
|
|
839
1027
|
if (ai !== null) {
|
|
840
1028
|
content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
|
|
841
1029
|
status = 'ai';
|
|
@@ -876,6 +1064,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
876
1064
|
}
|
|
877
1065
|
return { slug: pf.slug, path: filePath, status };
|
|
878
1066
|
});
|
|
1067
|
+
// The names the new tests still wait on, looked up once against this
|
|
1068
|
+
// journey's code facts: a field the facts show is filled now, not left
|
|
1069
|
+
// for the run. Kept only if the code itself has the value.
|
|
1070
|
+
if (useAI && !notBuilt && memory) {
|
|
1071
|
+
const facts = await grounding();
|
|
1072
|
+
const files = planned.map(pathOf).filter((p) => existsSync(p));
|
|
1073
|
+
const blank = [...new Set(files.flatMap((p) => { try {
|
|
1074
|
+
return blankNames(readFileSync(p, 'utf-8'));
|
|
1075
|
+
}
|
|
1076
|
+
catch {
|
|
1077
|
+
return [];
|
|
1078
|
+
} }))];
|
|
1079
|
+
if (facts && blank.length) {
|
|
1080
|
+
const res = scanNames(blank, sourceRoot(), facts);
|
|
1081
|
+
if (res.filled.length)
|
|
1082
|
+
warnings.push(`filled ${res.filled.length} name(s) from the code facts: ${res.filled.slice(0, 6).map((f) => `${f.name} = ${f.value}`).join(', ')}${res.filled.length > 6 ? ', …' : ''}`);
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
879
1085
|
if (memory && written.length) {
|
|
880
1086
|
const sum = recordWritten(journey, written);
|
|
881
1087
|
console.log(` ${journey.id}: ${sum}`);
|
|
@@ -119,3 +119,90 @@ export function renameInBlocks(content, blocks, pairs) {
|
|
|
119
119
|
});
|
|
120
120
|
return { content: spliceBlocks(content, edits), replaced };
|
|
121
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* Tests written when the writer took kit.insert's answer for a row
|
|
124
|
+
* (`const op = await kit.insert(…); op[ID_FIELD]` with ID_FIELD =
|
|
125
|
+
* need('inserted row id field')). kit.insert returns the id itself, so
|
|
126
|
+
* `op[ID_FIELD]` is `op`: rewritten here, the blank name goes, and no AI is
|
|
127
|
+
* asked to write the file again. Only variables assigned from kit.insert.
|
|
128
|
+
*/
|
|
129
|
+
export function useInsertIds(content) {
|
|
130
|
+
const decl = /^[ \t]*(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]*\binsert\w*\b[^'"`\n]*\bid\b[^'"`\n]*)\2\s*\);?[ \t]*\n/gim;
|
|
131
|
+
let out = content;
|
|
132
|
+
for (const m of [...content.matchAll(decl)]) {
|
|
133
|
+
const [line, name] = m;
|
|
134
|
+
const fromInsert = new Set([...out.matchAll(/\b([A-Za-z_$][\w$]*)\s*=\s*await\s+kit\.insert\(/g)].map((x) => x[1]));
|
|
135
|
+
const esc = name.replace(/\$/g, '\\$');
|
|
136
|
+
out = out.replace(new RegExp(`\\b([A-Za-z_$][\\w$]*)\\[${esc}\\]`, 'g'), (all, v) => (fromInsert.has(v) ? v : all));
|
|
137
|
+
// The constant goes only when nothing uses it any more.
|
|
138
|
+
if (!new RegExp(`\\b${esc}\\b`).test(out.replace(line, '')))
|
|
139
|
+
out = out.replace(line, '');
|
|
140
|
+
}
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
/** Index just past the bracket closing the one at `open` ((, { or [), skipping strings, templates and comments; -1 if none. */
|
|
144
|
+
export function closeBracket(src, open) {
|
|
145
|
+
const pairs = { '(': ')', '{': '}', '[': ']' };
|
|
146
|
+
const stack = [];
|
|
147
|
+
for (let i = open; i < src.length; i++) {
|
|
148
|
+
const c = src[i];
|
|
149
|
+
if (c === '/' && src[i + 1] === '/') {
|
|
150
|
+
const n = src.indexOf('\n', i);
|
|
151
|
+
if (n < 0)
|
|
152
|
+
return -1;
|
|
153
|
+
i = n;
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
if (c === '/' && src[i + 1] === '*') {
|
|
157
|
+
const n = src.indexOf('*/', i + 2);
|
|
158
|
+
if (n < 0)
|
|
159
|
+
return -1;
|
|
160
|
+
i = n + 1;
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
if (c === '"' || c === "'" || c === '`') {
|
|
164
|
+
for (i += 1; i < src.length && src[i] !== c; i++)
|
|
165
|
+
if (src[i] === '\\')
|
|
166
|
+
i += 1;
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
if (pairs[c])
|
|
170
|
+
stack.push(pairs[c]);
|
|
171
|
+
else if (c === ')' || c === '}' || c === ']') {
|
|
172
|
+
if (stack.pop() !== c)
|
|
173
|
+
return -1;
|
|
174
|
+
if (!stack.length)
|
|
175
|
+
return i + 1;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return -1;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* The file's setup outside its tests: hooks (beforeAll/beforeEach/afterAll/
|
|
182
|
+
* afterEach) and helper functions (function x() {…}, const x = async () => {…}).
|
|
183
|
+
*/
|
|
184
|
+
export function findSetupBlocks(content) {
|
|
185
|
+
const tests = findTestBlocks(content);
|
|
186
|
+
const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
|
|
187
|
+
const out = [];
|
|
188
|
+
const START = /^[ \t]*(?:(?:test\.)?(beforeAll|beforeEach|afterAll|afterEach)\s*\(|(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(|(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)\n]*\)|[A-Za-z_$][\w$]*)\s*=>)/gm;
|
|
189
|
+
for (const m of content.matchAll(START)) {
|
|
190
|
+
const at = m.index;
|
|
191
|
+
if (inTest(at) || out.some((b) => at >= b.start && at < b.end))
|
|
192
|
+
continue;
|
|
193
|
+
let end = -1;
|
|
194
|
+
if (m[1])
|
|
195
|
+
end = closeBracket(content, content.indexOf('(', at));
|
|
196
|
+
else {
|
|
197
|
+
const arrow = m[3] ? content.indexOf('=>', at) : -1;
|
|
198
|
+
const brace = content.indexOf('{', m[2] ? closeBracket(content, content.indexOf('(', at)) : arrow);
|
|
199
|
+
end = brace < 0 ? -1 : closeBracket(content, brace);
|
|
200
|
+
}
|
|
201
|
+
if (end < 0)
|
|
202
|
+
continue;
|
|
203
|
+
if (content[end] === ';')
|
|
204
|
+
end += 1;
|
|
205
|
+
out.push({ start: at, end, name: m[1] ?? m[2] ?? m[3], text: content.slice(at, end) });
|
|
206
|
+
}
|
|
207
|
+
return out;
|
|
208
|
+
}
|
|
@@ -17,7 +17,9 @@ export function envNames(text) {
|
|
|
17
17
|
const out = new Set();
|
|
18
18
|
for (const line of text.split('\n')) {
|
|
19
19
|
const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
|
|
20
|
-
|
|
20
|
+
// An unquoted value ends at a comment: `KEY= # hint` (an unfilled template line) is unset.
|
|
21
|
+
const v = /^\s*(['"]).*\1\s*$/.test(m?.[2] ?? '') ? (m?.[2] ?? '').trim() : (m?.[2] ?? '').replace(/(^|\s)#.*$/, '').trim();
|
|
22
|
+
if (m && v.replace(/^(['"])\1$/, '') !== '')
|
|
21
23
|
out.add(m[1]);
|
|
22
24
|
}
|
|
23
25
|
return out;
|
|
@@ -36,10 +36,18 @@ export async function runSetup(srcDir, opts = {}) {
|
|
|
36
36
|
runSetupCall(loadConfig().relatedDirs ?? []);
|
|
37
37
|
if (existsSync('testforge.config.json')) {
|
|
38
38
|
const problems = checkSetupConfig();
|
|
39
|
-
|
|
39
|
+
if (problems.length) {
|
|
40
|
+
// Drafting docs on a config with problems spends AI calls on a project
|
|
41
|
+
// that cannot run yet: stop, and say what to fix.
|
|
42
|
+
console.log(` testforge.config.json written, with problems to fix first:\n ${problems.join('\n ')}\n`);
|
|
43
|
+
console.log('Fix these, then run `testforge setup` again (or `testforge discover`).');
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
console.log(' done — testforge.config.json');
|
|
40
47
|
}
|
|
41
48
|
else {
|
|
42
|
-
console.log('
|
|
49
|
+
console.log(' testforge.config.json was not created — nothing else was done. Run `testforge setup` again.');
|
|
50
|
+
return;
|
|
43
51
|
}
|
|
44
52
|
console.log(existsSync('testforge.md') ? ' done — testforge.md\n' : ' warning: testforge.md was not created\n');
|
|
45
53
|
// Step 3 — discover. This is where the source-of-truth barrier lives:
|
|
@@ -183,6 +183,9 @@ E2. What you touch comes from <code_facts> and <testkit>.
|
|
|
183
183
|
- Look in ROUTES, SHAPES and DATA before need(): a name any of them shows is
|
|
184
184
|
written as it is there. need() is only for a name none of them has (see
|
|
185
185
|
NAMES). Never guess one.
|
|
186
|
+
- A path from need() may hold parameters ({booking}, :id): fill them with
|
|
187
|
+
fillPath(URL, value, …) from the same module as need(), never with
|
|
188
|
+
.replace() on a guessed placeholder name.
|
|
186
189
|
- Never need() what the testkit already gives: kit.insert returns the new
|
|
187
190
|
row's id; a second person of a role (another operator, another customer)
|
|
188
191
|
is a row made with kit.insert; the roles are the ones <testkit> lists.
|
|
@@ -199,9 +202,11 @@ of making it.
|
|
|
199
202
|
- Owner-scoped data ("my orders") follows the token owner, not a query
|
|
200
203
|
parameter. A non-owner check needs a second token; without one, skip and
|
|
201
204
|
name the missing identity.
|
|
202
|
-
- If a precondition really cannot be met
|
|
203
|
-
|
|
204
|
-
|
|
205
|
+
- If a precondition really cannot be met (an account, an env variable or data
|
|
206
|
+
the environment lacks — never an element missing from the page), skip at
|
|
207
|
+
run time inside the test and state what was observed ("TEST_X is not set",
|
|
208
|
+
"GET /x answered 404"), never a guessed cause. That is the only skip: no
|
|
209
|
+
skipped test declarations, describe.skip or test.todo.
|
|
205
210
|
|
|
206
211
|
E4. What you wait for is the event that actually happens. Wait on a
|
|
207
212
|
condition, never for a fixed time.
|
|
@@ -290,8 +295,11 @@ export const WEB = `WEB (Playwright)
|
|
|
290
295
|
title. A stubbed item carries every field the screen reads to show it,
|
|
291
296
|
including the value the list filters on (its status), and the test finds it
|
|
292
297
|
on screen by a field the screen renders as text, never by one it does not
|
|
293
|
-
show.
|
|
294
|
-
|
|
298
|
+
show. Finding it is not the check: assert what the product does with that
|
|
299
|
+
state (the badge, the message, the button it disables or hides), never only
|
|
300
|
+
that the text you stubbed appears. Match the stub by exact host, port, path
|
|
301
|
+
prefix or a request predicate, never by a substring that also matches the
|
|
302
|
+
app's own pages.
|
|
295
303
|
- Waiting: prefer retrying assertions. Register waitForResponse or
|
|
296
304
|
waitForRequest before the action that triggers it. After a login, sign-out
|
|
297
305
|
or navigation, wait for the control you will use, then assert the URL the
|