@activate-core/test-forge 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +24 -3
- package/dist/cli/index.js +2 -1
- package/dist/core/docs/excerpts.js +186 -6
- package/dist/core/docs/grounding.js +82 -16
- package/dist/core/docs/reference-text.js +2 -0
- package/dist/core/gates/coverage.js +9 -0
- package/dist/core/gates/validate.js +50 -23
- package/dist/core/generate/ai-generator.js +9 -3
- package/dist/core/generate/ai-usage.js +5 -2
- package/dist/core/generate/complete-file.js +70 -10
- package/dist/core/generate/defect-review.js +2 -1
- package/dist/core/generate/harness.js +34 -0
- package/dist/core/generate/interface-file.js +12 -0
- package/dist/core/generate/journey-memory.js +7 -1
- package/dist/core/generate/passed.js +24 -0
- package/dist/core/generate/run-generate.js +201 -20
- package/dist/core/generate/test-blocks.js +34 -10
- package/dist/core/project/config.js +5 -0
- package/dist/core/project/env-diff.js +3 -1
- package/dist/core/project/setup.js +10 -2
- package/dist/core/prompts/blocks.js +37 -8
- package/dist/core/run/name-scan.js +178 -17
- package/dist/core/run/report.js +9 -4
- package/dist/core/run/run-tests.js +58 -4
- package/package.json +5 -3
- package/reporters/_shared.cjs +56 -3
- package/reporters/interface.cjs +40 -3
- package/reporters/jest-reporter.cjs +7 -3
- package/reporters/playwright-reporter.cjs +6 -2
- package/reporters/run-data.cjs +6 -1
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* so tests/seeds land where the project expects them. Defaults to
|
|
17
17
|
* generated-tests/ when not configured.
|
|
18
18
|
*/
|
|
19
|
-
import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
|
|
19
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, } from 'node:fs';
|
|
20
20
|
import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
|
|
21
21
|
import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored, journeyDir } from '../paths.js';
|
|
22
22
|
import { lintGenerated } from '../gates/lint.js';
|
|
@@ -27,13 +27,49 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
|
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
29
|
import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
|
-
import { completeFile
|
|
30
|
+
import { completeFile } from './complete-file.js';
|
|
31
|
+
import { tuckStrayTests } from './test-blocks.js';
|
|
32
|
+
import { passedTitles } from './passed.js';
|
|
33
|
+
import { scanNames } from '../run/name-scan.js';
|
|
31
34
|
import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
|
|
32
|
-
|
|
35
|
+
/** Names the file asks for that testforge/interface.json still has blank. */
|
|
36
|
+
/** Does this test use a name the product has not given yet (directly, or through the constant it was put in)? */
|
|
37
|
+
function usesBlankName(content, block) {
|
|
38
|
+
const text = content.slice(block.start, block.end);
|
|
39
|
+
return blankNames(content).some((n) => {
|
|
40
|
+
if (text.includes(`'${n}'`) || text.includes(`"${n}"`) || text.includes(`\`${n}\``))
|
|
41
|
+
return true;
|
|
42
|
+
const esc = n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
43
|
+
const v = new RegExp(`\\b(?:const|let|var)\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*need\\(\\s*(['"\`])${esc}\\2`).exec(content)?.[1];
|
|
44
|
+
return !!v && new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(text);
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
/** The names a testkit exports (module.exports = { … } or export …). */
|
|
48
|
+
export function exportNames(src) {
|
|
49
|
+
const names = new Set();
|
|
50
|
+
const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(src);
|
|
51
|
+
if (cjs)
|
|
52
|
+
for (const n of cjs[1].split(',')) {
|
|
53
|
+
const m = /^\s*(\w+)/.exec(n);
|
|
54
|
+
if (m)
|
|
55
|
+
names.add(m[1]);
|
|
56
|
+
}
|
|
57
|
+
for (const m of src.matchAll(/^export\s+(?:async\s+)?(?:function\*?|const|let|class)\s+(\w+)/gm))
|
|
58
|
+
names.add(m[1]);
|
|
59
|
+
return [...names];
|
|
60
|
+
}
|
|
61
|
+
function blankNames(content) {
|
|
62
|
+
const iface = readInterface();
|
|
63
|
+
return [...content.matchAll(/\bneed\(\s*(['"`])([^'"`]+)\1\s*\)/g)].map((m) => m[2]).filter((n) => {
|
|
64
|
+
const v = iface[n];
|
|
65
|
+
return typeof v === 'string' ? !v.trim() : v === undefined || v === null;
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
import { harnessSteps, ruleCoverage } from '../gates/test-quality.js';
|
|
33
69
|
import { normalisePending } from './pending.js';
|
|
34
70
|
import { packageRoot } from '../run/run-tests.js';
|
|
35
71
|
import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
|
|
36
|
-
import { groundJourney } from '../docs/grounding.js';
|
|
72
|
+
import { factsCodeMoved, journeyCodeHash, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
|
|
37
73
|
import { generateSeed, seedProblem } from './seed-generator.js';
|
|
38
74
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
39
75
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
@@ -43,7 +79,7 @@ import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
|
43
79
|
import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
|
|
44
80
|
import { updateJourney } from './update-journey.js';
|
|
45
81
|
import { sourceRoot, sourceText } from '../docs/excerpts.js';
|
|
46
|
-
import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
82
|
+
import { isSyntaxValid, checkTestGates, trimExtraTests, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
47
83
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
48
84
|
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
49
85
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
@@ -92,7 +128,9 @@ export async function pool(items, limit, fn) {
|
|
|
92
128
|
/** Async version of generateValidated (for concurrent per-file gen). */
|
|
93
129
|
async function generateValidatedAsync(generate, ext, gate,
|
|
94
130
|
/** Why an earlier draft of the same file (from a batch) was refused, with that draft. */
|
|
95
|
-
firstRefusal
|
|
131
|
+
firstRefusal,
|
|
132
|
+
/** Mechanical fixes applied before the gate: what code can fix costs no retry. */
|
|
133
|
+
fix = (c) => c) {
|
|
96
134
|
let content = '';
|
|
97
135
|
let refusal = firstRefusal;
|
|
98
136
|
for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
|
|
@@ -100,7 +138,7 @@ firstRefusal) {
|
|
|
100
138
|
return null; // retrying a dead CLI only burns time
|
|
101
139
|
try {
|
|
102
140
|
// A second try sees what it wrote and why that was refused; asked blind, it made the same file again.
|
|
103
|
-
content = await generate(refusal);
|
|
141
|
+
content = fix(await generate(refusal));
|
|
104
142
|
}
|
|
105
143
|
catch (e) {
|
|
106
144
|
noteAiFailure(e);
|
|
@@ -273,7 +311,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
273
311
|
// command. Test files are untouched.
|
|
274
312
|
console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
|
|
275
313
|
}
|
|
314
|
+
const old = existsSync(info.path) ? readFileSync(info.path, 'utf-8') : '';
|
|
276
315
|
const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
|
|
316
|
+
// The tests already written call the old testkit: a rebuild that fails, or
|
|
317
|
+
// drops something they use, is not taken — the old one stays.
|
|
318
|
+
const lost = old && kit !== null ? exportNames(old).filter((n) => !exportNames(kit).includes(n)) : [];
|
|
319
|
+
if (old && (kit === null || lost.length)) {
|
|
320
|
+
console.log(` kept the existing ${area} test harness: the rebuild ${kit === null ? 'failed' : `dropped ${lost.join(', ')}`}`);
|
|
321
|
+
exportFind(info.path);
|
|
322
|
+
harnessByArea.set(area, info.path);
|
|
323
|
+
return info.path;
|
|
324
|
+
}
|
|
277
325
|
if (kit === null) {
|
|
278
326
|
harnessByArea.set(area, null);
|
|
279
327
|
return undefined;
|
|
@@ -432,6 +480,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
432
480
|
const appTexts = () => framework === 'playwright'
|
|
433
481
|
? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
434
482
|
: undefined;
|
|
483
|
+
const fixTests = (content) => trimExtraTests(content, config.maxTestsPerRule ?? 2);
|
|
435
484
|
const genGate = (content, filePath) => testGate(content) ??
|
|
436
485
|
checkTestGates(content, ext, {
|
|
437
486
|
framework, forbidPlaceholders: true, filePath,
|
|
@@ -442,7 +491,21 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
442
491
|
// to retry (a resume pass fills those in without touching real files).
|
|
443
492
|
// A suite whose tests leave rules out or cannot fail is not done either:
|
|
444
493
|
// it goes on, and only those tests are written again.
|
|
494
|
+
// Rules asked for and refused before, with this doc and this code: not asked again until one moves.
|
|
495
|
+
const refusedPath = join(journeyDir(journey), 'refused.json');
|
|
496
|
+
let refusedKeyMemo;
|
|
497
|
+
const refusedKey = () => (refusedKeyMemo ??= `${hash}:${journeyCodeHash(journey, sourceRoot())}`);
|
|
498
|
+
const readRefused = () => {
|
|
499
|
+
try {
|
|
500
|
+
const r = JSON.parse(readFileSync(refusedPath, 'utf-8'));
|
|
501
|
+
return r.key === refusedKey() ? (r.rules ?? {}) : {};
|
|
502
|
+
}
|
|
503
|
+
catch {
|
|
504
|
+
return {};
|
|
505
|
+
}
|
|
506
|
+
};
|
|
445
507
|
const suiteComplete = () => {
|
|
508
|
+
const refused = readRefused();
|
|
446
509
|
const plan = readPlan(journey);
|
|
447
510
|
if (!plan?.some((f) => f.rules?.length))
|
|
448
511
|
return true;
|
|
@@ -452,10 +515,39 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
452
515
|
const p = join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`);
|
|
453
516
|
if (!f.rules?.length || !existsSync(p))
|
|
454
517
|
return true;
|
|
455
|
-
|
|
518
|
+
// Rules left without a test make a suite unfinished. A check that flags an
|
|
519
|
+
// existing test does not: kept tests are reported, never rewritten.
|
|
520
|
+
return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.filter((id) => !(refused[f.slug] ?? []).includes(id)).length;
|
|
456
521
|
});
|
|
457
522
|
};
|
|
523
|
+
// Tests an earlier version appended after the outer describe go back inside
|
|
524
|
+
// it, so they get the file's setup. No AI, also for an unchanged journey.
|
|
525
|
+
if (existsSync(testDir)) {
|
|
526
|
+
for (const f of readdirSync(testDir)) {
|
|
527
|
+
if (!f.endsWith(ext))
|
|
528
|
+
continue;
|
|
529
|
+
const p = join(testDir, f);
|
|
530
|
+
const before = readFileSync(p, 'utf-8');
|
|
531
|
+
const after = tuckStrayTests(before);
|
|
532
|
+
if (after !== before)
|
|
533
|
+
writeFileSync(p, after, 'utf-8');
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
// Code that arrived after the tests were written: files still asking for
|
|
537
|
+
// names the product did not have are written again from the new code.
|
|
538
|
+
const codeMoved = useAI && (opts.refreshCode || factsCodeMoved(journey, sourceRoot()));
|
|
539
|
+
const testFiles0 = existsSync(testDir) ? readdirSync(testDir).filter((f) => f.endsWith(ext)).map((f) => join(testDir, f)) : [];
|
|
540
|
+
// Tests written while the feature had no code: their "how" was a guess, so
|
|
541
|
+
// the code arriving is a reason to look again even with every name filled.
|
|
542
|
+
const blindBefore = writtenWithoutCode(journey);
|
|
543
|
+
const waitsOnCode = codeMoved && (blindBefore || testFiles0.some((p) => { try {
|
|
544
|
+
return blankNames(readFileSync(p, 'utf-8')).length > 0;
|
|
545
|
+
}
|
|
546
|
+
catch {
|
|
547
|
+
return false;
|
|
548
|
+
} }));
|
|
458
549
|
if (!force &&
|
|
550
|
+
!waitsOnCode &&
|
|
459
551
|
cache[file] === hash &&
|
|
460
552
|
!hasDeferredStubs(testDir, ext) &&
|
|
461
553
|
suiteComplete()) {
|
|
@@ -474,6 +566,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
474
566
|
// A feature written up before it is built: its tests come from the doc
|
|
475
567
|
// alone. There is no code to read for facts and no app to run them on.
|
|
476
568
|
const notBuilt = /^status:\s*not built/im.test(markdown);
|
|
569
|
+
if (notBuilt && useAI)
|
|
570
|
+
markWrittenWithoutCode(journey, sourceRoot());
|
|
477
571
|
// Not live yet (not built, or behind a flag): every file is stamped, and
|
|
478
572
|
// `testforge run` holds it back until the doc's Status line goes.
|
|
479
573
|
const pending = pendingReason(markdown);
|
|
@@ -522,19 +616,38 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
522
616
|
// What each file really tests once written: the memory and the report come from this, not the plan.
|
|
523
617
|
const written = [];
|
|
524
618
|
// need() names asked so far (the project's and this run's): handed to every later file.
|
|
525
|
-
|
|
526
|
-
|
|
619
|
+
// Only names the product has given are handed on for reuse: a blank one in
|
|
620
|
+
// that list reads as "use this name", and the new test waits on it again.
|
|
621
|
+
const filledNow = readInterface();
|
|
622
|
+
const askedNames = new Set(Object.keys(filledNow).filter((n) => isThingName(n) && String(filledNow[n] ?? '').trim()));
|
|
623
|
+
const finish = async (content, pf, filePath, kept = false) => {
|
|
527
624
|
if (!memory || !pf.rules?.length)
|
|
528
625
|
return content;
|
|
529
626
|
const { mem, facts } = await fileMemory(pf);
|
|
530
627
|
const done = await completeFile(content, {
|
|
628
|
+
kept, locked: passedTitles(filePath),
|
|
629
|
+
redo: kept && stale.has(filePath) ? (redoAll ? () => true : (b) => usesBlankName(content, b)) : undefined,
|
|
630
|
+
isBlank: (n) => blankNames(`need('${n}')`).length > 0,
|
|
631
|
+
skip: new Set(readRefused()[pf.slug] ?? []),
|
|
632
|
+
fix: fixTests,
|
|
531
633
|
journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
|
|
532
634
|
harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
|
|
533
635
|
check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
|
|
534
636
|
});
|
|
535
637
|
written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
|
|
638
|
+
// What was asked and still has no test is not asked again for this doc and code.
|
|
639
|
+
if (done.refusedRules.length) {
|
|
640
|
+
try {
|
|
641
|
+
const all = readRefused();
|
|
642
|
+
all[pf.slug] = [...new Set([...(all[pf.slug] ?? []), ...done.refusedRules])];
|
|
643
|
+
mkdirSync(dirname(refusedPath), { recursive: true });
|
|
644
|
+
writeFileSync(refusedPath, JSON.stringify({ key: refusedKey(), rules: all }, null, 2), 'utf-8');
|
|
645
|
+
}
|
|
646
|
+
catch { /* best effort */ }
|
|
647
|
+
}
|
|
536
648
|
for (const n of neededNames(done.content))
|
|
537
|
-
|
|
649
|
+
if (!blankNames(`need('${n}')`).length)
|
|
650
|
+
askedNames.add(n);
|
|
538
651
|
for (const n of done.notes)
|
|
539
652
|
warnings.push(`${pf.slug}: ${n}`);
|
|
540
653
|
return done.content;
|
|
@@ -635,7 +748,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
635
748
|
// on first use: re-checking an existing suite costs no grounding at all.
|
|
636
749
|
let groundingP;
|
|
637
750
|
const grounding = () => (groundingP ??= useAI && !notBuilt
|
|
638
|
-
? groundJourney(journey, hash, sourceRoot(), markdown).then((g) => g ?? undefined)
|
|
751
|
+
? groundJourney(journey, hash, sourceRoot(), markdown, false, opts.refreshCode).then((g) => g ?? undefined)
|
|
639
752
|
: Promise.resolve(undefined));
|
|
640
753
|
const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
|
|
641
754
|
if (staleRemoved.length) {
|
|
@@ -651,7 +764,40 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
651
764
|
// once. A drafted file that fails its checks is written again on its own,
|
|
652
765
|
// told why; a file the batch left out is written on its own.
|
|
653
766
|
const pathOf = (pf) => join(testDir, `${flat ? `${journey.id}-${pf.slug}` : pf.slug}${ext}`);
|
|
767
|
+
// A file written from older code facts that still waits on names: written again.
|
|
768
|
+
const stale = new Set();
|
|
769
|
+
// Every test of a journey first written with no code is looked at again once its code is there.
|
|
770
|
+
let redoAll = false;
|
|
771
|
+
if (useAI && !notBuilt && blindBefore && testFiles0.length && (await grounding())) {
|
|
772
|
+
for (const p of planned.map(pathOf))
|
|
773
|
+
if (existsSync(p))
|
|
774
|
+
stale.add(p);
|
|
775
|
+
redoAll = stale.size > 0;
|
|
776
|
+
if (redoAll)
|
|
777
|
+
warnings.push(`its code is here now: the tests first written without it are written again from the code (tests that passed are kept)`);
|
|
778
|
+
}
|
|
779
|
+
if (useAI && !notBuilt && !redoAll) {
|
|
780
|
+
const waiting = planned.map(pathOf).filter((p) => { try {
|
|
781
|
+
return existsSync(p) && blankNames(readFileSync(p, 'utf-8')).length > 0;
|
|
782
|
+
}
|
|
783
|
+
catch {
|
|
784
|
+
return false;
|
|
785
|
+
} });
|
|
786
|
+
// Only new code (or --refresh-code) rewrites a waiting file: facts read
|
|
787
|
+
// again from the same code may word things differently, and that alone
|
|
788
|
+
// must never cost a rewrite.
|
|
789
|
+
if (waiting.length && (opts.refreshCode || codeMoved) && (await grounding())) {
|
|
790
|
+
for (const p of waiting)
|
|
791
|
+
stale.add(p);
|
|
792
|
+
if (stale.size)
|
|
793
|
+
warnings.push(`${stale.size} file(s) waiting on names the code now has: only their waiting tests are written again`);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
// A doc without rule ids has no per-test redo: its waiting files are written again whole, as before.
|
|
797
|
+
const rewriteWhole = (p) => stale.has(p) && !memory;
|
|
654
798
|
const keeps = (p) => {
|
|
799
|
+
if (rewriteWhole(p))
|
|
800
|
+
return false;
|
|
655
801
|
if ((force && !redone.has(i)) || !existsSync(p))
|
|
656
802
|
return false;
|
|
657
803
|
try {
|
|
@@ -687,7 +833,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
687
833
|
const got = await writeFilesWithAI(files);
|
|
688
834
|
noteAiSuccess();
|
|
689
835
|
for (const [p, c] of got)
|
|
690
|
-
drafts.set(p, c);
|
|
836
|
+
drafts.set(p, fixTests(c));
|
|
691
837
|
}
|
|
692
838
|
catch (e) {
|
|
693
839
|
noteAiFailure(e);
|
|
@@ -711,7 +857,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
711
857
|
// midway resumes from exactly the files it hadn't written yet.
|
|
712
858
|
// (--force ignores this and regenerates.)
|
|
713
859
|
// A redo after stubs fills in only the stubs: --force already rewrote the rest once this run.
|
|
714
|
-
if ((!force || redone.has(i)) && existsSync(filePath)) {
|
|
860
|
+
if ((!force || redone.has(i)) && existsSync(filePath) && !rewriteWhole(filePath)) {
|
|
715
861
|
try {
|
|
716
862
|
const existing = readFileSync(filePath, 'utf-8');
|
|
717
863
|
// Keep a real, parsing file — but a DEFERRED stub is not real
|
|
@@ -722,7 +868,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
722
868
|
const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
|
|
723
869
|
// A kept file that leaves rules out or holds tests that cannot fail:
|
|
724
870
|
// only those tests are written again.
|
|
725
|
-
const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath) : normalised, pending);
|
|
871
|
+
const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath, true) : normalised, pending);
|
|
726
872
|
if (finished !== existing)
|
|
727
873
|
writeFileSync(filePath, finished, 'utf-8');
|
|
728
874
|
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
@@ -735,15 +881,32 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
735
881
|
let content;
|
|
736
882
|
let status;
|
|
737
883
|
if (useAI) {
|
|
738
|
-
|
|
739
|
-
|
|
884
|
+
let draft = drafts.get(filePath);
|
|
885
|
+
let draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
|
|
886
|
+
// A draft that parses but holds a few refused tests: only those tests are
|
|
887
|
+
// written again (the way a kept file is finished), not the whole file.
|
|
888
|
+
if (draft !== undefined && draftRefusal && isSyntaxValid(draft, ext)) {
|
|
889
|
+
const finished = await finish(framework === 'playwright' ? normalisePending(draft) : draft, pf, filePath);
|
|
890
|
+
const still = isSyntaxValid(finished, ext) ? genGate(finished, filePath) : syntaxReason(finished, ext);
|
|
891
|
+
if (!still) {
|
|
892
|
+
draft = finished;
|
|
893
|
+
draftRefusal = undefined;
|
|
894
|
+
}
|
|
895
|
+
}
|
|
740
896
|
const { mem, facts } = await fileMemory(pf);
|
|
741
|
-
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
|
|
897
|
+
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined, fixTests);
|
|
742
898
|
if (ai !== null) {
|
|
743
899
|
content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
|
|
744
900
|
status = 'ai';
|
|
745
901
|
}
|
|
746
902
|
else {
|
|
903
|
+
// The AI could not write it: a real file already there stays as it is,
|
|
904
|
+
// rather than being replaced by a placeholder.
|
|
905
|
+
const existing = existsSync(filePath) ? readFileSync(filePath, 'utf-8') : '';
|
|
906
|
+
if (existing && isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
|
|
907
|
+
warnings.push(`${pf.slug}: could not be written again — the earlier file is kept`);
|
|
908
|
+
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
909
|
+
}
|
|
747
910
|
const built = buildTest(journey, false, ext, filePath, framework);
|
|
748
911
|
content = built.content;
|
|
749
912
|
status = 'basic';
|
|
@@ -772,6 +935,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
772
935
|
}
|
|
773
936
|
return { slug: pf.slug, path: filePath, status };
|
|
774
937
|
});
|
|
938
|
+
// The names the new tests still wait on, looked up once against this
|
|
939
|
+
// journey's code facts: a field the facts show is filled now, not left
|
|
940
|
+
// for the run. Kept only if the code itself has the value.
|
|
941
|
+
if (useAI && !notBuilt && memory) {
|
|
942
|
+
const facts = await grounding();
|
|
943
|
+
const files = planned.map(pathOf).filter((p) => existsSync(p));
|
|
944
|
+
const blank = [...new Set(files.flatMap((p) => { try {
|
|
945
|
+
return blankNames(readFileSync(p, 'utf-8'));
|
|
946
|
+
}
|
|
947
|
+
catch {
|
|
948
|
+
return [];
|
|
949
|
+
} }))];
|
|
950
|
+
if (facts && blank.length) {
|
|
951
|
+
const res = scanNames(blank, sourceRoot(), facts);
|
|
952
|
+
if (res.filled.length)
|
|
953
|
+
warnings.push(`filled ${res.filled.length} name(s) from the code facts: ${res.filled.slice(0, 6).map((f) => `${f.name} = ${f.value}`).join(', ')}${res.filled.length > 6 ? ', …' : ''}`);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
775
956
|
if (memory && written.length) {
|
|
776
957
|
const sum = recordWritten(journey, written);
|
|
777
958
|
console.log(` ${journey.id}: ${sum}`);
|
|
@@ -809,8 +990,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
809
990
|
if (useAI && dataAccess(config, packageRoot(areaDir)) === 'api') {
|
|
810
991
|
writeRequiredData(seedDir, journey.id, journey.name, markdown);
|
|
811
992
|
}
|
|
812
|
-
else if (useAI && !force && existsSync(seedPath)) {
|
|
813
|
-
seedStatus = 'ok'; // kept, like an existing test file
|
|
993
|
+
else if (useAI && !force && existsSync(seedPath) && !redoAll) {
|
|
994
|
+
seedStatus = 'ok'; // kept, like an existing test file — unless it was written before the code existed
|
|
814
995
|
}
|
|
815
996
|
else if (useAI && pending) {
|
|
816
997
|
// Not live yet: there is nothing to seed for, and its tests make their own data.
|
|
@@ -66,18 +66,42 @@ export function spliceBlocks(content, edits) {
|
|
|
66
66
|
}
|
|
67
67
|
/** Add tests inside the file's outermost describe, before it closes; at the end when there is none. */
|
|
68
68
|
export function appendTests(content, tests) {
|
|
69
|
-
const d =
|
|
70
|
-
if (d)
|
|
71
|
-
|
|
72
|
-
if (close !== -1) {
|
|
73
|
-
// The describe's body closes with "}" just before ")".
|
|
74
|
-
const brace = content.lastIndexOf('}', close);
|
|
75
|
-
if (brace > d.index)
|
|
76
|
-
return `${content.slice(0, brace).trimEnd()}\n\n${indent(tests.trim(), ' ')}\n${content.slice(brace)}`;
|
|
77
|
-
}
|
|
78
|
-
}
|
|
69
|
+
const d = outerDescribe(content);
|
|
70
|
+
if (d)
|
|
71
|
+
return `${content.slice(0, d.brace).trimEnd()}\n\n${indent(tests.trim(), ' ')}\n${content.slice(d.brace)}`;
|
|
79
72
|
return `${content.trimEnd()}\n\n${tests.trim()}\n`;
|
|
80
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* The file's outer describe (Playwright's test.describe or Jest's describe):
|
|
76
|
+
* where it starts, the "}" closing its body and the ")" closing the call.
|
|
77
|
+
*/
|
|
78
|
+
function outerDescribe(content) {
|
|
79
|
+
const d = /^(?:test\.)?describe(?:\.\w+)?\s*\(/m.exec(content) ?? /\b(?:test\.)?describe(?:\.\w+)?\s*\(/.exec(content);
|
|
80
|
+
if (!d)
|
|
81
|
+
return null;
|
|
82
|
+
const close = matchParen(content, content.indexOf('(', d.index));
|
|
83
|
+
if (close === -1)
|
|
84
|
+
return null;
|
|
85
|
+
// The describe's body closes with "}" just before ")".
|
|
86
|
+
const brace = content.lastIndexOf('}', close);
|
|
87
|
+
return brace > d.index ? { start: d.index, brace, close } : null;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Tests that ended up after the outer describe (earlier versions appended
|
|
91
|
+
* them there) are moved inside it, so they get the file's setup and cleanup.
|
|
92
|
+
* No AI; a file with nothing outside is returned as it is.
|
|
93
|
+
*/
|
|
94
|
+
export function tuckStrayTests(content) {
|
|
95
|
+
const d = outerDescribe(content);
|
|
96
|
+
if (!d)
|
|
97
|
+
return content;
|
|
98
|
+
const stray = findTestBlocks(content).filter((b) => b.start > d.close);
|
|
99
|
+
if (!stray.length)
|
|
100
|
+
return content;
|
|
101
|
+
const moved = stray.map((b) => content.slice(b.start, b.end).trim()).join('\n\n');
|
|
102
|
+
const rest = spliceBlocks(content, stray.map((block) => ({ block, text: '' }))).replace(/\n{3,}/g, '\n\n');
|
|
103
|
+
return appendTests(rest, moved);
|
|
104
|
+
}
|
|
81
105
|
const indent = (t, pad) => t.split('\n').map((l) => (l ? pad + l : l)).join('\n');
|
|
82
106
|
/** Literal replacement of each old text with the new one, only inside the given blocks. */
|
|
83
107
|
export function renameInBlocks(content, blocks, pairs) {
|
|
@@ -282,6 +282,11 @@ export function loadEnvFile(path) {
|
|
|
282
282
|
(value.startsWith("'") && value.endsWith("'"))) {
|
|
283
283
|
value = value.slice(1, -1);
|
|
284
284
|
}
|
|
285
|
+
else {
|
|
286
|
+
// An unquoted value ends at a comment: `KEY= # hint` (the template's
|
|
287
|
+
// unfilled line) is empty, not the hint.
|
|
288
|
+
value = value.replace(/(^|\s)#.*$/, '').trim();
|
|
289
|
+
}
|
|
285
290
|
out[key] = value;
|
|
286
291
|
}
|
|
287
292
|
return out;
|
|
@@ -17,7 +17,9 @@ export function envNames(text) {
|
|
|
17
17
|
const out = new Set();
|
|
18
18
|
for (const line of text.split('\n')) {
|
|
19
19
|
const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
|
|
20
|
-
|
|
20
|
+
// An unquoted value ends at a comment: `KEY= # hint` (an unfilled template line) is unset.
|
|
21
|
+
const v = /^\s*(['"]).*\1\s*$/.test(m?.[2] ?? '') ? (m?.[2] ?? '').trim() : (m?.[2] ?? '').replace(/(^|\s)#.*$/, '').trim();
|
|
22
|
+
if (m && v.replace(/^(['"])\1$/, '') !== '')
|
|
21
23
|
out.add(m[1]);
|
|
22
24
|
}
|
|
23
25
|
return out;
|
|
@@ -36,10 +36,18 @@ export async function runSetup(srcDir, opts = {}) {
|
|
|
36
36
|
runSetupCall(loadConfig().relatedDirs ?? []);
|
|
37
37
|
if (existsSync('testforge.config.json')) {
|
|
38
38
|
const problems = checkSetupConfig();
|
|
39
|
-
|
|
39
|
+
if (problems.length) {
|
|
40
|
+
// Drafting docs on a config with problems spends AI calls on a project
|
|
41
|
+
// that cannot run yet: stop, and say what to fix.
|
|
42
|
+
console.log(` testforge.config.json written, with problems to fix first:\n ${problems.join('\n ')}\n`);
|
|
43
|
+
console.log('Fix these, then run `testforge setup` again (or `testforge discover`).');
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
console.log(' done — testforge.config.json');
|
|
40
47
|
}
|
|
41
48
|
else {
|
|
42
|
-
console.log('
|
|
49
|
+
console.log(' testforge.config.json was not created — nothing else was done. Run `testforge setup` again.');
|
|
50
|
+
return;
|
|
43
51
|
}
|
|
44
52
|
console.log(existsSync('testforge.md') ? ' done — testforge.md\n' : ' warning: testforge.md was not created\n');
|
|
45
53
|
// Step 3 — discover. This is where the source-of-truth barrier lives:
|
|
@@ -110,7 +110,10 @@ FORMAT
|
|
|
110
110
|
"## Test setup", which lists what the environment must provide (accounts and
|
|
111
111
|
records to seed, flags to turn on, external services to stub with their
|
|
112
112
|
endpoint or SDK, browser dialogs), written as instructions to the test
|
|
113
|
-
generator
|
|
113
|
+
generator. Name every request the tests send by its method and path
|
|
114
|
+
(POST /api/v2/booking-order), even a booking or payment the journey only
|
|
115
|
+
passes through: only a route named here has its handler and validation
|
|
116
|
+
rules read for the tests;
|
|
114
117
|
"## Questions for product (not asserted)", which holds only what neither the
|
|
115
118
|
reference nor the code settles. Keep it short.
|
|
116
119
|
- Do not write a Coverage table. It is generated from the [C<ids>] on each
|
|
@@ -174,8 +177,18 @@ E2. What you touch comes from <code_facts> and <testkit>.
|
|
|
174
177
|
any version prefix, and use the same prefix throughout the file.
|
|
175
178
|
- A response field comes from the serializer for that endpoint in
|
|
176
179
|
<code_facts>, not from the types of another endpoint.
|
|
177
|
-
- A
|
|
178
|
-
|
|
180
|
+
- A request is sent with the fields its handler validates, as SHAPES in
|
|
181
|
+
<code_facts> lists them, nested as written (address.latitude is
|
|
182
|
+
{ address: { latitude } }). A column, table or status value comes from DATA.
|
|
183
|
+
- Look in ROUTES, SHAPES and DATA before need(): a name any of them shows is
|
|
184
|
+
written as it is there. need() is only for a name none of them has (see
|
|
185
|
+
NAMES). Never guess one.
|
|
186
|
+
- A path from need() may hold parameters ({booking}, :id): fill them with
|
|
187
|
+
fillPath(URL, value, …) from the same module as need(), never with
|
|
188
|
+
.replace() on a guessed placeholder name.
|
|
189
|
+
- Never need() what the testkit already gives: kit.insert returns the new
|
|
190
|
+
row's id; a second person of a role (another operator, another customer)
|
|
191
|
+
is a row made with kit.insert; the roles are the ones <testkit> lists.
|
|
179
192
|
- Call testkit helpers exactly as <testkit> lists them, and never write your
|
|
180
193
|
own copy.
|
|
181
194
|
|
|
@@ -189,9 +202,11 @@ of making it.
|
|
|
189
202
|
- Owner-scoped data ("my orders") follows the token owner, not a query
|
|
190
203
|
parameter. A non-owner check needs a second token; without one, skip and
|
|
191
204
|
name the missing identity.
|
|
192
|
-
- If a precondition really cannot be met
|
|
193
|
-
|
|
194
|
-
|
|
205
|
+
- If a precondition really cannot be met (an account, an env variable or data
|
|
206
|
+
the environment lacks — never an element missing from the page), skip at
|
|
207
|
+
run time inside the test and state what was observed ("TEST_X is not set",
|
|
208
|
+
"GET /x answered 404"), never a guessed cause. That is the only skip: no
|
|
209
|
+
skipped test declarations, describe.skip or test.todo.
|
|
195
210
|
|
|
196
211
|
E4. What you wait for is the event that actually happens. Wait on a
|
|
197
212
|
condition, never for a fixed time.
|
|
@@ -280,8 +295,11 @@ export const WEB = `WEB (Playwright)
|
|
|
280
295
|
title. A stubbed item carries every field the screen reads to show it,
|
|
281
296
|
including the value the list filters on (its status), and the test finds it
|
|
282
297
|
on screen by a field the screen renders as text, never by one it does not
|
|
283
|
-
show.
|
|
284
|
-
|
|
298
|
+
show. Finding it is not the check: assert what the product does with that
|
|
299
|
+
state (the badge, the message, the button it disables or hides), never only
|
|
300
|
+
that the text you stubbed appears. Match the stub by exact host, port, path
|
|
301
|
+
prefix or a request predicate, never by a substring that also matches the
|
|
302
|
+
app's own pages.
|
|
285
303
|
- Waiting: prefer retrying assertions. Register waitForResponse or
|
|
286
304
|
waitForRequest before the action that triggers it. After a login, sign-out
|
|
287
305
|
or navigation, wait for the control you will use, then assert the URL the
|
|
@@ -306,6 +324,17 @@ export const API = `API (Jest)
|
|
|
306
324
|
token, the wrong role, and a different owner.
|
|
307
325
|
- Check stored state only through testkit resolvers or helpers. Never open
|
|
308
326
|
your own database connection.
|
|
327
|
+
- Build the data a scenario describes (operators and their locations,
|
|
328
|
+
services, hours and earlier jobs; a setting switched on or off) with
|
|
329
|
+
kit.insert(table, row) and kit.update(table, match, values), using the
|
|
330
|
+
table and column names from <code_facts>. Never ask for a URL or command to
|
|
331
|
+
seed, rank or switch something: data is written, and the outcome is read
|
|
332
|
+
from the product's own endpoints (the operator a booking gets, a status).
|
|
333
|
+
- A value the product works out from other records (a standing, an average
|
|
334
|
+
per day) is not a column: set it with the testkit's _tf_ column from its
|
|
335
|
+
DERIVED export. Where the product picks one record among many (the nearest,
|
|
336
|
+
the best), build the scenario at kit.AREA so nothing already in the
|
|
337
|
+
database takes part.
|
|
309
338
|
- Keep tests hermetic. Every external service listed under EXTERNALS in
|
|
310
339
|
<code_facts> is stood in for:
|
|
311
340
|
- when the server runs inside the test process, with jest.mock on the exact
|