@activate-core/test-forge 1.0.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +27 -2
- package/dist/core/docs/doc-guidance.js +25 -5
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +17 -1
- package/dist/core/generate/ai-generator.js +208 -283
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +166 -236
- package/dist/core/generate/pending.js +122 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +63 -68
- package/dist/core/generate/suite-planner.js +52 -3
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
|
@@ -27,7 +27,9 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
|
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
29
|
import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
|
|
30
|
-
import {
|
|
30
|
+
import { browserPreflight, missingPackages } from './verify.js';
|
|
31
|
+
import { verifyJourney } from './verify-journey.js';
|
|
32
|
+
import { normalisePending } from './pending.js';
|
|
31
33
|
import { packageRoot } from '../run/run-tests.js';
|
|
32
34
|
import { groundJourney } from '../docs/grounding.js';
|
|
33
35
|
import { generateSeed } from './seed-generator.js';
|
|
@@ -180,7 +182,6 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
180
182
|
// "on" never means "fail the run because nothing is running".
|
|
181
183
|
const verifyOn = opts.verify ?? config.verify ?? true;
|
|
182
184
|
const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
|
|
183
|
-
const verifySkipNoted = new Set();
|
|
184
185
|
const ext = config.testExtension ?? '.test.ts';
|
|
185
186
|
const seedDir = config.seedOutput ?? 'generated-tests/seed';
|
|
186
187
|
const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
|
|
@@ -399,15 +400,22 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
399
400
|
selectorsByArea.set(areaDir, knownSelectors);
|
|
400
401
|
}
|
|
401
402
|
}
|
|
402
|
-
const testGate = (content) => checkTestGates(content, ext, {
|
|
403
|
+
const testGate = (content) => checkTestGates(framework === 'playwright' ? normalisePending(content) : content, ext, {
|
|
403
404
|
framework,
|
|
404
405
|
allowedModules,
|
|
405
406
|
knownSelectors,
|
|
406
407
|
externalModules: config.externalModules,
|
|
407
408
|
});
|
|
409
|
+
// A freshly written file also may not hold placeholders; a repair is held to
|
|
410
|
+
// the repair guard instead (it may not add a skip), so old files stay repairable.
|
|
411
|
+
const genGate = (content) => testGate(content) ??
|
|
412
|
+
checkTestGates(content, ext, { framework, forbidPlaceholders: true });
|
|
408
413
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
409
414
|
// to retry (a resume pass fills those in without touching real files).
|
|
410
|
-
if (!force &&
|
|
415
|
+
if (!force &&
|
|
416
|
+
cache[file] === hash &&
|
|
417
|
+
(!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
|
|
418
|
+
!hasDeferredStubs(testDir, ext)) {
|
|
411
419
|
results.push({
|
|
412
420
|
journeyName: journey.name,
|
|
413
421
|
testDir,
|
|
@@ -461,9 +469,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
461
469
|
// the single biggest cost in a multi-file journey. Best-effort: if it
|
|
462
470
|
// fails, `grounding` stays undefined and each file falls back to reading
|
|
463
471
|
// the source itself, which is slower but still correct.
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
472
|
+
// Only a file that has to be written or repaired needs it, so it is read
|
|
473
|
+
// on first use: re-checking an existing suite costs no grounding at all.
|
|
474
|
+
let groundingP;
|
|
475
|
+
const grounding = () => (groundingP ??= useAI
|
|
476
|
+
? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
|
|
477
|
+
: Promise.resolve(undefined));
|
|
467
478
|
const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
|
|
468
479
|
if (staleRemoved.length) {
|
|
469
480
|
warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
|
|
@@ -490,6 +501,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
490
501
|
// Keep a real, parsing file — but a DEFERRED stub is not real
|
|
491
502
|
// coverage, so fall through and re-attempt it on this resume pass.
|
|
492
503
|
if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
|
|
504
|
+
// A suite written before pending tests carried their reason gets
|
|
505
|
+
// it now, deterministically — no regeneration, no AI.
|
|
506
|
+
const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
|
|
507
|
+
if (normalised !== existing)
|
|
508
|
+
writeFileSync(filePath, normalised, 'utf-8');
|
|
493
509
|
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
494
510
|
}
|
|
495
511
|
}
|
|
@@ -500,9 +516,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
500
516
|
let content;
|
|
501
517
|
let status;
|
|
502
518
|
if (useAI) {
|
|
503
|
-
const
|
|
519
|
+
const facts = await grounding();
|
|
520
|
+
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, genGate);
|
|
504
521
|
if (ai !== null) {
|
|
505
|
-
content = ai;
|
|
522
|
+
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
506
523
|
status = 'ai';
|
|
507
524
|
}
|
|
508
525
|
else {
|
|
@@ -552,7 +569,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
552
569
|
}
|
|
553
570
|
// 2. Seed file (only when using AI — basic mode has no seed concept).
|
|
554
571
|
let seedStatus = 'none';
|
|
555
|
-
if (useAI) {
|
|
572
|
+
if (useAI && !force && existsSync(seedPath)) {
|
|
573
|
+
seedStatus = 'ok'; // kept, like an existing test file
|
|
574
|
+
}
|
|
575
|
+
else if (useAI) {
|
|
556
576
|
const seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
|
|
557
577
|
if (seed !== null) {
|
|
558
578
|
mkdirSync(dirname(seedPath), { recursive: true });
|
|
@@ -593,12 +613,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
593
613
|
warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
|
|
594
614
|
lint.remaining.slice(0, 5).join('; '));
|
|
595
615
|
}
|
|
596
|
-
// Run what we just wrote
|
|
597
|
-
//
|
|
598
|
-
//
|
|
599
|
-
//
|
|
600
|
-
//
|
|
616
|
+
// Run what we just wrote against the real application before calling it
|
|
617
|
+
// done — file by file, then the whole journey together, then an independent
|
|
618
|
+
// review of every claimed defect (see verify-journey.ts). Anything that
|
|
619
|
+
// could not run, or still fails unexplained, leaves the journey UNVERIFIED
|
|
620
|
+
// and uncached, so the next run checks it again.
|
|
601
621
|
let verified;
|
|
622
|
+
let reviewed = [];
|
|
602
623
|
const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
|
|
603
624
|
const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
|
|
604
625
|
if (useAI && verifyOn && targets.length && missing.length) {
|
|
@@ -608,66 +629,38 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
608
629
|
warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
|
|
609
630
|
}
|
|
610
631
|
else if (useAI && verifyOn && targets.length) {
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
break;
|
|
629
|
-
}
|
|
630
|
-
if (outcome.ok) {
|
|
631
|
-
verified = true;
|
|
632
|
-
if (attempt > 0)
|
|
633
|
-
console.log(` verified after ${attempt} repair(s)`);
|
|
634
|
-
break;
|
|
635
|
-
}
|
|
636
|
-
verified = false;
|
|
637
|
-
if (attempt === verifyAttempts) {
|
|
638
|
-
warnings.push(`verify: still failing after ${verifyAttempts} repair attempt(s) — ` +
|
|
639
|
-
'this may be a real defect in the application, or a generation ' +
|
|
640
|
-
'defect this run could not correct. Left uncached for a retry.');
|
|
641
|
-
break;
|
|
642
|
-
}
|
|
643
|
-
console.log(` failed when run — repairing (attempt ${attempt + 1}/${verifyAttempts})`);
|
|
644
|
-
for (const target of targets) {
|
|
645
|
-
let current;
|
|
646
|
-
try {
|
|
647
|
-
current = readFileSync(target, 'utf-8');
|
|
648
|
-
}
|
|
649
|
-
catch {
|
|
650
|
-
continue;
|
|
651
|
-
}
|
|
652
|
-
const repaired = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, outcome.output, harnessPath, transport, grounding), ext, testGate);
|
|
653
|
-
if (repaired === null)
|
|
654
|
-
continue;
|
|
655
|
-
const stamped = repaired.includes(GEN_MARKER)
|
|
656
|
-
? repaired
|
|
657
|
-
: `// ${GEN_MARKER}\n${repaired}`;
|
|
658
|
-
writeFileSync(target, stamped, 'utf-8');
|
|
659
|
-
}
|
|
660
|
-
}
|
|
632
|
+
const result = await verifyJourney({
|
|
633
|
+
config,
|
|
634
|
+
journey,
|
|
635
|
+
targets,
|
|
636
|
+
attempts: verifyAttempts,
|
|
637
|
+
repair: async (target, current, failure) => {
|
|
638
|
+
const facts = await grounding();
|
|
639
|
+
const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
|
|
640
|
+
if (out === null)
|
|
641
|
+
return null;
|
|
642
|
+
const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
|
|
643
|
+
return framework === 'playwright' ? normalisePending(stamped) : stamped;
|
|
644
|
+
},
|
|
645
|
+
});
|
|
646
|
+
verified = result.verified;
|
|
647
|
+
reviewed = result.reviewed;
|
|
648
|
+
warnings.push(...result.warnings);
|
|
661
649
|
}
|
|
662
650
|
const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
|
|
663
651
|
if (!degraded) {
|
|
664
652
|
cache[file] = hash;
|
|
653
|
+
// Remembered apart, so a journey cached before it was ever run is
|
|
654
|
+
// re-checked (not regenerated) once verification is on.
|
|
655
|
+
if (verified)
|
|
656
|
+
cache[`verified:${file}`] = hash;
|
|
665
657
|
}
|
|
666
658
|
else {
|
|
667
659
|
// Actively UN-mark a degraded journey (even under --force, which may
|
|
668
660
|
// be re-processing a stale "done" from an earlier buggy run), so the
|
|
669
661
|
// next `generate` re-does it instead of skipping a thin fallback.
|
|
670
662
|
delete cache[file];
|
|
663
|
+
delete cache[`verified:${file}`];
|
|
671
664
|
}
|
|
672
665
|
saveCache(CACHE_PATH, cache);
|
|
673
666
|
resultRow.set(file, results.length);
|
|
@@ -684,6 +677,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
684
677
|
seedStatus,
|
|
685
678
|
verified,
|
|
686
679
|
warnings: warnings.length ? warnings : undefined,
|
|
680
|
+
reviewed: reviewed.length ? reviewed : undefined,
|
|
687
681
|
});
|
|
688
682
|
}
|
|
689
683
|
for (const [area, b] of blockedAreas) {
|
|
@@ -725,8 +719,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
725
719
|
* Write the per-area runner config.
|
|
726
720
|
* - playwright: global-setup.js + playwright.config.js
|
|
727
721
|
* - jest: helpers/jest.setup.js + jest.config.js
|
|
728
|
-
*
|
|
729
|
-
* never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
|
|
722
|
+
* A config TestForge wrote is rewritten to the current template; a project's
|
|
723
|
+
* own (no marker) is never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
|
|
730
724
|
*/
|
|
731
725
|
export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
|
|
732
726
|
const posix = (from, to) => relative(from, resolve(to)).split(sep).join('/') || '.';
|
|
@@ -735,7 +729,8 @@ export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
|
|
|
735
729
|
writeFileSync(gsPath, globalSetupContent(), 'utf-8');
|
|
736
730
|
const home = configHome(areaDir, 'playwright.config.js');
|
|
737
731
|
const cfg = join(home, 'playwright.config.js');
|
|
738
|
-
|
|
732
|
+
// Ours (it carries the marker) is kept current; a project's own is never touched.
|
|
733
|
+
if (!existsSync(cfg) || readFileSync(cfg, 'utf-8').includes(GEN_MARKER)) {
|
|
739
734
|
writeFileSync(cfg, playwrightConfig(`./${posix(home, areaDir)}`, `./${posix(home, gsPath)}`, area, posix(home, RESULTS_DIR)), 'utf-8');
|
|
740
735
|
}
|
|
741
736
|
}
|
|
@@ -15,11 +15,48 @@
|
|
|
15
15
|
* per-file generator turns each entry into an actual focused test.
|
|
16
16
|
*/
|
|
17
17
|
import { execFileSync } from 'node:child_process';
|
|
18
|
-
import { readFileSync, existsSync } from 'node:fs';
|
|
18
|
+
import { readFileSync, existsSync, readdirSync } from 'node:fs';
|
|
19
|
+
import { join } from 'node:path';
|
|
19
20
|
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
21
|
+
import { PLANS_DIR } from '../paths.js';
|
|
20
22
|
const MAP_PATH = 'testforge.md';
|
|
23
|
+
/**
|
|
24
|
+
* The other journeys in the same area — their names, and the files already
|
|
25
|
+
* planned for them. A planner that sees only its own journey re-tests every
|
|
26
|
+
* shared form, rule and component it touches; on one real suite that was
|
|
27
|
+
* well over half of all tests. With the siblings in view, each behaviour is
|
|
28
|
+
* planned once, in the journey that owns it.
|
|
29
|
+
*/
|
|
30
|
+
export function siblingJourneys(journey, docsDir = 'docs') {
|
|
31
|
+
const lines = [];
|
|
32
|
+
const planned = new Set();
|
|
33
|
+
const dir = join(PLANS_DIR, journey.area);
|
|
34
|
+
for (const f of existsSync(dir) ? readdirSync(dir) : []) {
|
|
35
|
+
const id = f.replace(/\.json$/, '');
|
|
36
|
+
if (!f.endsWith('.json') || id === journey.id)
|
|
37
|
+
continue;
|
|
38
|
+
try {
|
|
39
|
+
const plan = JSON.parse(readFileSync(join(dir, f), 'utf-8'));
|
|
40
|
+
lines.push(`- ${id} (planned):`, ...(plan.files ?? []).map((x) => ` - ${x.title ?? ''}: ${x.covers ?? ''}`));
|
|
41
|
+
planned.add(id);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// unreadable plan — the doc name below still names the journey
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const area = join(docsDir, journey.area);
|
|
48
|
+
for (const f of existsSync(area) ? readdirSync(area) : []) {
|
|
49
|
+
if (!f.endsWith('.md'))
|
|
50
|
+
continue;
|
|
51
|
+
const name = (readFileSync(join(area, f), 'utf-8').match(/^# Journey:\s*(.+)$/m) ?? [])[1];
|
|
52
|
+
if (name && name.trim() !== journey.name && !lines.some((l) => l.includes(name.trim()))) {
|
|
53
|
+
lines.push(`- ${name.trim()} (not planned yet)`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return lines.join('\n');
|
|
57
|
+
}
|
|
21
58
|
/** Build the planning prompt for one journey. */
|
|
22
|
-
function buildPlanPrompt(journey, map) {
|
|
59
|
+
export function buildPlanPrompt(journey, map, siblings = '') {
|
|
23
60
|
return [
|
|
24
61
|
`You are a senior test architect. Plan how to split ONE user journey`,
|
|
25
62
|
`into a set of FOCUSED test files — one concern per file — the way a`,
|
|
@@ -43,7 +80,19 @@ function buildPlanPrompt(journey, map) {
|
|
|
43
80
|
`- "## Controls" and "## Coverage" are indexes, "## Test setup" is what`,
|
|
44
81
|
` every file needs to run, and "## Questions for product (not`,
|
|
45
82
|
` asserted)" is not tested: plan no file for any of them.`,
|
|
83
|
+
`- ONE OWNER PER BEHAVIOUR. Every behaviour is tested in exactly one`,
|
|
84
|
+
` file of exactly one journey. A form, component or rule this journey`,
|
|
85
|
+
` shares with another journey (a login form, a code-entry step, a`,
|
|
86
|
+
` password rule, a sign-out) belongs to the journey whose subject it is;`,
|
|
87
|
+
` this journey tests only what is its OWN: how the customer reaches it`,
|
|
88
|
+
` from here, and the outcome specific to this journey. Say so in`,
|
|
89
|
+
` "covers" ("opening the login form from X and landing on X after`,
|
|
90
|
+
` login — the form's own validation is owned by <journey>").`,
|
|
91
|
+
`- Within this journey, no two files test the same behaviour.`,
|
|
46
92
|
``,
|
|
93
|
+
...(siblings
|
|
94
|
+
? [`OTHER JOURNEYS IN THIS AREA (what they own or will own):`, siblings, ``]
|
|
95
|
+
: []),
|
|
47
96
|
`Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
|
|
48
97
|
` [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,
|
|
49
98
|
` "covers": "<one or two sentences: exactly what this file tests>" }]`,
|
|
@@ -99,7 +148,7 @@ function safeSlug(raw) {
|
|
|
99
148
|
*/
|
|
100
149
|
export function planSuite(journey) {
|
|
101
150
|
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
102
|
-
const prompt = buildPlanPrompt(journey, map);
|
|
151
|
+
const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey));
|
|
103
152
|
const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
|
|
104
153
|
let raw = '';
|
|
105
154
|
try {
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — check a journey's generated tests against the real app before
|
|
3
|
+
* they are handed over.
|
|
4
|
+
*
|
|
5
|
+
* A test that passes every static gate can still target a selector, route or
|
|
6
|
+
* field the app does not have; only running it can tell. Three passes:
|
|
7
|
+
*
|
|
8
|
+
* 1. Each file on its own. A failure, or a skip the test decided by itself,
|
|
9
|
+
* is repaired from the runner's own output. A test the repair judges to
|
|
10
|
+
* be failing on the APPLICATION is left as it is and marked.
|
|
11
|
+
* 2. The whole journey in one run, as a team or CI runs it. Files that pass
|
|
12
|
+
* alone can still break each other (one signs in again as a shared
|
|
13
|
+
* account and ends another's session); those are repaired with that
|
|
14
|
+
* context.
|
|
15
|
+
* 3. An independent review of every marked failure. Only what it confirms
|
|
16
|
+
* is reported; the rest is repaired once more or reported as unverified.
|
|
17
|
+
*
|
|
18
|
+
* A guard refuses any repair that deletes a test, adds a skip or loses an
|
|
19
|
+
* assertion, so no repair can turn a caught defect into a silent one.
|
|
20
|
+
*/
|
|
21
|
+
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
22
|
+
import { basename, join, resolve } from 'node:path';
|
|
23
|
+
import { STATE_DIR } from '../paths.js';
|
|
24
|
+
import { DEFECT_MARK } from './ai-generator.js';
|
|
25
|
+
import { groupClaims, reviewDefects } from './defect-review.js';
|
|
26
|
+
import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
|
|
27
|
+
const describe = (label, issues) => issues.length
|
|
28
|
+
? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
|
|
29
|
+
: [];
|
|
30
|
+
const key = (i) => `${i.file}:${i.line}:${i.title}`;
|
|
31
|
+
const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
|
|
32
|
+
const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
|
|
33
|
+
const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
|
|
34
|
+
const usable = (o) => o.ran && !o.infra;
|
|
35
|
+
export async function verifyJourney(ctx) {
|
|
36
|
+
const { config, journey, targets } = ctx;
|
|
37
|
+
const warnings = [];
|
|
38
|
+
const notRun = [];
|
|
39
|
+
// One guarded repair of one file; true when the file changed.
|
|
40
|
+
const repairOnce = async (target, open, suspect, note = '') => {
|
|
41
|
+
const current = readFileSync(target, 'utf-8');
|
|
42
|
+
const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
|
|
43
|
+
const next = await ctx.repair(target, current, failure);
|
|
44
|
+
if (next === null)
|
|
45
|
+
return false;
|
|
46
|
+
const refused = repairGuard(current, next);
|
|
47
|
+
if (refused) {
|
|
48
|
+
warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
writeFileSync(target, next, 'utf-8');
|
|
52
|
+
return true;
|
|
53
|
+
};
|
|
54
|
+
// 1. Each file on its own: what is wrong inside the file.
|
|
55
|
+
let spent = 0;
|
|
56
|
+
const soloSkips = new Set();
|
|
57
|
+
const lastSolo = new Map();
|
|
58
|
+
for (const target of targets) {
|
|
59
|
+
const run = async () => {
|
|
60
|
+
const started = Date.now();
|
|
61
|
+
const o = await verifyFile(config, journey.area, target);
|
|
62
|
+
spent += Date.now() - started;
|
|
63
|
+
return o;
|
|
64
|
+
};
|
|
65
|
+
let outcome = await run();
|
|
66
|
+
for (let attempt = 0;; attempt += 1) {
|
|
67
|
+
if (!usable(outcome)) {
|
|
68
|
+
notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
const open = unclaimed(outcome.failures);
|
|
72
|
+
// A skip the test decided by itself is a suspect once: its own setup may
|
|
73
|
+
// have looked in the wrong place. A skip that only names a missing env
|
|
74
|
+
// var or opt-in flag is the environment's to fix.
|
|
75
|
+
const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
|
|
76
|
+
if ((!open.length && !suspect.length) || attempt === ctx.attempts)
|
|
77
|
+
break;
|
|
78
|
+
console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
79
|
+
if (!(await repairOnce(target, open, suspect)))
|
|
80
|
+
break;
|
|
81
|
+
outcome = await run();
|
|
82
|
+
}
|
|
83
|
+
lastSolo.set(target, outcome);
|
|
84
|
+
for (const k of outcome.skips)
|
|
85
|
+
soloSkips.add(key(k));
|
|
86
|
+
}
|
|
87
|
+
// 2. The whole journey in ONE run. Whatever fails only here is repaired with
|
|
88
|
+
// that context, once.
|
|
89
|
+
let together;
|
|
90
|
+
if (!notRun.length && targets.length > 1) {
|
|
91
|
+
const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
|
|
92
|
+
const order = targets.map((t) => basename(t)).sort().join(', ');
|
|
93
|
+
const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
|
|
94
|
+
`each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
|
|
95
|
+
`(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
|
|
96
|
+
`backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
|
|
97
|
+
`data it sets up itself.`;
|
|
98
|
+
// The runner reports absolute paths; targets may be relative.
|
|
99
|
+
const byAbs = new Map(targets.map((t) => [resolve(t), t]));
|
|
100
|
+
console.log(` running all ${targets.length} files together…`);
|
|
101
|
+
together = await verifyRun(config, journey.area, targets, budget);
|
|
102
|
+
if (usable(together)) {
|
|
103
|
+
const open = unclaimed(together.failures);
|
|
104
|
+
const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
|
|
105
|
+
const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
|
|
106
|
+
if (files.length) {
|
|
107
|
+
console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
|
|
108
|
+
for (const f of files) {
|
|
109
|
+
await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
|
|
110
|
+
}
|
|
111
|
+
together = await verifyRun(config, journey.area, targets, budget);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (!usable(together))
|
|
115
|
+
notRun.push(`all files together: ${together.infra ?? together.reason}`);
|
|
116
|
+
}
|
|
117
|
+
// 3. Judge what is left on the LAST run that covered each file. Nothing is
|
|
118
|
+
// reported as a defect on the repair's word alone.
|
|
119
|
+
const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
|
|
120
|
+
const failures = finals.flatMap((o) => o.failures);
|
|
121
|
+
const unresolved = unclaimed(failures).map(label);
|
|
122
|
+
let reviewed = [];
|
|
123
|
+
const groups = groupClaims(failures, claimOf);
|
|
124
|
+
if (groups.length) {
|
|
125
|
+
console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
|
|
126
|
+
const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes));
|
|
127
|
+
reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
|
|
128
|
+
// A rejected claim is the test's fault until shown otherwise: one guarded
|
|
129
|
+
// repair with the reviewer's reason; whatever still fails is unverified and
|
|
130
|
+
// is not re-claimed as a defect.
|
|
131
|
+
for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
|
|
132
|
+
const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
|
|
133
|
+
`Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
|
|
134
|
+
for (const f of new Set(r.tests.map((t) => t.file))) {
|
|
135
|
+
const target = targets.find((t) => resolve(t) === f) ?? f;
|
|
136
|
+
const mine = r.tests.filter((t) => t.file === f);
|
|
137
|
+
if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
|
|
138
|
+
const again = await verifyFile(config, journey.area, target);
|
|
139
|
+
// Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
|
|
140
|
+
for (const t of again.failures) {
|
|
141
|
+
if (mine.some((m) => m.title === t.title) || !claimOf(t))
|
|
142
|
+
unresolved.push(`${label(t)} (after review)`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
for (const t of mine)
|
|
147
|
+
unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
|
|
152
|
+
writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
|
|
153
|
+
}
|
|
154
|
+
if (notRun.length) {
|
|
155
|
+
warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
|
|
156
|
+
}
|
|
157
|
+
if (unresolved.length) {
|
|
158
|
+
warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
|
|
159
|
+
`do not report them; ${unresolved.slice(0, 5).join('; ')}`);
|
|
160
|
+
}
|
|
161
|
+
return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
|
|
162
|
+
}
|