@activate-core/test-forge 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +29 -11
- package/dist/core/docs/doc-guidance.js +10 -0
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +12 -1
- package/dist/core/generate/ai-generator.js +181 -267
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +181 -226
- package/dist/core/generate/pending.js +118 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +70 -73
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
|
@@ -27,13 +27,15 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
|
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
29
|
import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
|
|
30
|
-
import {
|
|
30
|
+
import { browserPreflight, missingPackages } from './verify.js';
|
|
31
|
+
import { verifyJourney } from './verify-journey.js';
|
|
32
|
+
import { normalisePending } from './pending.js';
|
|
31
33
|
import { packageRoot } from '../run/run-tests.js';
|
|
32
34
|
import { groundJourney } from '../docs/grounding.js';
|
|
33
35
|
import { generateSeed } from './seed-generator.js';
|
|
34
36
|
import { isSyntaxValid, checkTestGates } from '../gates/validate.js';
|
|
35
37
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
36
|
-
import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
38
|
+
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
37
39
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
38
40
|
import { findMarkdownFiles } from '../find-files.js';
|
|
39
41
|
import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
|
|
@@ -160,7 +162,7 @@ export async function runHarness(only) {
|
|
|
160
162
|
continue;
|
|
161
163
|
}
|
|
162
164
|
mkdirSync(dirname(info.path), { recursive: true });
|
|
163
|
-
writeFileSync(info.path, kit, 'utf-8');
|
|
165
|
+
writeFileSync(info.path, stampHarness(kit), 'utf-8');
|
|
164
166
|
writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
|
|
165
167
|
writeHarnessConfigs(framework, areaDir, info.dir, area);
|
|
166
168
|
written.push(info.path);
|
|
@@ -180,7 +182,6 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
180
182
|
// "on" never means "fail the run because nothing is running".
|
|
181
183
|
const verifyOn = opts.verify ?? config.verify ?? true;
|
|
182
184
|
const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
|
|
183
|
-
const verifySkipNoted = new Set();
|
|
184
185
|
const ext = config.testExtension ?? '.test.ts';
|
|
185
186
|
const seedDir = config.seedOutput ?? 'generated-tests/seed';
|
|
186
187
|
const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
|
|
@@ -246,8 +247,14 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
246
247
|
// extra AI call (and no risk of that call eating the token limit).
|
|
247
248
|
// Pass --force to rebuild a stale testkit (e.g. after upgrading).
|
|
248
249
|
if (!force && existsSync(info.path)) {
|
|
249
|
-
|
|
250
|
-
|
|
250
|
+
if (harnessIsCurrent(readFileSync(info.path, 'utf-8'))) {
|
|
251
|
+
harnessByArea.set(area, info.path);
|
|
252
|
+
return info.path;
|
|
253
|
+
}
|
|
254
|
+
// Built by an older TestForge whose harness rules have since changed:
|
|
255
|
+
// rebuild it once here, so the fixes reach this project with no extra
|
|
256
|
+
// command. Test files are untouched.
|
|
257
|
+
console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
|
|
251
258
|
}
|
|
252
259
|
const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
|
|
253
260
|
if (kit === null) {
|
|
@@ -255,7 +262,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
255
262
|
return undefined;
|
|
256
263
|
}
|
|
257
264
|
mkdirSync(dirname(info.path), { recursive: true });
|
|
258
|
-
writeFileSync(info.path, kit, 'utf-8');
|
|
265
|
+
writeFileSync(info.path, stampHarness(kit), 'utf-8');
|
|
259
266
|
writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
|
|
260
267
|
// Config + globalSetup (playwright) / config + setup (jest) so the specs run
|
|
261
268
|
// AND get their tokens minted once, plus the runner config scaffolding.
|
|
@@ -393,7 +400,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
393
400
|
selectorsByArea.set(areaDir, knownSelectors);
|
|
394
401
|
}
|
|
395
402
|
}
|
|
396
|
-
const testGate = (content) => checkTestGates(content, ext, {
|
|
403
|
+
const testGate = (content) => checkTestGates(framework === 'playwright' ? normalisePending(content) : content, ext, {
|
|
397
404
|
framework,
|
|
398
405
|
allowedModules,
|
|
399
406
|
knownSelectors,
|
|
@@ -401,7 +408,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
401
408
|
});
|
|
402
409
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
403
410
|
// to retry (a resume pass fills those in without touching real files).
|
|
404
|
-
if (!force &&
|
|
411
|
+
if (!force &&
|
|
412
|
+
cache[file] === hash &&
|
|
413
|
+
(!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
|
|
414
|
+
!hasDeferredStubs(testDir, ext)) {
|
|
405
415
|
results.push({
|
|
406
416
|
journeyName: journey.name,
|
|
407
417
|
testDir,
|
|
@@ -455,9 +465,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
455
465
|
// the single biggest cost in a multi-file journey. Best-effort: if it
|
|
456
466
|
// fails, `grounding` stays undefined and each file falls back to reading
|
|
457
467
|
// the source itself, which is slower but still correct.
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
468
|
+
// Only a file that has to be written or repaired needs it, so it is read
|
|
469
|
+
// on first use: re-checking an existing suite costs no grounding at all.
|
|
470
|
+
let groundingP;
|
|
471
|
+
const grounding = () => (groundingP ??= useAI
|
|
472
|
+
? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
|
|
473
|
+
: Promise.resolve(undefined));
|
|
461
474
|
const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
|
|
462
475
|
if (staleRemoved.length) {
|
|
463
476
|
warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
|
|
@@ -484,6 +497,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
484
497
|
// Keep a real, parsing file — but a DEFERRED stub is not real
|
|
485
498
|
// coverage, so fall through and re-attempt it on this resume pass.
|
|
486
499
|
if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
|
|
500
|
+
// A suite written before pending tests carried their reason gets
|
|
501
|
+
// it now, deterministically — no regeneration, no AI.
|
|
502
|
+
const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
|
|
503
|
+
if (normalised !== existing)
|
|
504
|
+
writeFileSync(filePath, normalised, 'utf-8');
|
|
487
505
|
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
488
506
|
}
|
|
489
507
|
}
|
|
@@ -494,9 +512,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
494
512
|
let content;
|
|
495
513
|
let status;
|
|
496
514
|
if (useAI) {
|
|
497
|
-
const
|
|
515
|
+
const facts = await grounding();
|
|
516
|
+
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, testGate);
|
|
498
517
|
if (ai !== null) {
|
|
499
|
-
content = ai;
|
|
518
|
+
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
500
519
|
status = 'ai';
|
|
501
520
|
}
|
|
502
521
|
else {
|
|
@@ -546,7 +565,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
546
565
|
}
|
|
547
566
|
// 2. Seed file (only when using AI — basic mode has no seed concept).
|
|
548
567
|
let seedStatus = 'none';
|
|
549
|
-
if (useAI) {
|
|
568
|
+
if (useAI && !force && existsSync(seedPath)) {
|
|
569
|
+
seedStatus = 'ok'; // kept, like an existing test file
|
|
570
|
+
}
|
|
571
|
+
else if (useAI) {
|
|
550
572
|
const seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
|
|
551
573
|
if (seed !== null) {
|
|
552
574
|
mkdirSync(dirname(seedPath), { recursive: true });
|
|
@@ -587,12 +609,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
587
609
|
warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
|
|
588
610
|
lint.remaining.slice(0, 5).join('; '));
|
|
589
611
|
}
|
|
590
|
-
// Run what we just wrote
|
|
591
|
-
//
|
|
592
|
-
//
|
|
593
|
-
//
|
|
594
|
-
//
|
|
612
|
+
// Run what we just wrote against the real application before calling it
|
|
613
|
+
// done — file by file, then the whole journey together, then an independent
|
|
614
|
+
// review of every claimed defect (see verify-journey.ts). Anything that
|
|
615
|
+
// could not run, or still fails unexplained, leaves the journey UNVERIFIED
|
|
616
|
+
// and uncached, so the next run checks it again.
|
|
595
617
|
let verified;
|
|
618
|
+
let reviewed = [];
|
|
596
619
|
const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
|
|
597
620
|
const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
|
|
598
621
|
if (useAI && verifyOn && targets.length && missing.length) {
|
|
@@ -602,66 +625,38 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
602
625
|
warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
|
|
603
626
|
}
|
|
604
627
|
else if (useAI && verifyOn && targets.length) {
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
break;
|
|
623
|
-
}
|
|
624
|
-
if (outcome.ok) {
|
|
625
|
-
verified = true;
|
|
626
|
-
if (attempt > 0)
|
|
627
|
-
console.log(` verified after ${attempt} repair(s)`);
|
|
628
|
-
break;
|
|
629
|
-
}
|
|
630
|
-
verified = false;
|
|
631
|
-
if (attempt === verifyAttempts) {
|
|
632
|
-
warnings.push(`verify: still failing after ${verifyAttempts} repair attempt(s) — ` +
|
|
633
|
-
'this may be a real defect in the application, or a generation ' +
|
|
634
|
-
'defect this run could not correct. Left uncached for a retry.');
|
|
635
|
-
break;
|
|
636
|
-
}
|
|
637
|
-
console.log(` failed when run — repairing (attempt ${attempt + 1}/${verifyAttempts})`);
|
|
638
|
-
for (const target of targets) {
|
|
639
|
-
let current;
|
|
640
|
-
try {
|
|
641
|
-
current = readFileSync(target, 'utf-8');
|
|
642
|
-
}
|
|
643
|
-
catch {
|
|
644
|
-
continue;
|
|
645
|
-
}
|
|
646
|
-
const repaired = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, outcome.output, harnessPath, transport, grounding), ext, testGate);
|
|
647
|
-
if (repaired === null)
|
|
648
|
-
continue;
|
|
649
|
-
const stamped = repaired.includes(GEN_MARKER)
|
|
650
|
-
? repaired
|
|
651
|
-
: `// ${GEN_MARKER}\n${repaired}`;
|
|
652
|
-
writeFileSync(target, stamped, 'utf-8');
|
|
653
|
-
}
|
|
654
|
-
}
|
|
628
|
+
const result = await verifyJourney({
|
|
629
|
+
config,
|
|
630
|
+
journey,
|
|
631
|
+
targets,
|
|
632
|
+
attempts: verifyAttempts,
|
|
633
|
+
repair: async (target, current, failure) => {
|
|
634
|
+
const facts = await grounding();
|
|
635
|
+
const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
|
|
636
|
+
if (out === null)
|
|
637
|
+
return null;
|
|
638
|
+
const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
|
|
639
|
+
return framework === 'playwright' ? normalisePending(stamped) : stamped;
|
|
640
|
+
},
|
|
641
|
+
});
|
|
642
|
+
verified = result.verified;
|
|
643
|
+
reviewed = result.reviewed;
|
|
644
|
+
warnings.push(...result.warnings);
|
|
655
645
|
}
|
|
656
646
|
const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
|
|
657
647
|
if (!degraded) {
|
|
658
648
|
cache[file] = hash;
|
|
649
|
+
// Remembered apart, so a journey cached before it was ever run is
|
|
650
|
+
// re-checked (not regenerated) once verification is on.
|
|
651
|
+
if (verified)
|
|
652
|
+
cache[`verified:${file}`] = hash;
|
|
659
653
|
}
|
|
660
654
|
else {
|
|
661
655
|
// Actively UN-mark a degraded journey (even under --force, which may
|
|
662
656
|
// be re-processing a stale "done" from an earlier buggy run), so the
|
|
663
657
|
// next `generate` re-does it instead of skipping a thin fallback.
|
|
664
658
|
delete cache[file];
|
|
659
|
+
delete cache[`verified:${file}`];
|
|
665
660
|
}
|
|
666
661
|
saveCache(CACHE_PATH, cache);
|
|
667
662
|
resultRow.set(file, results.length);
|
|
@@ -678,6 +673,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
678
673
|
seedStatus,
|
|
679
674
|
verified,
|
|
680
675
|
warnings: warnings.length ? warnings : undefined,
|
|
676
|
+
reviewed: reviewed.length ? reviewed : undefined,
|
|
681
677
|
});
|
|
682
678
|
}
|
|
683
679
|
for (const [area, b] of blockedAreas) {
|
|
@@ -719,8 +715,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
719
715
|
* Write the per-area runner config.
|
|
720
716
|
* - playwright: global-setup.js + playwright.config.js
|
|
721
717
|
* - jest: helpers/jest.setup.js + jest.config.js
|
|
722
|
-
*
|
|
723
|
-
* never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
|
|
718
|
+
* A config TestForge wrote is rewritten to the current template; a project's
|
|
719
|
+
* own (no marker) is never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
|
|
724
720
|
*/
|
|
725
721
|
export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
|
|
726
722
|
const posix = (from, to) => relative(from, resolve(to)).split(sep).join('/') || '.';
|
|
@@ -729,7 +725,8 @@ export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
|
|
|
729
725
|
writeFileSync(gsPath, globalSetupContent(), 'utf-8');
|
|
730
726
|
const home = configHome(areaDir, 'playwright.config.js');
|
|
731
727
|
const cfg = join(home, 'playwright.config.js');
|
|
732
|
-
|
|
728
|
+
// Ours (it carries the marker) is kept current; a project's own is never touched.
|
|
729
|
+
if (!existsSync(cfg) || readFileSync(cfg, 'utf-8').includes(GEN_MARKER)) {
|
|
733
730
|
writeFileSync(cfg, playwrightConfig(`./${posix(home, areaDir)}`, `./${posix(home, gsPath)}`, area, posix(home, RESULTS_DIR)), 'utf-8');
|
|
734
731
|
}
|
|
735
732
|
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — check a journey's generated tests against the real app before
|
|
3
|
+
* they are handed over.
|
|
4
|
+
*
|
|
5
|
+
* A test that passes every static gate can still target a selector, route or
|
|
6
|
+
* field the app does not have; only running it can tell. Three passes:
|
|
7
|
+
*
|
|
8
|
+
* 1. Each file on its own. A failure, or a skip the test decided by itself,
|
|
9
|
+
* is repaired from the runner's own output. A test the repair judges to
|
|
10
|
+
* be failing on the APPLICATION is left as it is and marked.
|
|
11
|
+
* 2. The whole journey in one run, as a team or CI runs it. Files that pass
|
|
12
|
+
* alone can still break each other (one signs in again as a shared
|
|
13
|
+
* account and ends another's session); those are repaired with that
|
|
14
|
+
* context.
|
|
15
|
+
* 3. An independent review of every marked failure. Only what it confirms
|
|
16
|
+
* is reported; the rest is repaired once more or reported as unverified.
|
|
17
|
+
*
|
|
18
|
+
* A guard refuses any repair that deletes a test, adds a skip or loses an
|
|
19
|
+
* assertion, so no repair can turn a caught defect into a silent one.
|
|
20
|
+
*/
|
|
21
|
+
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
22
|
+
import { basename, join, resolve } from 'node:path';
|
|
23
|
+
import { STATE_DIR } from '../paths.js';
|
|
24
|
+
import { DEFECT_MARK } from './ai-generator.js';
|
|
25
|
+
import { groupClaims, reviewDefects } from './defect-review.js';
|
|
26
|
+
import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
|
|
27
|
+
const describe = (label, issues) => issues.length
|
|
28
|
+
? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
|
|
29
|
+
: [];
|
|
30
|
+
const key = (i) => `${i.file}:${i.line}:${i.title}`;
|
|
31
|
+
const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
|
|
32
|
+
const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
|
|
33
|
+
const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
|
|
34
|
+
const usable = (o) => o.ran && !o.infra;
|
|
35
|
+
export async function verifyJourney(ctx) {
|
|
36
|
+
const { config, journey, targets } = ctx;
|
|
37
|
+
const warnings = [];
|
|
38
|
+
const notRun = [];
|
|
39
|
+
// One guarded repair of one file; true when the file changed.
|
|
40
|
+
const repairOnce = async (target, open, suspect, note = '') => {
|
|
41
|
+
const current = readFileSync(target, 'utf-8');
|
|
42
|
+
const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
|
|
43
|
+
const next = await ctx.repair(target, current, failure);
|
|
44
|
+
if (next === null)
|
|
45
|
+
return false;
|
|
46
|
+
const refused = repairGuard(current, next);
|
|
47
|
+
if (refused) {
|
|
48
|
+
warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
writeFileSync(target, next, 'utf-8');
|
|
52
|
+
return true;
|
|
53
|
+
};
|
|
54
|
+
// 1. Each file on its own: what is wrong inside the file.
|
|
55
|
+
let spent = 0;
|
|
56
|
+
const soloSkips = new Set();
|
|
57
|
+
const lastSolo = new Map();
|
|
58
|
+
for (const target of targets) {
|
|
59
|
+
const run = async () => {
|
|
60
|
+
const started = Date.now();
|
|
61
|
+
const o = await verifyFile(config, journey.area, target);
|
|
62
|
+
spent += Date.now() - started;
|
|
63
|
+
return o;
|
|
64
|
+
};
|
|
65
|
+
let outcome = await run();
|
|
66
|
+
for (let attempt = 0;; attempt += 1) {
|
|
67
|
+
if (!usable(outcome)) {
|
|
68
|
+
notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
const open = unclaimed(outcome.failures);
|
|
72
|
+
// A skip the test decided by itself is a suspect once: its own setup may
|
|
73
|
+
// have looked in the wrong place. A skip that only names a missing env
|
|
74
|
+
// var or opt-in flag is the environment's to fix.
|
|
75
|
+
const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
|
|
76
|
+
if ((!open.length && !suspect.length) || attempt === ctx.attempts)
|
|
77
|
+
break;
|
|
78
|
+
console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
79
|
+
if (!(await repairOnce(target, open, suspect)))
|
|
80
|
+
break;
|
|
81
|
+
outcome = await run();
|
|
82
|
+
}
|
|
83
|
+
lastSolo.set(target, outcome);
|
|
84
|
+
for (const k of outcome.skips)
|
|
85
|
+
soloSkips.add(key(k));
|
|
86
|
+
}
|
|
87
|
+
// 2. The whole journey in ONE run. Whatever fails only here is repaired with
|
|
88
|
+
// that context, once.
|
|
89
|
+
let together;
|
|
90
|
+
if (!notRun.length && targets.length > 1) {
|
|
91
|
+
const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
|
|
92
|
+
const order = targets.map((t) => basename(t)).sort().join(', ');
|
|
93
|
+
const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
|
|
94
|
+
`each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
|
|
95
|
+
`(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
|
|
96
|
+
`backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
|
|
97
|
+
`data it sets up itself.`;
|
|
98
|
+
// The runner reports absolute paths; targets may be relative.
|
|
99
|
+
const byAbs = new Map(targets.map((t) => [resolve(t), t]));
|
|
100
|
+
console.log(` running all ${targets.length} files together…`);
|
|
101
|
+
together = await verifyRun(config, journey.area, targets, budget);
|
|
102
|
+
if (usable(together)) {
|
|
103
|
+
const open = unclaimed(together.failures);
|
|
104
|
+
const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
|
|
105
|
+
const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
|
|
106
|
+
if (files.length) {
|
|
107
|
+
console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
|
|
108
|
+
for (const f of files) {
|
|
109
|
+
await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
|
|
110
|
+
}
|
|
111
|
+
together = await verifyRun(config, journey.area, targets, budget);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (!usable(together))
|
|
115
|
+
notRun.push(`all files together: ${together.infra ?? together.reason}`);
|
|
116
|
+
}
|
|
117
|
+
// 3. Judge what is left on the LAST run that covered each file. Nothing is
|
|
118
|
+
// reported as a defect on the repair's word alone.
|
|
119
|
+
const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
|
|
120
|
+
const failures = finals.flatMap((o) => o.failures);
|
|
121
|
+
const unresolved = unclaimed(failures).map(label);
|
|
122
|
+
let reviewed = [];
|
|
123
|
+
const groups = groupClaims(failures, claimOf);
|
|
124
|
+
if (groups.length) {
|
|
125
|
+
console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
|
|
126
|
+
const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes));
|
|
127
|
+
reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
|
|
128
|
+
// A rejected claim is the test's fault until shown otherwise: one guarded
|
|
129
|
+
// repair with the reviewer's reason; whatever still fails is unverified and
|
|
130
|
+
// is not re-claimed as a defect.
|
|
131
|
+
for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
|
|
132
|
+
const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
|
|
133
|
+
`Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
|
|
134
|
+
for (const f of new Set(r.tests.map((t) => t.file))) {
|
|
135
|
+
const target = targets.find((t) => resolve(t) === f) ?? f;
|
|
136
|
+
const mine = r.tests.filter((t) => t.file === f);
|
|
137
|
+
if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
|
|
138
|
+
const again = await verifyFile(config, journey.area, target);
|
|
139
|
+
// Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
|
|
140
|
+
for (const t of again.failures) {
|
|
141
|
+
if (mine.some((m) => m.title === t.title) || !claimOf(t))
|
|
142
|
+
unresolved.push(`${label(t)} (after review)`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
for (const t of mine)
|
|
147
|
+
unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
|
|
152
|
+
writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
|
|
153
|
+
}
|
|
154
|
+
if (notRun.length) {
|
|
155
|
+
warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
|
|
156
|
+
}
|
|
157
|
+
if (unresolved.length) {
|
|
158
|
+
warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
|
|
159
|
+
`do not report them; ${unresolved.slice(0, 5).join('; ')}`);
|
|
160
|
+
}
|
|
161
|
+
return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
|
|
162
|
+
}
|