@activate-core/test-forge 1.0.4 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +27 -2
- package/dist/core/docs/doc-guidance.js +10 -0
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +12 -1
- package/dist/core/generate/ai-generator.js +181 -271
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +166 -236
- package/dist/core/generate/pending.js +118 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +59 -68
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
|
@@ -27,7 +27,9 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
|
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
29
|
import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
|
|
30
|
-
import {
|
|
30
|
+
import { browserPreflight, missingPackages } from './verify.js';
|
|
31
|
+
import { verifyJourney } from './verify-journey.js';
|
|
32
|
+
import { normalisePending } from './pending.js';
|
|
31
33
|
import { packageRoot } from '../run/run-tests.js';
|
|
32
34
|
import { groundJourney } from '../docs/grounding.js';
|
|
33
35
|
import { generateSeed } from './seed-generator.js';
|
|
@@ -180,7 +182,6 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
180
182
|
// "on" never means "fail the run because nothing is running".
|
|
181
183
|
const verifyOn = opts.verify ?? config.verify ?? true;
|
|
182
184
|
const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
|
|
183
|
-
const verifySkipNoted = new Set();
|
|
184
185
|
const ext = config.testExtension ?? '.test.ts';
|
|
185
186
|
const seedDir = config.seedOutput ?? 'generated-tests/seed';
|
|
186
187
|
const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
|
|
@@ -399,7 +400,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
399
400
|
selectorsByArea.set(areaDir, knownSelectors);
|
|
400
401
|
}
|
|
401
402
|
}
|
|
402
|
-
const testGate = (content) => checkTestGates(content, ext, {
|
|
403
|
+
const testGate = (content) => checkTestGates(framework === 'playwright' ? normalisePending(content) : content, ext, {
|
|
403
404
|
framework,
|
|
404
405
|
allowedModules,
|
|
405
406
|
knownSelectors,
|
|
@@ -407,7 +408,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
407
408
|
});
|
|
408
409
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
409
410
|
// to retry (a resume pass fills those in without touching real files).
|
|
410
|
-
if (!force &&
|
|
411
|
+
if (!force &&
|
|
412
|
+
cache[file] === hash &&
|
|
413
|
+
(!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
|
|
414
|
+
!hasDeferredStubs(testDir, ext)) {
|
|
411
415
|
results.push({
|
|
412
416
|
journeyName: journey.name,
|
|
413
417
|
testDir,
|
|
@@ -461,9 +465,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
461
465
|
// the single biggest cost in a multi-file journey. Best-effort: if it
|
|
462
466
|
// fails, `grounding` stays undefined and each file falls back to reading
|
|
463
467
|
// the source itself, which is slower but still correct.
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
468
|
+
// Only a file that has to be written or repaired needs it, so it is read
|
|
469
|
+
// on first use: re-checking an existing suite costs no grounding at all.
|
|
470
|
+
let groundingP;
|
|
471
|
+
const grounding = () => (groundingP ??= useAI
|
|
472
|
+
? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
|
|
473
|
+
: Promise.resolve(undefined));
|
|
467
474
|
const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
|
|
468
475
|
if (staleRemoved.length) {
|
|
469
476
|
warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
|
|
@@ -490,6 +497,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
490
497
|
// Keep a real, parsing file — but a DEFERRED stub is not real
|
|
491
498
|
// coverage, so fall through and re-attempt it on this resume pass.
|
|
492
499
|
if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
|
|
500
|
+
// A suite written before pending tests carried their reason gets
|
|
501
|
+
// it now, deterministically — no regeneration, no AI.
|
|
502
|
+
const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
|
|
503
|
+
if (normalised !== existing)
|
|
504
|
+
writeFileSync(filePath, normalised, 'utf-8');
|
|
493
505
|
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
494
506
|
}
|
|
495
507
|
}
|
|
@@ -500,9 +512,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
500
512
|
let content;
|
|
501
513
|
let status;
|
|
502
514
|
if (useAI) {
|
|
503
|
-
const
|
|
515
|
+
const facts = await grounding();
|
|
516
|
+
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, testGate);
|
|
504
517
|
if (ai !== null) {
|
|
505
|
-
content = ai;
|
|
518
|
+
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
506
519
|
status = 'ai';
|
|
507
520
|
}
|
|
508
521
|
else {
|
|
@@ -552,7 +565,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
552
565
|
}
|
|
553
566
|
// 2. Seed file (only when using AI — basic mode has no seed concept).
|
|
554
567
|
let seedStatus = 'none';
|
|
555
|
-
if (useAI) {
|
|
568
|
+
if (useAI && !force && existsSync(seedPath)) {
|
|
569
|
+
seedStatus = 'ok'; // kept, like an existing test file
|
|
570
|
+
}
|
|
571
|
+
else if (useAI) {
|
|
556
572
|
const seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
|
|
557
573
|
if (seed !== null) {
|
|
558
574
|
mkdirSync(dirname(seedPath), { recursive: true });
|
|
@@ -593,12 +609,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
593
609
|
warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
|
|
594
610
|
lint.remaining.slice(0, 5).join('; '));
|
|
595
611
|
}
|
|
596
|
-
// Run what we just wrote
|
|
597
|
-
//
|
|
598
|
-
//
|
|
599
|
-
//
|
|
600
|
-
//
|
|
612
|
+
// Run what we just wrote against the real application before calling it
|
|
613
|
+
// done — file by file, then the whole journey together, then an independent
|
|
614
|
+
// review of every claimed defect (see verify-journey.ts). Anything that
|
|
615
|
+
// could not run, or still fails unexplained, leaves the journey UNVERIFIED
|
|
616
|
+
// and uncached, so the next run checks it again.
|
|
601
617
|
let verified;
|
|
618
|
+
let reviewed = [];
|
|
602
619
|
const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
|
|
603
620
|
const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
|
|
604
621
|
if (useAI && verifyOn && targets.length && missing.length) {
|
|
@@ -608,66 +625,38 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
608
625
|
warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
|
|
609
626
|
}
|
|
610
627
|
else if (useAI && verifyOn && targets.length) {
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
break;
|
|
629
|
-
}
|
|
630
|
-
if (outcome.ok) {
|
|
631
|
-
verified = true;
|
|
632
|
-
if (attempt > 0)
|
|
633
|
-
console.log(` verified after ${attempt} repair(s)`);
|
|
634
|
-
break;
|
|
635
|
-
}
|
|
636
|
-
verified = false;
|
|
637
|
-
if (attempt === verifyAttempts) {
|
|
638
|
-
warnings.push(`verify: still failing after ${verifyAttempts} repair attempt(s) — ` +
|
|
639
|
-
'this may be a real defect in the application, or a generation ' +
|
|
640
|
-
'defect this run could not correct. Left uncached for a retry.');
|
|
641
|
-
break;
|
|
642
|
-
}
|
|
643
|
-
console.log(` failed when run — repairing (attempt ${attempt + 1}/${verifyAttempts})`);
|
|
644
|
-
for (const target of targets) {
|
|
645
|
-
let current;
|
|
646
|
-
try {
|
|
647
|
-
current = readFileSync(target, 'utf-8');
|
|
648
|
-
}
|
|
649
|
-
catch {
|
|
650
|
-
continue;
|
|
651
|
-
}
|
|
652
|
-
const repaired = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, outcome.output, harnessPath, transport, grounding), ext, testGate);
|
|
653
|
-
if (repaired === null)
|
|
654
|
-
continue;
|
|
655
|
-
const stamped = repaired.includes(GEN_MARKER)
|
|
656
|
-
? repaired
|
|
657
|
-
: `// ${GEN_MARKER}\n${repaired}`;
|
|
658
|
-
writeFileSync(target, stamped, 'utf-8');
|
|
659
|
-
}
|
|
660
|
-
}
|
|
628
|
+
const result = await verifyJourney({
|
|
629
|
+
config,
|
|
630
|
+
journey,
|
|
631
|
+
targets,
|
|
632
|
+
attempts: verifyAttempts,
|
|
633
|
+
repair: async (target, current, failure) => {
|
|
634
|
+
const facts = await grounding();
|
|
635
|
+
const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
|
|
636
|
+
if (out === null)
|
|
637
|
+
return null;
|
|
638
|
+
const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
|
|
639
|
+
return framework === 'playwright' ? normalisePending(stamped) : stamped;
|
|
640
|
+
},
|
|
641
|
+
});
|
|
642
|
+
verified = result.verified;
|
|
643
|
+
reviewed = result.reviewed;
|
|
644
|
+
warnings.push(...result.warnings);
|
|
661
645
|
}
|
|
662
646
|
const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
|
|
663
647
|
if (!degraded) {
|
|
664
648
|
cache[file] = hash;
|
|
649
|
+
// Remembered apart, so a journey cached before it was ever run is
|
|
650
|
+
// re-checked (not regenerated) once verification is on.
|
|
651
|
+
if (verified)
|
|
652
|
+
cache[`verified:${file}`] = hash;
|
|
665
653
|
}
|
|
666
654
|
else {
|
|
667
655
|
// Actively UN-mark a degraded journey (even under --force, which may
|
|
668
656
|
// be re-processing a stale "done" from an earlier buggy run), so the
|
|
669
657
|
// next `generate` re-does it instead of skipping a thin fallback.
|
|
670
658
|
delete cache[file];
|
|
659
|
+
delete cache[`verified:${file}`];
|
|
671
660
|
}
|
|
672
661
|
saveCache(CACHE_PATH, cache);
|
|
673
662
|
resultRow.set(file, results.length);
|
|
@@ -684,6 +673,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
684
673
|
seedStatus,
|
|
685
674
|
verified,
|
|
686
675
|
warnings: warnings.length ? warnings : undefined,
|
|
676
|
+
reviewed: reviewed.length ? reviewed : undefined,
|
|
687
677
|
});
|
|
688
678
|
}
|
|
689
679
|
for (const [area, b] of blockedAreas) {
|
|
@@ -725,8 +715,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
725
715
|
* Write the per-area runner config.
|
|
726
716
|
* - playwright: global-setup.js + playwright.config.js
|
|
727
717
|
* - jest: helpers/jest.setup.js + jest.config.js
|
|
728
|
-
*
|
|
729
|
-
* never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
|
|
718
|
+
* A config TestForge wrote is rewritten to the current template; a project's
|
|
719
|
+
* own (no marker) is never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
|
|
730
720
|
*/
|
|
731
721
|
export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
|
|
732
722
|
const posix = (from, to) => relative(from, resolve(to)).split(sep).join('/') || '.';
|
|
@@ -735,7 +725,8 @@ export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
|
|
|
735
725
|
writeFileSync(gsPath, globalSetupContent(), 'utf-8');
|
|
736
726
|
const home = configHome(areaDir, 'playwright.config.js');
|
|
737
727
|
const cfg = join(home, 'playwright.config.js');
|
|
738
|
-
|
|
728
|
+
// Ours (it carries the marker) is kept current; a project's own is never touched.
|
|
729
|
+
if (!existsSync(cfg) || readFileSync(cfg, 'utf-8').includes(GEN_MARKER)) {
|
|
739
730
|
writeFileSync(cfg, playwrightConfig(`./${posix(home, areaDir)}`, `./${posix(home, gsPath)}`, area, posix(home, RESULTS_DIR)), 'utf-8');
|
|
740
731
|
}
|
|
741
732
|
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — check a journey's generated tests against the real app before
|
|
3
|
+
* they are handed over.
|
|
4
|
+
*
|
|
5
|
+
* A test that passes every static gate can still target a selector, route or
|
|
6
|
+
* field the app does not have; only running it can tell. Three passes:
|
|
7
|
+
*
|
|
8
|
+
* 1. Each file on its own. A failure, or a skip the test decided by itself,
|
|
9
|
+
* is repaired from the runner's own output. A test the repair judges to
|
|
10
|
+
* be failing on the APPLICATION is left as it is and marked.
|
|
11
|
+
* 2. The whole journey in one run, as a team or CI runs it. Files that pass
|
|
12
|
+
* alone can still break each other (one signs in again as a shared
|
|
13
|
+
* account and ends another's session); those are repaired with that
|
|
14
|
+
* context.
|
|
15
|
+
* 3. An independent review of every marked failure. Only what it confirms
|
|
16
|
+
* is reported; the rest is repaired once more or reported as unverified.
|
|
17
|
+
*
|
|
18
|
+
* A guard refuses any repair that deletes a test, adds a skip or loses an
|
|
19
|
+
* assertion, so no repair can turn a caught defect into a silent one.
|
|
20
|
+
*/
|
|
21
|
+
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
|
22
|
+
import { basename, join, resolve } from 'node:path';
|
|
23
|
+
import { STATE_DIR } from '../paths.js';
|
|
24
|
+
import { DEFECT_MARK } from './ai-generator.js';
|
|
25
|
+
import { groupClaims, reviewDefects } from './defect-review.js';
|
|
26
|
+
import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
|
|
27
|
+
const describe = (label, issues) => issues.length
|
|
28
|
+
? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
|
|
29
|
+
: [];
|
|
30
|
+
const key = (i) => `${i.file}:${i.line}:${i.title}`;
|
|
31
|
+
const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
|
|
32
|
+
const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
|
|
33
|
+
const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
|
|
34
|
+
const usable = (o) => o.ran && !o.infra;
|
|
35
|
+
export async function verifyJourney(ctx) {
|
|
36
|
+
const { config, journey, targets } = ctx;
|
|
37
|
+
const warnings = [];
|
|
38
|
+
const notRun = [];
|
|
39
|
+
// One guarded repair of one file; true when the file changed.
|
|
40
|
+
const repairOnce = async (target, open, suspect, note = '') => {
|
|
41
|
+
const current = readFileSync(target, 'utf-8');
|
|
42
|
+
const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
|
|
43
|
+
const next = await ctx.repair(target, current, failure);
|
|
44
|
+
if (next === null)
|
|
45
|
+
return false;
|
|
46
|
+
const refused = repairGuard(current, next);
|
|
47
|
+
if (refused) {
|
|
48
|
+
warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
writeFileSync(target, next, 'utf-8');
|
|
52
|
+
return true;
|
|
53
|
+
};
|
|
54
|
+
// 1. Each file on its own: what is wrong inside the file.
|
|
55
|
+
let spent = 0;
|
|
56
|
+
const soloSkips = new Set();
|
|
57
|
+
const lastSolo = new Map();
|
|
58
|
+
for (const target of targets) {
|
|
59
|
+
const run = async () => {
|
|
60
|
+
const started = Date.now();
|
|
61
|
+
const o = await verifyFile(config, journey.area, target);
|
|
62
|
+
spent += Date.now() - started;
|
|
63
|
+
return o;
|
|
64
|
+
};
|
|
65
|
+
let outcome = await run();
|
|
66
|
+
for (let attempt = 0;; attempt += 1) {
|
|
67
|
+
if (!usable(outcome)) {
|
|
68
|
+
notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
const open = unclaimed(outcome.failures);
|
|
72
|
+
// A skip the test decided by itself is a suspect once: its own setup may
|
|
73
|
+
// have looked in the wrong place. A skip that only names a missing env
|
|
74
|
+
// var or opt-in flag is the environment's to fix.
|
|
75
|
+
const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
|
|
76
|
+
if ((!open.length && !suspect.length) || attempt === ctx.attempts)
|
|
77
|
+
break;
|
|
78
|
+
console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
79
|
+
if (!(await repairOnce(target, open, suspect)))
|
|
80
|
+
break;
|
|
81
|
+
outcome = await run();
|
|
82
|
+
}
|
|
83
|
+
lastSolo.set(target, outcome);
|
|
84
|
+
for (const k of outcome.skips)
|
|
85
|
+
soloSkips.add(key(k));
|
|
86
|
+
}
|
|
87
|
+
// 2. The whole journey in ONE run. Whatever fails only here is repaired with
|
|
88
|
+
// that context, once.
|
|
89
|
+
let together;
|
|
90
|
+
if (!notRun.length && targets.length > 1) {
|
|
91
|
+
const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
|
|
92
|
+
const order = targets.map((t) => basename(t)).sort().join(', ');
|
|
93
|
+
const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
|
|
94
|
+
`each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
|
|
95
|
+
`(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
|
|
96
|
+
`backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
|
|
97
|
+
`data it sets up itself.`;
|
|
98
|
+
// The runner reports absolute paths; targets may be relative.
|
|
99
|
+
const byAbs = new Map(targets.map((t) => [resolve(t), t]));
|
|
100
|
+
console.log(` running all ${targets.length} files together…`);
|
|
101
|
+
together = await verifyRun(config, journey.area, targets, budget);
|
|
102
|
+
if (usable(together)) {
|
|
103
|
+
const open = unclaimed(together.failures);
|
|
104
|
+
const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
|
|
105
|
+
const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
|
|
106
|
+
if (files.length) {
|
|
107
|
+
console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
|
|
108
|
+
for (const f of files) {
|
|
109
|
+
await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
|
|
110
|
+
}
|
|
111
|
+
together = await verifyRun(config, journey.area, targets, budget);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (!usable(together))
|
|
115
|
+
notRun.push(`all files together: ${together.infra ?? together.reason}`);
|
|
116
|
+
}
|
|
117
|
+
// 3. Judge what is left on the LAST run that covered each file. Nothing is
|
|
118
|
+
// reported as a defect on the repair's word alone.
|
|
119
|
+
const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
|
|
120
|
+
const failures = finals.flatMap((o) => o.failures);
|
|
121
|
+
const unresolved = unclaimed(failures).map(label);
|
|
122
|
+
let reviewed = [];
|
|
123
|
+
const groups = groupClaims(failures, claimOf);
|
|
124
|
+
if (groups.length) {
|
|
125
|
+
console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
|
|
126
|
+
const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes));
|
|
127
|
+
reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
|
|
128
|
+
// A rejected claim is the test's fault until shown otherwise: one guarded
|
|
129
|
+
// repair with the reviewer's reason; whatever still fails is unverified and
|
|
130
|
+
// is not re-claimed as a defect.
|
|
131
|
+
for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
|
|
132
|
+
const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
|
|
133
|
+
`Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
|
|
134
|
+
for (const f of new Set(r.tests.map((t) => t.file))) {
|
|
135
|
+
const target = targets.find((t) => resolve(t) === f) ?? f;
|
|
136
|
+
const mine = r.tests.filter((t) => t.file === f);
|
|
137
|
+
if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
|
|
138
|
+
const again = await verifyFile(config, journey.area, target);
|
|
139
|
+
// Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
|
|
140
|
+
for (const t of again.failures) {
|
|
141
|
+
if (mine.some((m) => m.title === t.title) || !claimOf(t))
|
|
142
|
+
unresolved.push(`${label(t)} (after review)`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
for (const t of mine)
|
|
147
|
+
unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
|
|
152
|
+
writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
|
|
153
|
+
}
|
|
154
|
+
if (notRun.length) {
|
|
155
|
+
warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
|
|
156
|
+
}
|
|
157
|
+
if (unresolved.length) {
|
|
158
|
+
warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
|
|
159
|
+
`do not report them; ${unresolved.slice(0, 5).join('; ')}`);
|
|
160
|
+
}
|
|
161
|
+
return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
|
|
162
|
+
}
|