@activate-core/test-forge 1.0.4 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -27,7 +27,9 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
29
  import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
30
- import { verifyFiles, browserPreflight, missingPackages } from './verify.js';
30
+ import { browserPreflight, missingPackages } from './verify.js';
31
+ import { verifyJourney } from './verify-journey.js';
32
+ import { normalisePending } from './pending.js';
31
33
  import { packageRoot } from '../run/run-tests.js';
32
34
  import { groundJourney } from '../docs/grounding.js';
33
35
  import { generateSeed } from './seed-generator.js';
@@ -180,7 +182,6 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
180
182
  // "on" never means "fail the run because nothing is running".
181
183
  const verifyOn = opts.verify ?? config.verify ?? true;
182
184
  const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
183
- const verifySkipNoted = new Set();
184
185
  const ext = config.testExtension ?? '.test.ts';
185
186
  const seedDir = config.seedOutput ?? 'generated-tests/seed';
186
187
  const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
@@ -399,15 +400,22 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
399
400
  selectorsByArea.set(areaDir, knownSelectors);
400
401
  }
401
402
  }
402
- const testGate = (content) => checkTestGates(content, ext, {
403
+ const testGate = (content) => checkTestGates(framework === 'playwright' ? normalisePending(content) : content, ext, {
403
404
  framework,
404
405
  allowedModules,
405
406
  knownSelectors,
406
407
  externalModules: config.externalModules,
407
408
  });
409
+ // A freshly written file also may not hold placeholders; a repair is held to
410
+ // the repair guard instead (it may not add a skip), so old files stay repairable.
411
+ const genGate = (content) => testGate(content) ??
412
+ checkTestGates(content, ext, { framework, forbidPlaceholders: true });
408
413
  // Skip if unchanged — but NOT if the journey still has deferred stubs
409
414
  // to retry (a resume pass fills those in without touching real files).
410
- if (!force && cache[file] === hash && !hasDeferredStubs(testDir, ext)) {
415
+ if (!force &&
416
+ cache[file] === hash &&
417
+ (!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
418
+ !hasDeferredStubs(testDir, ext)) {
411
419
  results.push({
412
420
  journeyName: journey.name,
413
421
  testDir,
@@ -461,9 +469,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
461
469
  // the single biggest cost in a multi-file journey. Best-effort: if it
462
470
  // fails, `grounding` stays undefined and each file falls back to reading
463
471
  // the source itself, which is slower but still correct.
464
- const grounding = useAI
465
- ? (await groundJourney(journey, hash, 'src', markdown)) ?? undefined
466
- : undefined;
472
+ // Only a file that has to be written or repaired needs it, so it is read
473
+ // on first use: re-checking an existing suite costs no grounding at all.
474
+ let groundingP;
475
+ const grounding = () => (groundingP ??= useAI
476
+ ? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
477
+ : Promise.resolve(undefined));
467
478
  const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
468
479
  if (staleRemoved.length) {
469
480
  warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
@@ -490,6 +501,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
490
501
  // Keep a real, parsing file — but a DEFERRED stub is not real
491
502
  // coverage, so fall through and re-attempt it on this resume pass.
492
503
  if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
504
+ // A suite written before pending tests carried their reason gets
505
+ // it now, deterministically — no regeneration, no AI.
506
+ const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
507
+ if (normalised !== existing)
508
+ writeFileSync(filePath, normalised, 'utf-8');
493
509
  return { slug: pf.slug, path: filePath, status: 'ai' };
494
510
  }
495
511
  }
@@ -500,9 +516,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
500
516
  let content;
501
517
  let status;
502
518
  if (useAI) {
503
- const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, grounding), ext, testGate);
519
+ const facts = await grounding();
520
+ const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, genGate);
504
521
  if (ai !== null) {
505
- content = ai;
522
+ content = framework === 'playwright' ? normalisePending(ai) : ai;
506
523
  status = 'ai';
507
524
  }
508
525
  else {
@@ -552,7 +569,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
552
569
  }
553
570
  // 2. Seed file (only when using AI — basic mode has no seed concept).
554
571
  let seedStatus = 'none';
555
- if (useAI) {
572
+ if (useAI && !force && existsSync(seedPath)) {
573
+ seedStatus = 'ok'; // kept, like an existing test file
574
+ }
575
+ else if (useAI) {
556
576
  const seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
557
577
  if (seed !== null) {
558
578
  mkdirSync(dirname(seedPath), { recursive: true });
@@ -593,12 +613,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
593
613
  warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
594
614
  lint.remaining.slice(0, 5).join('; '));
595
615
  }
596
- // Run what we just wrote, once, against the real application. A test
597
- // that passes every static gate can still target a selector, route or
598
- // column the app does not have — only running it can tell. The runner's
599
- // own output is the repair prompt; a journey that still fails afterwards
600
- // is reported and left UNCACHED, so it is retried rather than trusted.
616
+ // Run what we just wrote against the real application before calling it
617
+ // done — file by file, then the whole journey together, then an independent
618
+ // review of every claimed defect (see verify-journey.ts). Anything that
619
+ // could not run, or still fails unexplained, leaves the journey UNVERIFIED
620
+ // and uncached, so the next run checks it again.
601
621
  let verified;
622
+ let reviewed = [];
602
623
  const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
603
624
  const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
604
625
  if (useAI && verifyOn && targets.length && missing.length) {
@@ -608,66 +629,38 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
608
629
  warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
609
630
  }
610
631
  else if (useAI && verifyOn && targets.length) {
611
- for (let attempt = 0; attempt <= verifyAttempts; attempt += 1) {
612
- const outcome = verifyFiles(config, journey.area, targets);
613
- if (outcome.infra) {
614
- // Not the tests' fault, so no repair; and not verified, so the
615
- // journey stays uncached and the next run checks it again.
616
- verified = false;
617
- warnings.push(`verify: not run — ${outcome.reason}`);
618
- break;
619
- }
620
- if (!outcome.ran) {
621
- // Say WHY once. The reason is a property of the project (no env
622
- // file, no runner), not of this journey, so repeating it per
623
- // journey buries the rest of the output.
624
- if (outcome.reason && !verifySkipNoted.has(outcome.reason)) {
625
- verifySkipNoted.add(outcome.reason);
626
- console.log(` not verified — ${outcome.reason}`);
627
- }
628
- break;
629
- }
630
- if (outcome.ok) {
631
- verified = true;
632
- if (attempt > 0)
633
- console.log(` verified after ${attempt} repair(s)`);
634
- break;
635
- }
636
- verified = false;
637
- if (attempt === verifyAttempts) {
638
- warnings.push(`verify: still failing after ${verifyAttempts} repair attempt(s) — ` +
639
- 'this may be a real defect in the application, or a generation ' +
640
- 'defect this run could not correct. Left uncached for a retry.');
641
- break;
642
- }
643
- console.log(` failed when run — repairing (attempt ${attempt + 1}/${verifyAttempts})`);
644
- for (const target of targets) {
645
- let current;
646
- try {
647
- current = readFileSync(target, 'utf-8');
648
- }
649
- catch {
650
- continue;
651
- }
652
- const repaired = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, outcome.output, harnessPath, transport, grounding), ext, testGate);
653
- if (repaired === null)
654
- continue;
655
- const stamped = repaired.includes(GEN_MARKER)
656
- ? repaired
657
- : `// ${GEN_MARKER}\n${repaired}`;
658
- writeFileSync(target, stamped, 'utf-8');
659
- }
660
- }
632
+ const result = await verifyJourney({
633
+ config,
634
+ journey,
635
+ targets,
636
+ attempts: verifyAttempts,
637
+ repair: async (target, current, failure) => {
638
+ const facts = await grounding();
639
+ const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
640
+ if (out === null)
641
+ return null;
642
+ const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
643
+ return framework === 'playwright' ? normalisePending(stamped) : stamped;
644
+ },
645
+ });
646
+ verified = result.verified;
647
+ reviewed = result.reviewed;
648
+ warnings.push(...result.warnings);
661
649
  }
662
650
  const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
663
651
  if (!degraded) {
664
652
  cache[file] = hash;
653
+ // Remembered apart, so a journey cached before it was ever run is
654
+ // re-checked (not regenerated) once verification is on.
655
+ if (verified)
656
+ cache[`verified:${file}`] = hash;
665
657
  }
666
658
  else {
667
659
  // Actively UN-mark a degraded journey (even under --force, which may
668
660
  // be re-processing a stale "done" from an earlier buggy run), so the
669
661
  // next `generate` re-does it instead of skipping a thin fallback.
670
662
  delete cache[file];
663
+ delete cache[`verified:${file}`];
671
664
  }
672
665
  saveCache(CACHE_PATH, cache);
673
666
  resultRow.set(file, results.length);
@@ -684,6 +677,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
684
677
  seedStatus,
685
678
  verified,
686
679
  warnings: warnings.length ? warnings : undefined,
680
+ reviewed: reviewed.length ? reviewed : undefined,
687
681
  });
688
682
  }
689
683
  for (const [area, b] of blockedAreas) {
@@ -725,8 +719,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
725
719
  * Write the per-area runner config.
726
720
  * - playwright: global-setup.js + playwright.config.js
727
721
  * - jest: helpers/jest.setup.js + jest.config.js
728
- * Config files are written only if absent, so a project's hand-tuned config is
729
- * never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
722
+ * A config TestForge wrote is rewritten to the current template; a project's
723
+ * own (no marker) is never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
730
724
  */
731
725
  export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
732
726
  const posix = (from, to) => relative(from, resolve(to)).split(sep).join('/') || '.';
@@ -735,7 +729,8 @@ export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
735
729
  writeFileSync(gsPath, globalSetupContent(), 'utf-8');
736
730
  const home = configHome(areaDir, 'playwright.config.js');
737
731
  const cfg = join(home, 'playwright.config.js');
738
- if (!existsSync(cfg)) {
732
+ // Ours (it carries the marker) is kept current; a project's own is never touched.
733
+ if (!existsSync(cfg) || readFileSync(cfg, 'utf-8').includes(GEN_MARKER)) {
739
734
  writeFileSync(cfg, playwrightConfig(`./${posix(home, areaDir)}`, `./${posix(home, gsPath)}`, area, posix(home, RESULTS_DIR)), 'utf-8');
740
735
  }
741
736
  }
@@ -15,11 +15,48 @@
15
15
  * per-file generator turns each entry into an actual focused test.
16
16
  */
17
17
  import { execFileSync } from 'node:child_process';
18
- import { readFileSync, existsSync } from 'node:fs';
18
+ import { readFileSync, existsSync, readdirSync } from 'node:fs';
19
+ import { join } from 'node:path';
19
20
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
21
+ import { PLANS_DIR } from '../paths.js';
20
22
  const MAP_PATH = 'testforge.md';
23
+ /**
24
+ * The other journeys in the same area — their names, and the files already
25
+ * planned for them. A planner that sees only its own journey re-tests every
26
+ * shared form, rule and component it touches; on one real suite that was
27
+ * well over half of all tests. With the siblings in view, each behaviour is
28
+ * planned once, in the journey that owns it.
29
+ */
30
+ export function siblingJourneys(journey, docsDir = 'docs') {
31
+ const lines = [];
32
+ const planned = new Set();
33
+ const dir = join(PLANS_DIR, journey.area);
34
+ for (const f of existsSync(dir) ? readdirSync(dir) : []) {
35
+ const id = f.replace(/\.json$/, '');
36
+ if (!f.endsWith('.json') || id === journey.id)
37
+ continue;
38
+ try {
39
+ const plan = JSON.parse(readFileSync(join(dir, f), 'utf-8'));
40
+ lines.push(`- ${id} (planned):`, ...(plan.files ?? []).map((x) => ` - ${x.title ?? ''}: ${x.covers ?? ''}`));
41
+ planned.add(id);
42
+ }
43
+ catch {
44
+ // unreadable plan — the doc name below still names the journey
45
+ }
46
+ }
47
+ const area = join(docsDir, journey.area);
48
+ for (const f of existsSync(area) ? readdirSync(area) : []) {
49
+ if (!f.endsWith('.md'))
50
+ continue;
51
+ const name = (readFileSync(join(area, f), 'utf-8').match(/^# Journey:\s*(.+)$/m) ?? [])[1];
52
+ if (name && name.trim() !== journey.name && !lines.some((l) => l.includes(name.trim()))) {
53
+ lines.push(`- ${name.trim()} (not planned yet)`);
54
+ }
55
+ }
56
+ return lines.join('\n');
57
+ }
21
58
  /** Build the planning prompt for one journey. */
22
- function buildPlanPrompt(journey, map) {
59
+ export function buildPlanPrompt(journey, map, siblings = '') {
23
60
  return [
24
61
  `You are a senior test architect. Plan how to split ONE user journey`,
25
62
  `into a set of FOCUSED test files — one concern per file — the way a`,
@@ -43,7 +80,19 @@ function buildPlanPrompt(journey, map) {
43
80
  `- "## Controls" and "## Coverage" are indexes, "## Test setup" is what`,
44
81
  ` every file needs to run, and "## Questions for product (not`,
45
82
  ` asserted)" is not tested: plan no file for any of them.`,
83
+ `- ONE OWNER PER BEHAVIOUR. Every behaviour is tested in exactly one`,
84
+ ` file of exactly one journey. A form, component or rule this journey`,
85
+ ` shares with another journey (a login form, a code-entry step, a`,
86
+ ` password rule, a sign-out) belongs to the journey whose subject it is;`,
87
+ ` this journey tests only what is its OWN: how the customer reaches it`,
88
+ ` from here, and the outcome specific to this journey. Say so in`,
89
+ ` "covers" ("opening the login form from X and landing on X after`,
90
+ ` login — the form's own validation is owned by <journey>").`,
91
+ `- Within this journey, no two files test the same behaviour.`,
46
92
  ``,
93
+ ...(siblings
94
+ ? [`OTHER JOURNEYS IN THIS AREA (what they own or will own):`, siblings, ``]
95
+ : []),
47
96
  `Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
48
97
  ` [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,
49
98
  ` "covers": "<one or two sentences: exactly what this file tests>" }]`,
@@ -99,7 +148,7 @@ function safeSlug(raw) {
99
148
  */
100
149
  export function planSuite(journey) {
101
150
  const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
102
- const prompt = buildPlanPrompt(journey, map);
151
+ const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey));
103
152
  const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
104
153
  let raw = '';
105
154
  try {
@@ -0,0 +1,162 @@
1
+ /**
2
+ * testforge — check a journey's generated tests against the real app before
3
+ * they are handed over.
4
+ *
5
+ * A test that passes every static gate can still target a selector, route or
6
+ * field the app does not have; only running it can tell. Three passes:
7
+ *
8
+ * 1. Each file on its own. A failure, or a skip the test decided by itself,
9
+ * is repaired from the runner's own output. A test the repair judges to
10
+ * be failing on the APPLICATION is left as it is and marked.
11
+ * 2. The whole journey in one run, as a team or CI runs it. Files that pass
12
+ * alone can still break each other (one signs in again as a shared
13
+ * account and ends another's session); those are repaired with that
14
+ * context.
15
+ * 3. An independent review of every marked failure. Only what it confirms
16
+ * is reported; the rest is repaired once more or reported as unverified.
17
+ *
18
+ * A guard refuses any repair that deletes a test, adds a skip or loses an
19
+ * assertion, so no repair can turn a caught defect into a silent one.
20
+ */
21
+ import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
22
+ import { basename, join, resolve } from 'node:path';
23
+ import { STATE_DIR } from '../paths.js';
24
+ import { DEFECT_MARK } from './ai-generator.js';
25
+ import { groupClaims, reviewDefects } from './defect-review.js';
26
+ import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
27
+ const describe = (label, issues) => issues.length
28
+ ? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
29
+ : [];
30
+ const key = (i) => `${i.file}:${i.line}:${i.title}`;
31
+ const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
32
+ const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
33
+ const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
34
+ const usable = (o) => o.ran && !o.infra;
35
+ export async function verifyJourney(ctx) {
36
+ const { config, journey, targets } = ctx;
37
+ const warnings = [];
38
+ const notRun = [];
39
+ // One guarded repair of one file; true when the file changed.
40
+ const repairOnce = async (target, open, suspect, note = '') => {
41
+ const current = readFileSync(target, 'utf-8');
42
+ const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
43
+ const next = await ctx.repair(target, current, failure);
44
+ if (next === null)
45
+ return false;
46
+ const refused = repairGuard(current, next);
47
+ if (refused) {
48
+ warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
49
+ return false;
50
+ }
51
+ writeFileSync(target, next, 'utf-8');
52
+ return true;
53
+ };
54
+ // 1. Each file on its own: what is wrong inside the file.
55
+ let spent = 0;
56
+ const soloSkips = new Set();
57
+ const lastSolo = new Map();
58
+ for (const target of targets) {
59
+ const run = async () => {
60
+ const started = Date.now();
61
+ const o = await verifyFile(config, journey.area, target);
62
+ spent += Date.now() - started;
63
+ return o;
64
+ };
65
+ let outcome = await run();
66
+ for (let attempt = 0;; attempt += 1) {
67
+ if (!usable(outcome)) {
68
+ notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
69
+ break;
70
+ }
71
+ const open = unclaimed(outcome.failures);
72
+ // A skip the test decided by itself is a suspect once: its own setup may
73
+ // have looked in the wrong place. A skip that only names a missing env
74
+ // var or opt-in flag is the environment's to fix.
75
+ const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
76
+ if ((!open.length && !suspect.length) || attempt === ctx.attempts)
77
+ break;
78
+ console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
79
+ if (!(await repairOnce(target, open, suspect)))
80
+ break;
81
+ outcome = await run();
82
+ }
83
+ lastSolo.set(target, outcome);
84
+ for (const k of outcome.skips)
85
+ soloSkips.add(key(k));
86
+ }
87
+ // 2. The whole journey in ONE run. Whatever fails only here is repaired with
88
+ // that context, once.
89
+ let together;
90
+ if (!notRun.length && targets.length > 1) {
91
+ const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
92
+ const order = targets.map((t) => basename(t)).sort().join(', ');
93
+ const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
94
+ `each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
95
+ `(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
96
+ `backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
97
+ `data it sets up itself.`;
98
+ // The runner reports absolute paths; targets may be relative.
99
+ const byAbs = new Map(targets.map((t) => [resolve(t), t]));
100
+ console.log(` running all ${targets.length} files together…`);
101
+ together = await verifyRun(config, journey.area, targets, budget);
102
+ if (usable(together)) {
103
+ const open = unclaimed(together.failures);
104
+ const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
105
+ const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
106
+ if (files.length) {
107
+ console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
108
+ for (const f of files) {
109
+ await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
110
+ }
111
+ together = await verifyRun(config, journey.area, targets, budget);
112
+ }
113
+ }
114
+ if (!usable(together))
115
+ notRun.push(`all files together: ${together.infra ?? together.reason}`);
116
+ }
117
+ // 3. Judge what is left on the LAST run that covered each file. Nothing is
118
+ // reported as a defect on the repair's word alone.
119
+ const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
120
+ const failures = finals.flatMap((o) => o.failures);
121
+ const unresolved = unclaimed(failures).map(label);
122
+ let reviewed = [];
123
+ const groups = groupClaims(failures, claimOf);
124
+ if (groups.length) {
125
+ console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
126
+ const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes));
127
+ reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
128
+ // A rejected claim is the test's fault until shown otherwise: one guarded
129
+ // repair with the reviewer's reason; whatever still fails is unverified and
130
+ // is not re-claimed as a defect.
131
+ for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
132
+ const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
133
+ `Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
134
+ for (const f of new Set(r.tests.map((t) => t.file))) {
135
+ const target = targets.find((t) => resolve(t) === f) ?? f;
136
+ const mine = r.tests.filter((t) => t.file === f);
137
+ if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
138
+ const again = await verifyFile(config, journey.area, target);
139
+ // Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
140
+ for (const t of again.failures) {
141
+ if (mine.some((m) => m.title === t.title) || !claimOf(t))
142
+ unresolved.push(`${label(t)} (after review)`);
143
+ }
144
+ }
145
+ else {
146
+ for (const t of mine)
147
+ unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
148
+ }
149
+ }
150
+ }
151
+ mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
152
+ writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
153
+ }
154
+ if (notRun.length) {
155
+ warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
156
+ }
157
+ if (unresolved.length) {
158
+ warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
159
+ `do not report them; ${unresolved.slice(0, 5).join('; ')}`);
160
+ }
161
+ return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
162
+ }