@activate-core/test-forge 1.0.4 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -27,7 +27,9 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
29
  import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
30
- import { verifyFiles, browserPreflight, missingPackages } from './verify.js';
30
+ import { browserPreflight, missingPackages } from './verify.js';
31
+ import { verifyJourney } from './verify-journey.js';
32
+ import { normalisePending } from './pending.js';
31
33
  import { packageRoot } from '../run/run-tests.js';
32
34
  import { groundJourney } from '../docs/grounding.js';
33
35
  import { generateSeed } from './seed-generator.js';
@@ -180,7 +182,6 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
180
182
  // "on" never means "fail the run because nothing is running".
181
183
  const verifyOn = opts.verify ?? config.verify ?? true;
182
184
  const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
183
- const verifySkipNoted = new Set();
184
185
  const ext = config.testExtension ?? '.test.ts';
185
186
  const seedDir = config.seedOutput ?? 'generated-tests/seed';
186
187
  const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
@@ -399,7 +400,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
399
400
  selectorsByArea.set(areaDir, knownSelectors);
400
401
  }
401
402
  }
402
- const testGate = (content) => checkTestGates(content, ext, {
403
+ const testGate = (content) => checkTestGates(framework === 'playwright' ? normalisePending(content) : content, ext, {
403
404
  framework,
404
405
  allowedModules,
405
406
  knownSelectors,
@@ -407,7 +408,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
407
408
  });
408
409
  // Skip if unchanged — but NOT if the journey still has deferred stubs
409
410
  // to retry (a resume pass fills those in without touching real files).
410
- if (!force && cache[file] === hash && !hasDeferredStubs(testDir, ext)) {
411
+ if (!force &&
412
+ cache[file] === hash &&
413
+ (!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
414
+ !hasDeferredStubs(testDir, ext)) {
411
415
  results.push({
412
416
  journeyName: journey.name,
413
417
  testDir,
@@ -461,9 +465,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
461
465
  // the single biggest cost in a multi-file journey. Best-effort: if it
462
466
  // fails, `grounding` stays undefined and each file falls back to reading
463
467
  // the source itself, which is slower but still correct.
464
- const grounding = useAI
465
- ? (await groundJourney(journey, hash, 'src', markdown)) ?? undefined
466
- : undefined;
468
+ // Only a file that has to be written or repaired needs it, so it is read
469
+ // on first use: re-checking an existing suite costs no grounding at all.
470
+ let groundingP;
471
+ const grounding = () => (groundingP ??= useAI
472
+ ? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
473
+ : Promise.resolve(undefined));
467
474
  const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
468
475
  if (staleRemoved.length) {
469
476
  warnings.push(`removed ${staleRemoved.length} stale file(s) from a previous plan: ${staleRemoved.join(', ')}`);
@@ -490,6 +497,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
490
497
  // Keep a real, parsing file — but a DEFERRED stub is not real
491
498
  // coverage, so fall through and re-attempt it on this resume pass.
492
499
  if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
500
+ // A suite written before pending tests carried their reason gets
501
+ // it now, deterministically — no regeneration, no AI.
502
+ const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
503
+ if (normalised !== existing)
504
+ writeFileSync(filePath, normalised, 'utf-8');
493
505
  return { slug: pf.slug, path: filePath, status: 'ai' };
494
506
  }
495
507
  }
@@ -500,9 +512,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
500
512
  let content;
501
513
  let status;
502
514
  if (useAI) {
503
- const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, grounding), ext, testGate);
515
+ const facts = await grounding();
516
+ const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, testGate);
504
517
  if (ai !== null) {
505
- content = ai;
518
+ content = framework === 'playwright' ? normalisePending(ai) : ai;
506
519
  status = 'ai';
507
520
  }
508
521
  else {
@@ -552,7 +565,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
552
565
  }
553
566
  // 2. Seed file (only when using AI — basic mode has no seed concept).
554
567
  let seedStatus = 'none';
555
- if (useAI) {
568
+ if (useAI && !force && existsSync(seedPath)) {
569
+ seedStatus = 'ok'; // kept, like an existing test file
570
+ }
571
+ else if (useAI) {
556
572
  const seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
557
573
  if (seed !== null) {
558
574
  mkdirSync(dirname(seedPath), { recursive: true });
@@ -593,12 +609,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
593
609
  warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
594
610
  lint.remaining.slice(0, 5).join('; '));
595
611
  }
596
- // Run what we just wrote, once, against the real application. A test
597
- // that passes every static gate can still target a selector, route or
598
- // column the app does not have — only running it can tell. The runner's
599
- // own output is the repair prompt; a journey that still fails afterwards
600
- // is reported and left UNCACHED, so it is retried rather than trusted.
612
+ // Run what we just wrote against the real application before calling it
613
+ // done — file by file, then the whole journey together, then an independent
614
+ // review of every claimed defect (see verify-journey.ts). Anything that
615
+ // could not run, or still fails unexplained, leaves the journey UNVERIFIED
616
+ // and uncached, so the next run checks it again.
601
617
  let verified;
618
+ let reviewed = [];
602
619
  const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
603
620
  const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
604
621
  if (useAI && verifyOn && targets.length && missing.length) {
@@ -608,66 +625,38 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
608
625
  warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
609
626
  }
610
627
  else if (useAI && verifyOn && targets.length) {
611
- for (let attempt = 0; attempt <= verifyAttempts; attempt += 1) {
612
- const outcome = verifyFiles(config, journey.area, targets);
613
- if (outcome.infra) {
614
- // Not the tests' fault, so no repair; and not verified, so the
615
- // journey stays uncached and the next run checks it again.
616
- verified = false;
617
- warnings.push(`verify: not run — ${outcome.reason}`);
618
- break;
619
- }
620
- if (!outcome.ran) {
621
- // Say WHY once. The reason is a property of the project (no env
622
- // file, no runner), not of this journey, so repeating it per
623
- // journey buries the rest of the output.
624
- if (outcome.reason && !verifySkipNoted.has(outcome.reason)) {
625
- verifySkipNoted.add(outcome.reason);
626
- console.log(` not verified — ${outcome.reason}`);
627
- }
628
- break;
629
- }
630
- if (outcome.ok) {
631
- verified = true;
632
- if (attempt > 0)
633
- console.log(` verified after ${attempt} repair(s)`);
634
- break;
635
- }
636
- verified = false;
637
- if (attempt === verifyAttempts) {
638
- warnings.push(`verify: still failing after ${verifyAttempts} repair attempt(s) — ` +
639
- 'this may be a real defect in the application, or a generation ' +
640
- 'defect this run could not correct. Left uncached for a retry.');
641
- break;
642
- }
643
- console.log(` failed when run — repairing (attempt ${attempt + 1}/${verifyAttempts})`);
644
- for (const target of targets) {
645
- let current;
646
- try {
647
- current = readFileSync(target, 'utf-8');
648
- }
649
- catch {
650
- continue;
651
- }
652
- const repaired = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, outcome.output, harnessPath, transport, grounding), ext, testGate);
653
- if (repaired === null)
654
- continue;
655
- const stamped = repaired.includes(GEN_MARKER)
656
- ? repaired
657
- : `// ${GEN_MARKER}\n${repaired}`;
658
- writeFileSync(target, stamped, 'utf-8');
659
- }
660
- }
628
+ const result = await verifyJourney({
629
+ config,
630
+ journey,
631
+ targets,
632
+ attempts: verifyAttempts,
633
+ repair: async (target, current, failure) => {
634
+ const facts = await grounding();
635
+ const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
636
+ if (out === null)
637
+ return null;
638
+ const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
639
+ return framework === 'playwright' ? normalisePending(stamped) : stamped;
640
+ },
641
+ });
642
+ verified = result.verified;
643
+ reviewed = result.reviewed;
644
+ warnings.push(...result.warnings);
661
645
  }
662
646
  const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
663
647
  if (!degraded) {
664
648
  cache[file] = hash;
649
+ // Remembered apart, so a journey cached before it was ever run is
650
+ // re-checked (not regenerated) once verification is on.
651
+ if (verified)
652
+ cache[`verified:${file}`] = hash;
665
653
  }
666
654
  else {
667
655
  // Actively UN-mark a degraded journey (even under --force, which may
668
656
  // be re-processing a stale "done" from an earlier buggy run), so the
669
657
  // next `generate` re-does it instead of skipping a thin fallback.
670
658
  delete cache[file];
659
+ delete cache[`verified:${file}`];
671
660
  }
672
661
  saveCache(CACHE_PATH, cache);
673
662
  resultRow.set(file, results.length);
@@ -684,6 +673,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
684
673
  seedStatus,
685
674
  verified,
686
675
  warnings: warnings.length ? warnings : undefined,
676
+ reviewed: reviewed.length ? reviewed : undefined,
687
677
  });
688
678
  }
689
679
  for (const [area, b] of blockedAreas) {
@@ -725,8 +715,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
725
715
  * Write the per-area runner config.
726
716
  * - playwright: global-setup.js + playwright.config.js
727
717
  * - jest: helpers/jest.setup.js + jest.config.js
728
- * Config files are written only if absent, so a project's hand-tuned config is
729
- * never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
718
+ * A config TestForge wrote is rewritten to the current template; a project's
719
+ * own (no marker) is never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
730
720
  */
731
721
  export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
732
722
  const posix = (from, to) => relative(from, resolve(to)).split(sep).join('/') || '.';
@@ -735,7 +725,8 @@ export function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
735
725
  writeFileSync(gsPath, globalSetupContent(), 'utf-8');
736
726
  const home = configHome(areaDir, 'playwright.config.js');
737
727
  const cfg = join(home, 'playwright.config.js');
738
- if (!existsSync(cfg)) {
728
+ // Ours (it carries the marker) is kept current; a project's own is never touched.
729
+ if (!existsSync(cfg) || readFileSync(cfg, 'utf-8').includes(GEN_MARKER)) {
739
730
  writeFileSync(cfg, playwrightConfig(`./${posix(home, areaDir)}`, `./${posix(home, gsPath)}`, area, posix(home, RESULTS_DIR)), 'utf-8');
740
731
  }
741
732
  }
@@ -0,0 +1,162 @@
1
+ /**
2
+ * testforge — check a journey's generated tests against the real app before
3
+ * they are handed over.
4
+ *
5
+ * A test that passes every static gate can still target a selector, route or
6
+ * field the app does not have; only running it can tell. Three passes:
7
+ *
8
+ * 1. Each file on its own. A failure, or a skip the test decided by itself,
9
+ * is repaired from the runner's own output. A test the repair judges to
10
+ * be failing on the APPLICATION is left as it is and marked.
11
+ * 2. The whole journey in one run, as a team or CI runs it. Files that pass
12
+ * alone can still break each other (one signs in again as a shared
13
+ * account and ends another's session); those are repaired with that
14
+ * context.
15
+ * 3. An independent review of every marked failure. Only what it confirms
16
+ * is reported; the rest is repaired once more or reported as unverified.
17
+ *
18
+ * A guard refuses any repair that deletes a test, adds a skip or loses an
19
+ * assertion, so no repair can turn a caught defect into a silent one.
20
+ */
21
+ import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
22
+ import { basename, join, resolve } from 'node:path';
23
+ import { STATE_DIR } from '../paths.js';
24
+ import { DEFECT_MARK } from './ai-generator.js';
25
+ import { groupClaims, reviewDefects } from './defect-review.js';
26
+ import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
27
+ const describe = (label, issues) => issues.length
28
+ ? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
29
+ : [];
30
+ const key = (i) => `${i.file}:${i.line}:${i.title}`;
31
+ const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
32
+ const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
33
+ const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
34
+ const usable = (o) => o.ran && !o.infra;
35
+ export async function verifyJourney(ctx) {
36
+ const { config, journey, targets } = ctx;
37
+ const warnings = [];
38
+ const notRun = [];
39
+ // One guarded repair of one file; true when the file changed.
40
+ const repairOnce = async (target, open, suspect, note = '') => {
41
+ const current = readFileSync(target, 'utf-8');
42
+ const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
43
+ const next = await ctx.repair(target, current, failure);
44
+ if (next === null)
45
+ return false;
46
+ const refused = repairGuard(current, next);
47
+ if (refused) {
48
+ warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
49
+ return false;
50
+ }
51
+ writeFileSync(target, next, 'utf-8');
52
+ return true;
53
+ };
54
+ // 1. Each file on its own: what is wrong inside the file.
55
+ let spent = 0;
56
+ const soloSkips = new Set();
57
+ const lastSolo = new Map();
58
+ for (const target of targets) {
59
+ const run = async () => {
60
+ const started = Date.now();
61
+ const o = await verifyFile(config, journey.area, target);
62
+ spent += Date.now() - started;
63
+ return o;
64
+ };
65
+ let outcome = await run();
66
+ for (let attempt = 0;; attempt += 1) {
67
+ if (!usable(outcome)) {
68
+ notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
69
+ break;
70
+ }
71
+ const open = unclaimed(outcome.failures);
72
+ // A skip the test decided by itself is a suspect once: its own setup may
73
+ // have looked in the wrong place. A skip that only names a missing env
74
+ // var or opt-in flag is the environment's to fix.
75
+ const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
76
+ if ((!open.length && !suspect.length) || attempt === ctx.attempts)
77
+ break;
78
+ console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
79
+ if (!(await repairOnce(target, open, suspect)))
80
+ break;
81
+ outcome = await run();
82
+ }
83
+ lastSolo.set(target, outcome);
84
+ for (const k of outcome.skips)
85
+ soloSkips.add(key(k));
86
+ }
87
+ // 2. The whole journey in ONE run. Whatever fails only here is repaired with
88
+ // that context, once.
89
+ let together;
90
+ if (!notRun.length && targets.length > 1) {
91
+ const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
92
+ const order = targets.map((t) => basename(t)).sort().join(', ');
93
+ const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
94
+ `each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
95
+ `(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
96
+ `backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
97
+ `data it sets up itself.`;
98
+ // The runner reports absolute paths; targets may be relative.
99
+ const byAbs = new Map(targets.map((t) => [resolve(t), t]));
100
+ console.log(` running all ${targets.length} files together…`);
101
+ together = await verifyRun(config, journey.area, targets, budget);
102
+ if (usable(together)) {
103
+ const open = unclaimed(together.failures);
104
+ const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
105
+ const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
106
+ if (files.length) {
107
+ console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
108
+ for (const f of files) {
109
+ await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
110
+ }
111
+ together = await verifyRun(config, journey.area, targets, budget);
112
+ }
113
+ }
114
+ if (!usable(together))
115
+ notRun.push(`all files together: ${together.infra ?? together.reason}`);
116
+ }
117
+ // 3. Judge what is left on the LAST run that covered each file. Nothing is
118
+ // reported as a defect on the repair's word alone.
119
+ const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
120
+ const failures = finals.flatMap((o) => o.failures);
121
+ const unresolved = unclaimed(failures).map(label);
122
+ let reviewed = [];
123
+ const groups = groupClaims(failures, claimOf);
124
+ if (groups.length) {
125
+ console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
126
+ const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes));
127
+ reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
128
+ // A rejected claim is the test's fault until shown otherwise: one guarded
129
+ // repair with the reviewer's reason; whatever still fails is unverified and
130
+ // is not re-claimed as a defect.
131
+ for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
132
+ const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
133
+ `Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
134
+ for (const f of new Set(r.tests.map((t) => t.file))) {
135
+ const target = targets.find((t) => resolve(t) === f) ?? f;
136
+ const mine = r.tests.filter((t) => t.file === f);
137
+ if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
138
+ const again = await verifyFile(config, journey.area, target);
139
+ // Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
140
+ for (const t of again.failures) {
141
+ if (mine.some((m) => m.title === t.title) || !claimOf(t))
142
+ unresolved.push(`${label(t)} (after review)`);
143
+ }
144
+ }
145
+ else {
146
+ for (const t of mine)
147
+ unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
148
+ }
149
+ }
150
+ }
151
+ mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
152
+ writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
153
+ }
154
+ if (notRun.length) {
155
+ warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
156
+ }
157
+ if (unresolved.length) {
158
+ warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
159
+ `do not report them; ${unresolved.slice(0, 5).join('; ')}`);
160
+ }
161
+ return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
162
+ }