@activate-core/test-forge 1.3.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +39 -33
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +49 -2
  5. package/dist/core/docs/doc-guidance.js +70 -29
  6. package/dist/core/docs/excerpts.js +294 -0
  7. package/dist/core/docs/grounding.js +55 -22
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +60 -3
  10. package/dist/core/docs/review.js +38 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +3 -3
  13. package/dist/core/gates/coverage.js +1 -16
  14. package/dist/core/gates/test-quality.js +295 -0
  15. package/dist/core/gates/validate.js +171 -7
  16. package/dist/core/generate/ai-generator.js +267 -117
  17. package/dist/core/generate/ai-usage.js +116 -5
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/complete-file.js +77 -0
  20. package/dist/core/generate/defect-review.js +59 -9
  21. package/dist/core/generate/doc-diff.js +140 -0
  22. package/dist/core/generate/harness.js +18 -34
  23. package/dist/core/generate/journey-memory.js +394 -0
  24. package/dist/core/generate/plan-cache.js +114 -7
  25. package/dist/core/generate/review-journey.js +3 -3
  26. package/dist/core/generate/run-generate.js +240 -82
  27. package/dist/core/generate/suite-planner.js +128 -23
  28. package/dist/core/generate/test-blocks.js +84 -0
  29. package/dist/core/generate/update-journey.js +118 -0
  30. package/dist/core/paths.js +90 -13
  31. package/dist/core/project/config.js +25 -2
  32. package/dist/core/project/env-template.js +2 -1
  33. package/dist/core/run/ci.js +1 -1
  34. package/dist/core/run/clean.js +1 -1
  35. package/dist/core/run/readiness.js +146 -0
  36. package/dist/core/run/run-tests.js +26 -2
  37. package/package.json +1 -1
  38. package/reporters/_shared.cjs +2 -2
  39. package/reporters/run-data.cjs +1 -1
  40. package/reporters/ui-helpers.cjs +78 -1
  41. package/dist/core/generate/verify-journey.js +0 -215
@@ -18,29 +18,52 @@
18
18
  */
19
19
  import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
20
20
  import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
21
- import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored } from '../paths.js';
21
+ import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored, journeyDir } from '../paths.js';
22
22
  import { lintGenerated } from '../gates/lint.js';
23
23
  import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
24
24
  import { writeCoverageManifest } from '../gates/coverage.js';
25
- import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
25
+ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlan, readPlanSkips, writePlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
26
26
  import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
- import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
30
- import { browserPreflight, missingPackages } from './verify.js';
31
- import { verifyJourney } from './verify-journey.js';
29
+ import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
30
+ import { completeFile, fileIsComplete } from './complete-file.js';
31
+ import { harnessSteps } from '../gates/test-quality.js';
32
32
  import { normalisePending } from './pending.js';
33
33
  import { packageRoot } from '../run/run-tests.js';
34
+ import { pendingReason, withPendingStamp } from '../run/readiness.js';
34
35
  import { groundJourney } from '../docs/grounding.js';
35
36
  import { generateSeed } from './seed-generator.js';
36
37
  import { dataAccess, writeRequiredData } from './required-data.js';
37
38
  import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
38
- import { isSyntaxValid, checkTestGates, duplicatedHelpers } from '../gates/validate.js';
39
+ import { testedTags } from './suite-planner.js';
40
+ import { factsSlice, helperApi, journeySlice, readRules, recordWritten, saveMemory, testedElsewhere } from './journey-memory.js';
41
+ import { loadSourceChoice } from '../docs/source-of-truth.js';
42
+ import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
43
+ import { updateJourney } from './update-journey.js';
44
+ import { sourceText } from '../docs/excerpts.js';
45
+ import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
39
46
  import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
40
47
  import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
41
48
  import { hashContent, loadCache, saveCache } from '../docs/cache.js';
42
49
  import { findMarkdownFiles } from '../find-files.js';
43
50
  import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
51
+ /**
52
+ * kit.find is a shared helper the testkit re-exports beside settledClick. A
53
+ * testkit written before it existed gets the one line here, with no AI and no
54
+ * rebuild.
55
+ */
56
+ export function exportFind(harnessPath) {
57
+ const src = readFileSync(harnessPath, 'utf-8');
58
+ if (/^\s*find\s*[:,]/m.test(src) || !/settledClick:\s*(\w+)\.settledClick,?/.test(src))
59
+ return;
60
+ writeFileSync(harnessPath, src.replace(/^(\s*)settledClick:\s*(\w+)\.settledClick,?/m, (m, pad, mod) => `${m.endsWith(',') ? m : `${m},`}\n${pad}find: ${mod}.find,`), 'utf-8');
61
+ }
62
+ /** Why a file is not valid code, as precisely as the check can say. */
63
+ function syntaxReason(content, ext) {
64
+ const where = /\.(c|m)?js$/i.test(ext) ? null : unbalancedAt(content);
65
+ return `not valid ${ext}${where ? `: ${where}` : ' (syntax)'} — the answer must be the whole file, code only`;
66
+ }
44
67
  /** How many times to re-ask the AI when output won't parse. */
45
68
  const MAX_GEN_ATTEMPTS = 2;
46
69
  /** How many focused files to generate concurrently within a journey. */
@@ -66,13 +89,17 @@ export async function pool(items, limit, fn) {
66
89
  return results;
67
90
  }
68
91
  /** Async version of generateValidated (for concurrent per-file gen). */
69
- async function generateValidatedAsync(generate, ext, gate) {
92
+ async function generateValidatedAsync(generate, ext, gate,
93
+ /** Why an earlier draft of the same file (from a batch) was refused. */
94
+ firstRefusal) {
70
95
  let content = '';
96
+ let refusal = firstRefusal;
71
97
  for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
72
98
  if (aiUnavailableReason())
73
99
  return null; // retrying a dead CLI only burns time
74
100
  try {
75
- content = await generate();
101
+ // A second try is told why the first was refused; asked blind, it made the same file again.
102
+ content = await generate(refusal);
76
103
  }
77
104
  catch (e) {
78
105
  noteAiFailure(e);
@@ -82,7 +109,8 @@ async function generateValidatedAsync(generate, ext, gate) {
82
109
  const gateMsg = gate ? gate(content) : null;
83
110
  if (isSyntaxValid(content, ext) && !gateMsg)
84
111
  return content;
85
- noteAiRejected(gateMsg || `not valid ${ext} (syntax)`);
112
+ refusal = gateMsg || syntaxReason(content, ext);
113
+ noteAiRejected(refusal);
86
114
  }
87
115
  return null;
88
116
  }
@@ -165,6 +193,7 @@ export async function runHarness(only) {
165
193
  }
166
194
  mkdirSync(dirname(info.path), { recursive: true });
167
195
  writeFileSync(info.path, stampHarness(kit), 'utf-8');
196
+ exportFind(info.path);
168
197
  writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
169
198
  writeHarnessConfigs(framework, areaDir, info.dir, area);
170
199
  written.push(info.path);
@@ -179,27 +208,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
179
208
  return [];
180
209
  }
181
210
  const config = loadConfig();
182
- // Verify is on unless the project or the caller turns it off. verifyFiles
183
- // itself reports (and skips) the cases where the app cannot be reached, so
184
- // "on" never means "fail the run because nothing is running".
185
- const verifyOn = opts.verify ?? config.verify ?? true;
186
- const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
187
211
  const ext = config.testExtension ?? '.test.ts';
188
212
  const seedDir = config.seedOutput ?? 'generated-tests/seed';
189
213
  const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
190
- // A browser that cannot start fails every verify, and each failure would be
191
- // sent for repair. Find out once, before any AI call is spent.
192
- if (useAI && verifyOn) {
193
- for (const area of Object.keys(config.testOutput ?? {})) {
194
- if (resolveFramework(config, area, area) !== 'playwright')
195
- continue;
196
- const problem = browserPreflight(packageRoot(testDirForType(config, area)));
197
- if (problem) {
198
- throw new Error(`Cannot verify "${area}" tests: ${problem}.\n` +
199
- 'Fix that and re-run, or pass --no-verify to write tests without running them.');
200
- }
201
- }
202
- }
203
214
  ensureStateDirIgnored();
204
215
  const cache = loadCache(CACHE_PATH);
205
216
  const results = [];
@@ -247,9 +258,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
247
258
  // makes one-by-one (`generate --journey X`) cheap: the first journey
248
259
  // in an area builds the testkit, every later journey reuses it — no
249
260
  // extra AI call (and no risk of that call eating the token limit).
250
- // Pass --force to rebuild a stale testkit (e.g. after upgrading).
251
- if (!force && existsSync(info.path)) {
261
+ // --force rewrites a journey's tests, not the testkit every journey in the
262
+ // area shares: a current testkit is kept even then. A stale one is rebuilt.
263
+ if (existsSync(info.path)) {
252
264
  if (harnessIsCurrent(readFileSync(info.path, 'utf-8'))) {
265
+ exportFind(info.path);
253
266
  harnessByArea.set(area, info.path);
254
267
  return info.path;
255
268
  }
@@ -265,6 +278,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
265
278
  }
266
279
  mkdirSync(dirname(info.path), { recursive: true });
267
280
  writeFileSync(info.path, stampHarness(kit), 'utf-8');
281
+ exportFind(info.path);
268
282
  writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
269
283
  // Config + globalSetup (playwright) / config + setup (jest) so the specs run
270
284
  // AND get their tokens minted once, plus the runner config scaffolding.
@@ -410,14 +424,39 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
410
424
  });
411
425
  // A freshly written file also may not hold placeholders; a repair is held to
412
426
  // the repair guard instead (it may not add a skip), so old files stay repairable.
427
+ // What the app can show: its source and the doc. A fresh test's names and
428
+ // labels are checked against it before the test is run.
429
+ let knownTexts;
430
+ const appTexts = () => framework === 'playwright'
431
+ ? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
432
+ : undefined;
413
433
  const genGate = (content, filePath) => testGate(content) ??
414
- checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath, maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 4 });
434
+ checkTestGates(content, ext, {
435
+ framework, forbidPlaceholders: true, filePath,
436
+ maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 2,
437
+ knownTexts: appTexts(),
438
+ });
415
439
  // Skip if unchanged — but NOT if the journey still has deferred stubs
416
440
  // to retry (a resume pass fills those in without touching real files).
441
+ // A suite whose tests leave rules out or cannot fail is not done either:
442
+ // it goes on, and only those tests are written again.
443
+ const suiteComplete = () => {
444
+ const plan = readPlan(journey);
445
+ if (!plan?.some((f) => f.rules?.length))
446
+ return true;
447
+ const kit = harnessInfo(areaDir, framework, ext).path;
448
+ const steps = existsSync(kit) ? harnessSteps(readFileSync(kit, 'utf-8')) : [];
449
+ return plan.every((f) => {
450
+ const p = join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`);
451
+ if (!f.rules?.length || !existsSync(p))
452
+ return true;
453
+ return fileIsComplete(readFileSync(p, 'utf-8'), f.rules, steps, appTexts());
454
+ });
455
+ };
417
456
  if (!force &&
418
457
  cache[file] === hash &&
419
- (!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
420
- !hasDeferredStubs(testDir, ext)) {
458
+ !hasDeferredStubs(testDir, ext) &&
459
+ suiteComplete()) {
421
460
  results.push({
422
461
  journeyName: journey.name,
423
462
  testDir,
@@ -430,6 +469,14 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
430
469
  }
431
470
  const warnings = [];
432
471
  const usageStart = aiUsage();
472
+ // A feature written up before it is built: its tests come from the doc
473
+ // alone. There is no code to read for facts and no app to run them on.
474
+ const notBuilt = /^status:\s*not built/im.test(markdown);
475
+ // Not live yet (not built, or behind a flag): every file is stamped, and
476
+ // `testforge run` holds it back until the doc's Status line goes.
477
+ const pending = pendingReason(markdown);
478
+ if (pending)
479
+ warnings.push(`${pending}: its files are held back by \`testforge run\` until the doc's Status line is removed`);
433
480
  const collision = collisionWarning(areaDir, journey.id, ext);
434
481
  if (collision)
435
482
  warnings.push(collision);
@@ -462,9 +509,97 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
462
509
  // Capture the PREVIOUS plan's slugs first so we can clean up any
463
510
  // files whose slug no longer exists (F6 replace-don't-duplicate).
464
511
  const prevSlugs = useAI ? readPlanSlugs(journey) : [];
512
+ // The doc read ONCE: rules, controls and setup, plus the testkit's exports.
513
+ // From here on no AI call gets the whole doc; each gets its own slice.
514
+ // A doc without tagged rules keeps the old whole-doc flow.
515
+ const parsed = useAI ? readRules(markdown) : undefined;
516
+ const tested = testedTags();
517
+ let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
518
+ const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
519
+ const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
520
+ // What each file really tests once written: the memory and the report come from this, not the plan.
521
+ const written = [];
522
+ const finish = async (content, pf, filePath) => {
523
+ if (!memory || !pf.rules?.length)
524
+ return content;
525
+ const { mem, facts } = await fileMemory(pf);
526
+ const done = await completeFile(content, {
527
+ journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
528
+ harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
529
+ check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
530
+ });
531
+ written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
532
+ for (const n of done.notes)
533
+ warnings.push(`${pf.slug}: ${n}`);
534
+ return done.content;
535
+ };
536
+ // A doc that changed since its suite was written: only what changed is
537
+ // touched. Renames and removals are edits without AI; reworded and added
538
+ // rules get one small call each file; new sections become new files below.
539
+ const prevRecord = memory ? loadMemory(journey) : undefined;
540
+ // Rule ids always follow the last written version, even for --force and
541
+ // --plan-only: an id that moved would point the tests' oracle comments at
542
+ // another rule. Only the in-place update of the tests is skipped then.
543
+ let changes;
544
+ let docDiff;
545
+ if (memory && prevRecord && prevRecord.hash !== hash) {
546
+ const { memory: aligned, diff } = alignRules(prevRecord.rules, memory, (prevRecord.removed ?? []).map((r) => r.id));
547
+ memory = aligned;
548
+ docDiff = diff;
549
+ changes = { renamed: diff.renamed.map((r) => r.id), changed: diff.changed, removed: diff.removed };
550
+ }
551
+ const prevPlan = docDiff && !force && !opts.planOnly ? readPlan(journey) : undefined;
552
+ if (memory && docDiff && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
553
+ const aligned = memory;
554
+ const diff = docDiff;
555
+ console.log(` ${journey.id}: doc changed — ${describeDiff(diff)}`);
556
+ const facts = notBuilt ? undefined : (await groundJourney(journey, hash, 'src', markdown, true)) ?? undefined;
557
+ const up = await updateJourney({
558
+ journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
559
+ pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
560
+ check: (c, p) => (isSyntaxValid(c, ext) ? genGate(c, p) : syntaxReason(c, ext)),
561
+ });
562
+ for (const n of up.notes)
563
+ console.log(` ${n}`);
564
+ warnings.push(...up.notes);
565
+ writePlan(journey, hash, up.plan);
566
+ }
465
567
  const planned = useAI
466
- ? loadOrPlan(journey, hash)
568
+ ? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
467
569
  : [{ slug: journey.id, title: journey.name, covers: '' }];
570
+ // A plan-only run leaves the memory as the tests were last written, so the
571
+ // next generate still sees what changed in the doc and updates those tests.
572
+ if (memory && !opts.planOnly) {
573
+ const choice = loadSourceChoice();
574
+ saveMemory(journey, hash, memory, helpers, {
575
+ doc: file, reference: choice?.journeys?.[file] ?? choice?.reference,
576
+ plan: planned, skipped: readPlanSkips(journey), prev: prevRecord, changes,
577
+ });
578
+ }
579
+ if (opts.planOnly) {
580
+ results.push({
581
+ journeyName: journey.name,
582
+ testDir,
583
+ testFiles: [],
584
+ seedPath,
585
+ status: 'skipped',
586
+ seedStatus: 'none',
587
+ warnings: [`plan only: ${planned.length} file(s) planned, nothing written — see ${join(journeyDir(journey), 'plan-check.md')}`],
588
+ });
589
+ continue;
590
+ }
591
+ // What one file is given: its rules with their context, the facts for
592
+ // its controls, the testkit list. A plan made before rules existed
593
+ // gives every file all the rules that get tests.
594
+ const allTested = memory ? memory.rules.filter((r) => r.tag && tested.includes(r.tag)).map((r) => r.id) : [];
595
+ const fileMemory = async (pf) => {
596
+ const facts = await grounding();
597
+ if (!memory)
598
+ return { facts };
599
+ const ids = pf.rules?.length ? pf.rules : allTested;
600
+ const slice = journeySlice(memory, ids);
601
+ return { mem: { journey: slice.text, helpers, rules: ids }, facts: factsSlice(facts, slice.controls) };
602
+ };
468
603
  const plannedSlugs = planned.map((p) => p.slug);
469
604
  // Read the journey's real code ONCE, before any file is written, and hand
470
605
  // the same facts to every file. Without this each file re-opened the same
@@ -475,7 +610,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
475
610
  // Only a file that has to be written or repaired needs it, so it is read
476
611
  // on first use: re-checking an existing suite costs no grounding at all.
477
612
  let groundingP;
478
- const grounding = () => (groundingP ??= useAI
613
+ const grounding = () => (groundingP ??= useAI && !notBuilt
479
614
  ? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
480
615
  : Promise.resolve(undefined));
481
616
  const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
@@ -488,6 +623,59 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
488
623
  // parallel agentic AI runs starve each other and time out, so a flaky
489
624
  // environment can lower `aiConcurrency` (e.g. 2) for reliability.
490
625
  const concurrency = Math.max(1, config.aiConcurrency ?? GEN_CONCURRENCY);
626
+ // Files to write, a few to one AI call: each call carries the shared rules
627
+ // once. A drafted file that fails its checks is written again on its own,
628
+ // told why; a file the batch left out is written on its own.
629
+ const pathOf = (pf) => join(testDir, `${flat ? `${journey.id}-${pf.slug}` : pf.slug}${ext}`);
630
+ const keeps = (p) => {
631
+ if ((force && !redone.has(i)) || !existsSync(p))
632
+ return false;
633
+ try {
634
+ const c = readFileSync(p, 'utf-8');
635
+ return isSyntaxValid(c, ext) && !isDeferredStub(c);
636
+ }
637
+ catch {
638
+ return false;
639
+ }
640
+ };
641
+ const drafts = new Map();
642
+ const perCall = Math.max(1, config.filesPerCall ?? 3);
643
+ if (useAI && memory && perCall > 1) {
644
+ const toWrite = planned.filter((pf) => !keeps(pathOf(pf)));
645
+ const batches = [];
646
+ // A few files to one call, but never so many rules that the writer drops
647
+ // some: 21 rules in one answer left 14 of them without a test.
648
+ const maxRules = 12;
649
+ for (const pf of toWrite) {
650
+ const last = batches[batches.length - 1];
651
+ const rules = (b) => b.reduce((n, f) => n + (f.rules?.length ?? 0), 0);
652
+ if (last && last.length < perCall && rules(last) + (pf.rules?.length ?? 0) <= maxRules)
653
+ last.push(pf);
654
+ else
655
+ batches.push([pf]);
656
+ }
657
+ const writeBatch = async (batch) => {
658
+ const files = await Promise.all(batch.map(async (pf) => {
659
+ const { mem, facts } = await fileMemory(pf);
660
+ return { path: pathOf(pf), prompt: buildPrompt(journey, '', pathOf(pf), { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem) };
661
+ }));
662
+ try {
663
+ const got = await writeFilesWithAI(files);
664
+ noteAiSuccess();
665
+ for (const [p, c] of got)
666
+ drafts.set(p, c);
667
+ }
668
+ catch (e) {
669
+ noteAiFailure(e);
670
+ }
671
+ };
672
+ // The first call alone writes the shared prompt to the cache; the rest
673
+ // then read it instead of each writing its own copy at the same moment.
674
+ const multi = batches.filter((b) => b.length > 1);
675
+ if (multi.length)
676
+ await writeBatch(multi[0]);
677
+ await pool(multi.slice(1), concurrency, writeBatch);
678
+ }
491
679
  const generated = await pool(planned, concurrency, async (pf) => {
492
680
  // Flat (jest): prefix the file with the journey so many journeys can
493
681
  // share one dir without clashing. Folder (playwright): the journey
@@ -498,7 +686,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
498
686
  // — don't spend tokens regenerating it. So a journey interrupted
499
687
  // midway resumes from exactly the files it hadn't written yet.
500
688
  // (--force ignores this and regenerates.)
501
- if (!force && existsSync(filePath)) {
689
+ // A redo after stubs fills in only the stubs: --force already rewrote the rest once this run.
690
+ if ((!force || redone.has(i)) && existsSync(filePath)) {
502
691
  try {
503
692
  const existing = readFileSync(filePath, 'utf-8');
504
693
  // Keep a real, parsing file — but a DEFERRED stub is not real
@@ -507,8 +696,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
507
696
  // A suite written before pending tests carried their reason gets
508
697
  // it now, deterministically — no regeneration, no AI.
509
698
  const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
510
- if (normalised !== existing)
511
- writeFileSync(filePath, normalised, 'utf-8');
699
+ // A kept file that leaves rules out or holds tests that cannot fail:
700
+ // only those tests are written again.
701
+ const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath) : normalised, pending);
702
+ if (finished !== existing)
703
+ writeFileSync(filePath, finished, 'utf-8');
512
704
  return { slug: pf.slug, path: filePath, status: 'ai' };
513
705
  }
514
706
  }
@@ -519,10 +711,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
519
711
  let content;
520
712
  let status;
521
713
  if (useAI) {
522
- const facts = await grounding();
523
- const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, (c) => genGate(c, filePath));
714
+ const draft = drafts.get(filePath);
715
+ const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
716
+ const { mem, facts } = await fileMemory(pf);
717
+ const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
524
718
  if (ai !== null) {
525
- content = framework === 'playwright' ? normalisePending(ai) : ai;
719
+ content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
526
720
  status = 'ai';
527
721
  }
528
722
  else {
@@ -541,6 +735,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
541
735
  if (!content.includes(GEN_MARKER)) {
542
736
  content = `// ${GEN_MARKER}\n${content}`;
543
737
  }
738
+ content = withPendingStamp(content, pending);
544
739
  // Write IMMEDIATELY (not batched at the end of the journey), so an
545
740
  // interruption preserves every file generated so far.
546
741
  // keepDeferredStubs:false (config) — do NOT write a DEFERRED stub file;
@@ -553,6 +748,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
553
748
  }
554
749
  return { slug: pf.slug, path: filePath, status };
555
750
  });
751
+ if (memory && written.length) {
752
+ const sum = recordWritten(journey, written);
753
+ console.log(` ${journey.id}: ${sum}`);
754
+ }
556
755
  const testFiles = generated.map((g) => ({
557
756
  slug: g.slug,
558
757
  path: g.path,
@@ -631,54 +830,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
631
830
  warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
632
831
  lint.remaining.slice(0, 5).join('; '));
633
832
  }
634
- // Run what we just wrote against the real application before calling it
635
- // done — file by file, then the whole journey together, then an independent
636
- // review of every claimed defect (see verify-journey.ts). Anything that
637
- // could not run, or still fails unexplained, leaves the journey UNVERIFIED
638
- // and uncached, so the next run checks it again.
639
- let verified;
640
- let reviewed = [];
641
- const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
642
- const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
643
- if (useAI && verifyOn && targets.length && missing.length) {
644
- // The suite cannot even start; running it would only feed repair a
645
- // failure no test edit can fix.
646
- verified = false;
647
- warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
648
- }
649
- else if (useAI && verifyOn && targets.length) {
650
- const result = await verifyJourney({
651
- config,
652
- journey,
653
- targets,
654
- attempts: verifyAttempts,
655
- repair: async (target, current, failure) => {
656
- const facts = await grounding();
657
- const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
658
- if (out === null)
659
- return null;
660
- const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
661
- return framework === 'playwright' ? normalisePending(stamped) : stamped;
662
- },
663
- });
664
- verified = result.verified;
665
- reviewed = result.reviewed;
666
- warnings.push(...result.warnings);
667
- }
668
- const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
833
+ const degraded = useAI && (status === 'basic' || seedStatus === 'failed');
669
834
  if (!degraded) {
670
835
  cache[file] = hash;
671
- // Remembered apart, so a journey cached before it was ever run is
672
- // re-checked (not regenerated) once verification is on.
673
- if (verified)
674
- cache[`verified:${file}`] = hash;
675
836
  }
676
837
  else {
677
838
  // Actively UN-mark a degraded journey (even under --force, which may
678
839
  // be re-processing a stale "done" from an earlier buggy run), so the
679
840
  // next `generate` re-does it instead of skipping a thin fallback.
680
841
  delete cache[file];
681
- delete cache[`verified:${file}`];
682
842
  }
683
843
  saveCache(CACHE_PATH, cache);
684
844
  resultRow.set(file, results.length);
@@ -696,9 +856,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
696
856
  seedPath,
697
857
  status,
698
858
  seedStatus,
699
- verified,
700
859
  warnings: warnings.length ? warnings : undefined,
701
- reviewed: reviewed.length ? reviewed : undefined,
702
860
  ai,
703
861
  });
704
862
  }