@activate-core/test-forge 1.3.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +51 -28
  2. package/dist/cli/index.js +39 -33
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +49 -2
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +49 -22
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +60 -3
  10. package/dist/core/docs/review.js +38 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +3 -3
  13. package/dist/core/gates/coverage.js +1 -16
  14. package/dist/core/gates/validate.js +171 -7
  15. package/dist/core/generate/ai-generator.js +202 -104
  16. package/dist/core/generate/ai-usage.js +116 -5
  17. package/dist/core/generate/bind.js +91 -0
  18. package/dist/core/generate/defect-review.js +59 -9
  19. package/dist/core/generate/doc-diff.js +139 -0
  20. package/dist/core/generate/harness.js +3 -31
  21. package/dist/core/generate/journey-memory.js +325 -0
  22. package/dist/core/generate/plan-cache.js +114 -7
  23. package/dist/core/generate/review-journey.js +3 -3
  24. package/dist/core/generate/run-generate.js +169 -78
  25. package/dist/core/generate/suite-planner.js +128 -23
  26. package/dist/core/generate/test-blocks.js +84 -0
  27. package/dist/core/generate/update-journey.js +115 -0
  28. package/dist/core/paths.js +90 -13
  29. package/dist/core/project/config.js +25 -2
  30. package/dist/core/project/env-template.js +2 -1
  31. package/dist/core/run/ci.js +1 -1
  32. package/dist/core/run/clean.js +1 -1
  33. package/dist/core/run/readiness.js +119 -0
  34. package/dist/core/run/run-tests.js +23 -1
  35. package/package.json +1 -1
  36. package/reporters/_shared.cjs +2 -2
  37. package/reporters/run-data.cjs +1 -1
  38. package/reporters/ui-helpers.cjs +78 -1
  39. package/dist/core/generate/verify-journey.js +0 -215
@@ -18,29 +18,49 @@
18
18
  */
19
19
  import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
20
20
  import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
21
- import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored } from '../paths.js';
21
+ import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored, journeyDir } from '../paths.js';
22
22
  import { lintGenerated } from '../gates/lint.js';
23
23
  import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
24
24
  import { writeCoverageManifest } from '../gates/coverage.js';
25
- import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
25
+ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlan, readPlanSkips, writePlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
26
26
  import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
- import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
30
- import { browserPreflight, missingPackages } from './verify.js';
31
- import { verifyJourney } from './verify-journey.js';
29
+ import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, } from './ai-generator.js';
32
30
  import { normalisePending } from './pending.js';
33
31
  import { packageRoot } from '../run/run-tests.js';
34
32
  import { groundJourney } from '../docs/grounding.js';
35
33
  import { generateSeed } from './seed-generator.js';
36
34
  import { dataAccess, writeRequiredData } from './required-data.js';
37
35
  import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
38
- import { isSyntaxValid, checkTestGates, duplicatedHelpers } from '../gates/validate.js';
36
+ import { testedTags } from './suite-planner.js';
37
+ import { factsSlice, helperApi, journeySlice, readRules, saveMemory, testedElsewhere } from './journey-memory.js';
38
+ import { loadSourceChoice } from '../docs/source-of-truth.js';
39
+ import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
40
+ import { updateJourney } from './update-journey.js';
41
+ import { sourceText } from '../docs/excerpts.js';
42
+ import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
39
43
  import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
40
44
  import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
41
45
  import { hashContent, loadCache, saveCache } from '../docs/cache.js';
42
46
  import { findMarkdownFiles } from '../find-files.js';
43
47
  import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
48
+ /**
49
+ * kit.find is a shared helper the testkit re-exports beside settledClick. A
50
+ * testkit written before it existed gets the one line here, with no AI and no
51
+ * rebuild.
52
+ */
53
+ export function exportFind(harnessPath) {
54
+ const src = readFileSync(harnessPath, 'utf-8');
55
+ if (/^\s*find\s*[:,]/m.test(src) || !/settledClick:\s*(\w+)\.settledClick,?/.test(src))
56
+ return;
57
+ writeFileSync(harnessPath, src.replace(/^(\s*)settledClick:\s*(\w+)\.settledClick,?/m, (m, pad, mod) => `${m.endsWith(',') ? m : `${m},`}\n${pad}find: ${mod}.find,`), 'utf-8');
58
+ }
59
+ /** Why a file is not valid code, as precisely as the check can say. */
60
+ function syntaxReason(content, ext) {
61
+ const where = /\.(c|m)?js$/i.test(ext) ? null : unbalancedAt(content);
62
+ return `not valid ${ext}${where ? `: ${where}` : ' (syntax)'} — the answer must be the whole file, code only`;
63
+ }
44
64
  /** How many times to re-ask the AI when output won't parse. */
45
65
  const MAX_GEN_ATTEMPTS = 2;
46
66
  /** How many focused files to generate concurrently within a journey. */
@@ -66,13 +86,17 @@ export async function pool(items, limit, fn) {
66
86
  return results;
67
87
  }
68
88
  /** Async version of generateValidated (for concurrent per-file gen). */
69
- async function generateValidatedAsync(generate, ext, gate) {
89
+ async function generateValidatedAsync(generate, ext, gate,
90
+ /** Why an earlier draft of the same file (from a batch) was refused. */
91
+ firstRefusal) {
70
92
  let content = '';
93
+ let refusal = firstRefusal;
71
94
  for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
72
95
  if (aiUnavailableReason())
73
96
  return null; // retrying a dead CLI only burns time
74
97
  try {
75
- content = await generate();
98
+ // A second try is told why the first was refused; asked blind, it made the same file again.
99
+ content = await generate(refusal);
76
100
  }
77
101
  catch (e) {
78
102
  noteAiFailure(e);
@@ -82,7 +106,8 @@ async function generateValidatedAsync(generate, ext, gate) {
82
106
  const gateMsg = gate ? gate(content) : null;
83
107
  if (isSyntaxValid(content, ext) && !gateMsg)
84
108
  return content;
85
- noteAiRejected(gateMsg || `not valid ${ext} (syntax)`);
109
+ refusal = gateMsg || syntaxReason(content, ext);
110
+ noteAiRejected(refusal);
86
111
  }
87
112
  return null;
88
113
  }
@@ -165,6 +190,7 @@ export async function runHarness(only) {
165
190
  }
166
191
  mkdirSync(dirname(info.path), { recursive: true });
167
192
  writeFileSync(info.path, stampHarness(kit), 'utf-8');
193
+ exportFind(info.path);
168
194
  writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
169
195
  writeHarnessConfigs(framework, areaDir, info.dir, area);
170
196
  written.push(info.path);
@@ -179,27 +205,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
179
205
  return [];
180
206
  }
181
207
  const config = loadConfig();
182
- // Verify is on unless the project or the caller turns it off. verifyFiles
183
- // itself reports (and skips) the cases where the app cannot be reached, so
184
- // "on" never means "fail the run because nothing is running".
185
- const verifyOn = opts.verify ?? config.verify ?? true;
186
- const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
187
208
  const ext = config.testExtension ?? '.test.ts';
188
209
  const seedDir = config.seedOutput ?? 'generated-tests/seed';
189
210
  const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
190
- // A browser that cannot start fails every verify, and each failure would be
191
- // sent for repair. Find out once, before any AI call is spent.
192
- if (useAI && verifyOn) {
193
- for (const area of Object.keys(config.testOutput ?? {})) {
194
- if (resolveFramework(config, area, area) !== 'playwright')
195
- continue;
196
- const problem = browserPreflight(packageRoot(testDirForType(config, area)));
197
- if (problem) {
198
- throw new Error(`Cannot verify "${area}" tests: ${problem}.\n` +
199
- 'Fix that and re-run, or pass --no-verify to write tests without running them.');
200
- }
201
- }
202
- }
203
211
  ensureStateDirIgnored();
204
212
  const cache = loadCache(CACHE_PATH);
205
213
  const results = [];
@@ -247,9 +255,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
247
255
  // makes one-by-one (`generate --journey X`) cheap: the first journey
248
256
  // in an area builds the testkit, every later journey reuses it — no
249
257
  // extra AI call (and no risk of that call eating the token limit).
250
- // Pass --force to rebuild a stale testkit (e.g. after upgrading).
251
- if (!force && existsSync(info.path)) {
258
+ // --force rewrites a journey's tests, not the testkit every journey in the
259
+ // area shares: a current testkit is kept even then. A stale one is rebuilt.
260
+ if (existsSync(info.path)) {
252
261
  if (harnessIsCurrent(readFileSync(info.path, 'utf-8'))) {
262
+ exportFind(info.path);
253
263
  harnessByArea.set(area, info.path);
254
264
  return info.path;
255
265
  }
@@ -265,6 +275,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
265
275
  }
266
276
  mkdirSync(dirname(info.path), { recursive: true });
267
277
  writeFileSync(info.path, stampHarness(kit), 'utf-8');
278
+ exportFind(info.path);
268
279
  writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
269
280
  // Config + globalSetup (playwright) / config + setup (jest) so the specs run
270
281
  // AND get their tokens minted once, plus the runner config scaffolding.
@@ -410,13 +421,21 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
410
421
  });
411
422
  // A freshly written file also may not hold placeholders; a repair is held to
412
423
  // the repair guard instead (it may not add a skip), so old files stay repairable.
424
+ // What the app can show: its source and the doc. A fresh test's names and
425
+ // labels are checked against it before the test is run.
426
+ let knownTexts;
413
427
  const genGate = (content, filePath) => testGate(content) ??
414
- checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath, maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 4 });
428
+ checkTestGates(content, ext, {
429
+ framework, forbidPlaceholders: true, filePath,
430
+ maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 2,
431
+ knownTexts: framework === 'playwright'
432
+ ? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
433
+ : undefined,
434
+ });
415
435
  // Skip if unchanged — but NOT if the journey still has deferred stubs
416
436
  // to retry (a resume pass fills those in without touching real files).
417
437
  if (!force &&
418
438
  cache[file] === hash &&
419
- (!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
420
439
  !hasDeferredStubs(testDir, ext)) {
421
440
  results.push({
422
441
  journeyName: journey.name,
@@ -430,6 +449,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
430
449
  }
431
450
  const warnings = [];
432
451
  const usageStart = aiUsage();
452
+ // A feature written up before it is built: its tests come from the doc
453
+ // alone. There is no code to read for facts and no app to run them on.
454
+ const notBuilt = /^status:\s*not built/im.test(markdown);
455
+ if (notBuilt)
456
+ warnings.push('written before the feature exists (Status: not built yet) — run these tests once it ships');
433
457
  const collision = collisionWarning(areaDir, journey.id, ext);
434
458
  if (collision)
435
459
  warnings.push(collision);
@@ -462,9 +486,70 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
462
486
  // Capture the PREVIOUS plan's slugs first so we can clean up any
463
487
  // files whose slug no longer exists (F6 replace-don't-duplicate).
464
488
  const prevSlugs = useAI ? readPlanSlugs(journey) : [];
489
+ // The doc read ONCE: rules, controls and setup, plus the testkit's exports.
490
+ // From here on no AI call gets the whole doc; each gets its own slice.
491
+ // A doc without tagged rules keeps the old whole-doc flow.
492
+ const parsed = useAI ? readRules(markdown) : undefined;
493
+ const tested = testedTags();
494
+ let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
495
+ const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
496
+ // A doc that changed since its suite was written: only what changed is
497
+ // touched. Renames and removals are edits without AI; reworded and added
498
+ // rules get one small call each file; new sections become new files below.
499
+ const prevRecord = memory ? loadMemory(journey) : undefined;
500
+ const prevMemory = !force && !opts.planOnly ? prevRecord : undefined;
501
+ const prevPlan = prevMemory && prevMemory.hash !== hash ? readPlan(journey) : undefined;
502
+ let changes;
503
+ if (memory && prevMemory && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
504
+ const { memory: aligned, diff } = alignRules(prevMemory.rules, memory);
505
+ memory = aligned;
506
+ changes = { renamed: diff.renamed.map((r) => r.id), changed: diff.changed, removed: diff.removed };
507
+ console.log(` ${journey.id}: doc changed — ${describeDiff(diff)}`);
508
+ const facts = notBuilt ? undefined : (await groundJourney(journey, hash, 'src', markdown, true)) ?? undefined;
509
+ const up = await updateJourney({
510
+ journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
511
+ pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
512
+ check: (c, p) => (isSyntaxValid(c, ext) ? genGate(c, p) : syntaxReason(c, ext)),
513
+ });
514
+ for (const n of up.notes)
515
+ console.log(` ${n}`);
516
+ warnings.push(...up.notes);
517
+ writePlan(journey, hash, up.plan);
518
+ }
465
519
  const planned = useAI
466
- ? loadOrPlan(journey, hash)
520
+ ? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
467
521
  : [{ slug: journey.id, title: journey.name, covers: '' }];
522
+ if (memory) {
523
+ const choice = loadSourceChoice();
524
+ saveMemory(journey, hash, memory, helpers, {
525
+ doc: file, reference: choice?.journeys?.[file] ?? choice?.reference,
526
+ plan: planned, skipped: readPlanSkips(journey), prev: prevRecord, changes,
527
+ });
528
+ }
529
+ if (opts.planOnly) {
530
+ results.push({
531
+ journeyName: journey.name,
532
+ testDir,
533
+ testFiles: [],
534
+ seedPath,
535
+ status: 'skipped',
536
+ seedStatus: 'none',
537
+ warnings: [`plan only: ${planned.length} file(s) planned, nothing written — see ${join(journeyDir(journey), 'plan-check.md')}`],
538
+ });
539
+ continue;
540
+ }
541
+ // What one file is given: its rules with their context, the facts for
542
+ // its controls, the testkit list. A plan made before rules existed
543
+ // gives every file all the rules that get tests.
544
+ const allTested = memory ? memory.rules.filter((r) => r.tag && tested.includes(r.tag)).map((r) => r.id) : [];
545
+ const fileMemory = async (pf) => {
546
+ const facts = await grounding();
547
+ if (!memory)
548
+ return { facts };
549
+ const ids = pf.rules?.length ? pf.rules : allTested;
550
+ const slice = journeySlice(memory, ids);
551
+ return { mem: { journey: slice.text, helpers, rules: ids }, facts: factsSlice(facts, slice.controls) };
552
+ };
468
553
  const plannedSlugs = planned.map((p) => p.slug);
469
554
  // Read the journey's real code ONCE, before any file is written, and hand
470
555
  // the same facts to every file. Without this each file re-opened the same
@@ -475,7 +560,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
475
560
  // Only a file that has to be written or repaired needs it, so it is read
476
561
  // on first use: re-checking an existing suite costs no grounding at all.
477
562
  let groundingP;
478
- const grounding = () => (groundingP ??= useAI
563
+ const grounding = () => (groundingP ??= useAI && !notBuilt
479
564
  ? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
480
565
  : Promise.resolve(undefined));
481
566
  const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
@@ -488,6 +573,50 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
488
573
  // parallel agentic AI runs starve each other and time out, so a flaky
489
574
  // environment can lower `aiConcurrency` (e.g. 2) for reliability.
490
575
  const concurrency = Math.max(1, config.aiConcurrency ?? GEN_CONCURRENCY);
576
+ // Files to write, a few to one AI call: each call carries the shared rules
577
+ // once. A drafted file that fails its checks is written again on its own,
578
+ // told why; a file the batch left out is written on its own.
579
+ const pathOf = (pf) => join(testDir, `${flat ? `${journey.id}-${pf.slug}` : pf.slug}${ext}`);
580
+ const keeps = (p) => {
581
+ if ((force && !redone.has(i)) || !existsSync(p))
582
+ return false;
583
+ try {
584
+ const c = readFileSync(p, 'utf-8');
585
+ return isSyntaxValid(c, ext) && !isDeferredStub(c);
586
+ }
587
+ catch {
588
+ return false;
589
+ }
590
+ };
591
+ const drafts = new Map();
592
+ const perCall = Math.max(1, config.filesPerCall ?? 3);
593
+ if (useAI && memory && perCall > 1) {
594
+ const toWrite = planned.filter((pf) => !keeps(pathOf(pf)));
595
+ const batches = [];
596
+ for (let k = 0; k < toWrite.length; k += perCall)
597
+ batches.push(toWrite.slice(k, k + perCall));
598
+ const writeBatch = async (batch) => {
599
+ const files = await Promise.all(batch.map(async (pf) => {
600
+ const { mem, facts } = await fileMemory(pf);
601
+ return { path: pathOf(pf), prompt: buildPrompt(journey, '', pathOf(pf), { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem) };
602
+ }));
603
+ try {
604
+ const got = await writeFilesWithAI(files);
605
+ noteAiSuccess();
606
+ for (const [p, c] of got)
607
+ drafts.set(p, c);
608
+ }
609
+ catch (e) {
610
+ noteAiFailure(e);
611
+ }
612
+ };
613
+ // The first call alone writes the shared prompt to the cache; the rest
614
+ // then read it instead of each writing its own copy at the same moment.
615
+ const multi = batches.filter((b) => b.length > 1);
616
+ if (multi.length)
617
+ await writeBatch(multi[0]);
618
+ await pool(multi.slice(1), concurrency, writeBatch);
619
+ }
491
620
  const generated = await pool(planned, concurrency, async (pf) => {
492
621
  // Flat (jest): prefix the file with the journey so many journeys can
493
622
  // share one dir without clashing. Folder (playwright): the journey
@@ -498,7 +627,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
498
627
  // — don't spend tokens regenerating it. So a journey interrupted
499
628
  // midway resumes from exactly the files it hadn't written yet.
500
629
  // (--force ignores this and regenerates.)
501
- if (!force && existsSync(filePath)) {
630
+ // A redo after stubs fills in only the stubs: --force already rewrote the rest once this run.
631
+ if ((!force || redone.has(i)) && existsSync(filePath)) {
502
632
  try {
503
633
  const existing = readFileSync(filePath, 'utf-8');
504
634
  // Keep a real, parsing file — but a DEFERRED stub is not real
@@ -519,8 +649,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
519
649
  let content;
520
650
  let status;
521
651
  if (useAI) {
522
- const facts = await grounding();
523
- const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, (c) => genGate(c, filePath));
652
+ const draft = drafts.get(filePath);
653
+ const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
654
+ const { mem, facts } = await fileMemory(pf);
655
+ const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
524
656
  if (ai !== null) {
525
657
  content = framework === 'playwright' ? normalisePending(ai) : ai;
526
658
  status = 'ai';
@@ -631,54 +763,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
631
763
  warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
632
764
  lint.remaining.slice(0, 5).join('; '));
633
765
  }
634
- // Run what we just wrote against the real application before calling it
635
- // done — file by file, then the whole journey together, then an independent
636
- // review of every claimed defect (see verify-journey.ts). Anything that
637
- // could not run, or still fails unexplained, leaves the journey UNVERIFIED
638
- // and uncached, so the next run checks it again.
639
- let verified;
640
- let reviewed = [];
641
- const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
642
- const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
643
- if (useAI && verifyOn && targets.length && missing.length) {
644
- // The suite cannot even start; running it would only feed repair a
645
- // failure no test edit can fix.
646
- verified = false;
647
- warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
648
- }
649
- else if (useAI && verifyOn && targets.length) {
650
- const result = await verifyJourney({
651
- config,
652
- journey,
653
- targets,
654
- attempts: verifyAttempts,
655
- repair: async (target, current, failure) => {
656
- const facts = await grounding();
657
- const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
658
- if (out === null)
659
- return null;
660
- const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
661
- return framework === 'playwright' ? normalisePending(stamped) : stamped;
662
- },
663
- });
664
- verified = result.verified;
665
- reviewed = result.reviewed;
666
- warnings.push(...result.warnings);
667
- }
668
- const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
766
+ const degraded = useAI && (status === 'basic' || seedStatus === 'failed');
669
767
  if (!degraded) {
670
768
  cache[file] = hash;
671
- // Remembered apart, so a journey cached before it was ever run is
672
- // re-checked (not regenerated) once verification is on.
673
- if (verified)
674
- cache[`verified:${file}`] = hash;
675
769
  }
676
770
  else {
677
771
  // Actively UN-mark a degraded journey (even under --force, which may
678
772
  // be re-processing a stale "done" from an earlier buggy run), so the
679
773
  // next `generate` re-does it instead of skipping a thin fallback.
680
774
  delete cache[file];
681
- delete cache[`verified:${file}`];
682
775
  }
683
776
  saveCache(CACHE_PATH, cache);
684
777
  resultRow.set(file, results.length);
@@ -696,9 +789,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
696
789
  seedPath,
697
790
  status,
698
791
  seedStatus,
699
- verified,
700
792
  warnings: warnings.length ? warnings : undefined,
701
- reviewed: reviewed.length ? reviewed : undefined,
702
793
  ai,
703
794
  });
704
795
  }
@@ -16,8 +16,9 @@
16
16
  */
17
17
  import { readFileSync, existsSync, readdirSync } from 'node:fs';
18
18
  import { join } from 'node:path';
19
- import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
20
- import { PLANS_DIR } from '../paths.js';
19
+ import { loadConfig, splitCommand, withNoTools, withReadOnlyTools } from '../project/config.js';
20
+ import { ruleIndex } from './journey-memory.js';
21
+ import { MEMORY_DIR } from '../paths.js';
21
22
  import { runAiSync } from './ai-usage.js';
22
23
  const MAP_PATH = 'testforge.md';
23
24
  /**
@@ -30,13 +31,12 @@ const MAP_PATH = 'testforge.md';
30
31
  export function siblingJourneys(journey, docsDir = 'docs') {
31
32
  const lines = [];
32
33
  const planned = new Set();
33
- const dir = join(PLANS_DIR, journey.area);
34
- for (const f of existsSync(dir) ? readdirSync(dir) : []) {
35
- const id = f.replace(/\.json$/, '');
36
- if (!f.endsWith('.json') || id === journey.id)
34
+ const dir = join(MEMORY_DIR, journey.area);
35
+ for (const id of existsSync(dir) ? readdirSync(dir) : []) {
36
+ if (id === journey.id || !existsSync(join(dir, id, 'plan.json')))
37
37
  continue;
38
38
  try {
39
- const plan = JSON.parse(readFileSync(join(dir, f), 'utf-8'));
39
+ const plan = JSON.parse(readFileSync(join(dir, id, 'plan.json'), 'utf-8'));
40
40
  lines.push(`- ${id} (planned):`, ...(plan.files ?? []).map((x) => ` - ${x.title ?? ''}: ${x.covers ?? ''}`));
41
41
  planned.add(id);
42
42
  }
@@ -56,7 +56,7 @@ export function siblingJourneys(journey, docsDir = 'docs') {
56
56
  return lines.join('\n');
57
57
  }
58
58
  /** Build the planning prompt for one journey. */
59
- export function buildPlanPrompt(journey, map, siblings = '') {
59
+ export function buildPlanPrompt(journey, map, siblings = '', memory) {
60
60
  return [
61
61
  `You are a senior test architect. Plan how to split ONE user journey`,
62
62
  `into a set of FOCUSED test files — one concern per file — the way a`,
@@ -64,11 +64,19 @@ export function buildPlanPrompt(journey, map, siblings = '') {
64
64
  `success, failures, discount, 3D-Secure, zero-amount, price summary,`,
65
65
  `and so on). Do NOT plan a single monolithic file.`,
66
66
  ``,
67
- `You are inside the project's repository with read-only tools (Read,`,
68
- `Grep, Glob, LS). READ the real source for this journey — routes,`,
69
- `controllers, services, models — and combine it with the journey doc`,
70
- `to decide the split. Every file must map to a real behaviour that`,
71
- `exists in the code or is explicitly described in the doc.`,
67
+ ...(memory
68
+ ? [
69
+ `You get the journey as its list of RULES (one line each, with an id)`,
70
+ `and its controls, not the whole document, and you have no tools: plan`,
71
+ `from this list. Every file names the rule ids it tests.`,
72
+ ]
73
+ : [
74
+ `You are inside the project's repository with read-only tools (Read,`,
75
+ `Grep, Glob, LS). READ the real source for this journey — routes,`,
76
+ `controllers, services, models — and combine it with the journey doc`,
77
+ `to decide the split. Every file must map to a real behaviour that`,
78
+ `exists in the code or is explicitly described in the doc.`,
79
+ ]),
72
80
  ``,
73
81
  `Rules:`,
74
82
  `- Let the doc's sections and the real endpoints drive the split.`,
@@ -103,7 +111,18 @@ export function buildPlanPrompt(journey, map, siblings = '') {
103
111
  : []),
104
112
  `Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
105
113
  ` [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,
106
- ` "covers": "<one or two sentences: exactly what this file tests>" }]`,
114
+ ` "covers": "<one or two sentences: exactly what this file tests>"${memory ? ',\n "rules": ["R1", "R4"]' : ''} }]`,
115
+ ...(memory
116
+ ? [
117
+ `Every rule id below goes in exactly ONE file's "rules". A rule that`,
118
+ `another journey owns, or that cannot be checked from this journey, is`,
119
+ `left out and listed instead as { "skip": "R7", "why": "<short reason>" }`,
120
+ `in the same array. A rule that is neither planned nor skipped is`,
121
+ `reported as missing. Every [spec] rule and every rule marked "(must"`,
122
+ `(security, money and data, concurrency, error boundaries) is planned`,
123
+ `unless another journey owns it.`,
124
+ ]
125
+ : []),
107
126
  `slug: lowercase kebab-case, no path, no extension — and SHORT: name the`,
108
127
  ` one concern in 2–4 words (4 max), the way a human names a file. Do NOT`,
109
128
  ` restate the journey name and do NOT pile on qualifiers.`,
@@ -113,13 +132,25 @@ export function buildPlanPrompt(journey, map, siblings = '') {
113
132
  `title: human-readable and specific, e.g. "Payment — Amount validation".`,
114
133
  `The very first character of your response must be "[".`,
115
134
  ``,
116
- `--- PROJECT MAP (data layer) ---`,
117
- map,
118
- ``,
119
- `--- JOURNEY ---`,
120
- journey.rawMarkdown,
135
+ ...(memory
136
+ ? [
137
+ `--- JOURNEY: ${journey.name} ---`,
138
+ memory.header,
139
+ ``,
140
+ `CONTROLS:`,
141
+ ...Object.entries(memory.controls).map(([id, c]) => `${id}: ${c}`),
142
+ ``,
143
+ `RULES TO PLAN:`,
144
+ ruleIndex(memory, testedTags()),
145
+ ]
146
+ : [`--- PROJECT MAP (data layer) ---`, map, ``, `--- JOURNEY ---`, journey.rawMarkdown]),
121
147
  ].join('\n');
122
148
  }
149
+ /** Which rule tags get tests of their own; the rest only explain the app. */
150
+ export function testedTags() {
151
+ const t = loadConfig().testTags;
152
+ return Array.isArray(t) && t.length ? t : ['spec', 'baseline'];
153
+ }
123
154
  /**
124
155
  * Pull a JSON array out of the AI's reply, tolerating stray prose or
125
156
  * code fences around it. Returns [] if nothing parseable is found.
@@ -154,10 +185,11 @@ function safeSlug(raw) {
154
185
  * produces nothing usable, so generation never silently yields zero
155
186
  * files.
156
187
  */
157
- export function planSuite(journey) {
158
- const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
159
- const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey));
160
- const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
188
+ export function planSuite(journey, memory, skipped = []) {
189
+ const map = memory || !existsSync(MAP_PATH) ? '' : readFileSync(MAP_PATH, 'utf-8');
190
+ const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey), memory);
191
+ const cmd = splitCommand(loadConfig().aiCommand);
192
+ const { program, args } = memory ? withNoTools(cmd) : withReadOnlyTools(cmd);
161
193
  let raw = '';
162
194
  try {
163
195
  raw = runAiSync(program, args, {
@@ -175,6 +207,10 @@ export function planSuite(journey) {
175
207
  if (!entry || typeof entry !== 'object')
176
208
  continue;
177
209
  const e = entry;
210
+ if (typeof e.skip === 'string') {
211
+ skipped.push({ rule: e.skip, why: String(e.why ?? '').trim() });
212
+ continue;
213
+ }
178
214
  const slug = safeSlug(e.slug ?? e.title);
179
215
  if (!slug || seen.has(slug))
180
216
  continue;
@@ -183,6 +219,7 @@ export function planSuite(journey) {
183
219
  slug,
184
220
  title: String(e.title ?? journey.name).trim() || journey.name,
185
221
  covers: String(e.covers ?? '').trim(),
222
+ ...(memory && Array.isArray(e.rules) ? { rules: e.rules.map(String) } : {}),
186
223
  });
187
224
  }
188
225
  if (files.length === 0) {
@@ -196,3 +233,71 @@ export function planSuite(journey) {
196
233
  }
197
234
  return files;
198
235
  }
236
+ /** Sections larger than this are split by their sub-sections, then in chunks. */
237
+ const MAX_RULES_PER_FILE = 10;
238
+ /**
239
+ * The plan without the AI: the doc is already organised one concern per
240
+ * numbered section, so each section becomes one file and gets that section's
241
+ * rules. A section with more rules than one file should hold is split by its
242
+ * sub-sections, then in chunks. Rules another journey already tests come in
243
+ * as skips. Costs nothing.
244
+ */
245
+ export function planBySections(journey, memory, skipped = []) {
246
+ const tags = testedTags();
247
+ const skip = new Set(skipped.map((x) => x.rule));
248
+ const rules = memory.rules.filter((r) => r.tag && tags.includes(r.tag) && !skip.has(r.id));
249
+ const byKey = (key, rs) => {
250
+ const m = new Map();
251
+ for (const r of rs)
252
+ m.set(key(r), [...(m.get(key(r)) ?? []), r]);
253
+ return m;
254
+ };
255
+ const files = [];
256
+ const used = new Set();
257
+ const STOP = new Set(['a', 'an', 'the', 'on', 'of', 'and', 'or', 'for', 'to', 'in', 'with', 'from', 'at', 'by', 'is', 'its']);
258
+ const slugFor = (title) => {
259
+ const part = / part (\d+)$/.exec(title)?.[1];
260
+ const words = title.replace(/^§[\d.]+\s*/, '').replace(/ part \d+$/, '').toLowerCase().split(/[^a-z0-9]+/).filter((w) => w && !STOP.has(w));
261
+ const base = part ? `${words.slice(0, 3).join('-')}-${part}` : safeSlug(words.join('-')) || 'rules';
262
+ let slug = base;
263
+ for (let n = 2; used.has(slug); n++)
264
+ slug = `${base.split('-').slice(0, 3).join('-')}-${n}`;
265
+ used.add(slug);
266
+ return slug;
267
+ };
268
+ const push = (title, rs) => {
269
+ const name = title.replace(/^§[\d.]+\s*/, '');
270
+ files.push({ slug: slugFor(title), title: `${journey.name} — ${name}`, covers: `The ${rs.length} rule(s) of ${title}.`, rules: rs.map((r) => r.id) });
271
+ };
272
+ for (const [group, rs] of byKey((r) => r.group || r.section, rules)) {
273
+ if (rs.length <= MAX_RULES_PER_FILE) {
274
+ push(group, rs);
275
+ continue;
276
+ }
277
+ // Too many for one file: its sub-sections, small ones kept together, big ones in parts.
278
+ let titles = [];
279
+ let acc = [];
280
+ const flush = () => {
281
+ if (acc.length)
282
+ push(titles.join(' & '), acc);
283
+ titles = [];
284
+ acc = [];
285
+ };
286
+ for (const [section, part] of byKey((r) => r.section, rs)) {
287
+ if (part.length > MAX_RULES_PER_FILE) {
288
+ flush();
289
+ const parts = Math.ceil(part.length / MAX_RULES_PER_FILE);
290
+ const size = Math.ceil(part.length / parts);
291
+ for (let i = 0; i < part.length; i += size)
292
+ push(`${section} part ${i / size + 1}`, part.slice(i, i + size));
293
+ continue;
294
+ }
295
+ if (acc.length + part.length > MAX_RULES_PER_FILE)
296
+ flush();
297
+ titles.push(section.replace(/^§[\d.]+\s*/, ''));
298
+ acc.push(...part);
299
+ }
300
+ flush();
301
+ }
302
+ return files;
303
+ }