@activate-core/test-forge 2.6.0 → 2.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -26,12 +26,46 @@ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, is
26
26
  import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
- import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
30
- import { completeFile } from './complete-file.js';
31
- import { tuckStrayTests } from './test-blocks.js';
29
+ import { rewriteSetupWithAI, buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
30
+ import { completeFile, withoutBlankNames, withoutNames } from './complete-file.js';
31
+ import { findSetupBlocks, findTestBlocks, tuckStrayTests, useInsertIds } from './test-blocks.js';
32
32
  import { passedTitles } from './passed.js';
33
+ import { scanNames } from '../run/name-scan.js';
33
34
  import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
34
35
  /** Names the file asks for that testforge/interface.json still has blank. */
36
+ /**
37
+ * Is a blank name used outside every test (a hook, a helper, the file's
38
+ * setup)? Then the whole file waits on it, and only writing the whole file
39
+ * again — with the code facts — clears it; no test of it can have passed.
40
+ */
41
+ export function blankUsedOutsideTests(content) {
42
+ const blocks = findTestBlocks(content);
43
+ const outside = blocks.reduceRight((c, b) => c.slice(0, b.start) + c.slice(b.end), content);
44
+ return blankNames(content).some((n) => {
45
+ const esc = n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
46
+ const v = new RegExp(`\\b(?:const|let|var)\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*need\\(\\s*(['"\`])${esc}\\2`).exec(content)?.[1];
47
+ if (!v)
48
+ return false;
49
+ const uses = outside.match(new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`, 'g')) ?? [];
50
+ return uses.length > 1; // more than its own declaration
51
+ });
52
+ }
53
+ /**
54
+ * Where a file's blank names are used outside its tests: 'setup' when every
55
+ * such use sits in a hook or helper TestForge can rewrite on its own, 'file'
56
+ * when one sits somewhere it cannot split out (a parametrised table, a test
57
+ * built in a loop), undefined when none.
58
+ */
59
+ export function blankOutsideKind(content) {
60
+ if (!blankUsedOutsideTests(content))
61
+ return undefined;
62
+ const vars = [...content.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]+)\2/g)]
63
+ .filter((m) => blankNames(`need('${m[3]}')`).length > 0).map((m) => m[1]);
64
+ const covered = [...findTestBlocks(content), ...findSetupBlocks(content)];
65
+ const rest = covered.sort((a, b) => b.start - a.start).reduce((c, b) => c.slice(0, b.start) + c.slice(b.end), content);
66
+ const used = vars.some((v) => (rest.match(new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`, 'g')) ?? []).length > 1);
67
+ return used ? 'file' : 'setup';
68
+ }
35
69
  /** Does this test use a name the product has not given yet (directly, or through the constant it was put in)? */
36
70
  function usesBlankName(content, block) {
37
71
  const text = content.slice(block.start, block.end);
@@ -43,6 +77,20 @@ function usesBlankName(content, block) {
43
77
  return !!v && new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(text);
44
78
  });
45
79
  }
80
+ /** The names a testkit exports (module.exports = { … } or export …). */
81
+ export function exportNames(src) {
82
+ const names = new Set();
83
+ const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(src);
84
+ if (cjs)
85
+ for (const n of cjs[1].split(',')) {
86
+ const m = /^\s*(\w+)/.exec(n);
87
+ if (m)
88
+ names.add(m[1]);
89
+ }
90
+ for (const m of src.matchAll(/^export\s+(?:async\s+)?(?:function\*?|const|let|class)\s+(\w+)/gm))
91
+ names.add(m[1]);
92
+ return [...names];
93
+ }
46
94
  function blankNames(content) {
47
95
  const iface = readInterface();
48
96
  return [...content.matchAll(/\bneed\(\s*(['"`])([^'"`]+)\1\s*\)/g)].map((m) => m[2]).filter((n) => {
@@ -54,7 +102,7 @@ import { harnessSteps, ruleCoverage } from '../gates/test-quality.js';
54
102
  import { normalisePending } from './pending.js';
55
103
  import { packageRoot } from '../run/run-tests.js';
56
104
  import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
57
- import { factsCodeMoved, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
105
+ import { factsCodeMoved, journeyCodeHash, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
58
106
  import { generateSeed, seedProblem } from './seed-generator.js';
59
107
  import { dataAccess, writeRequiredData } from './required-data.js';
60
108
  import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
@@ -64,7 +112,7 @@ import { loadSourceChoice } from '../docs/source-of-truth.js';
64
112
  import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
65
113
  import { updateJourney } from './update-journey.js';
66
114
  import { sourceRoot, sourceText } from '../docs/excerpts.js';
67
- import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
115
+ import { isSyntaxValid, checkTestGates, trimExtraTests, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
68
116
  import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
69
117
  import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
70
118
  import { hashContent, loadCache, saveCache } from '../docs/cache.js';
@@ -113,7 +161,9 @@ export async function pool(items, limit, fn) {
113
161
  /** Async version of generateValidated (for concurrent per-file gen). */
114
162
  async function generateValidatedAsync(generate, ext, gate,
115
163
  /** Why an earlier draft of the same file (from a batch) was refused, with that draft. */
116
- firstRefusal) {
164
+ firstRefusal,
165
+ /** Mechanical fixes applied before the gate: what code can fix costs no retry. */
166
+ fix = (c) => c) {
117
167
  let content = '';
118
168
  let refusal = firstRefusal;
119
169
  for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
@@ -121,7 +171,7 @@ firstRefusal) {
121
171
  return null; // retrying a dead CLI only burns time
122
172
  try {
123
173
  // A second try sees what it wrote and why that was refused; asked blind, it made the same file again.
124
- content = await generate(refusal);
174
+ content = fix(await generate(refusal));
125
175
  }
126
176
  catch (e) {
127
177
  noteAiFailure(e);
@@ -294,7 +344,17 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
294
344
  // command. Test files are untouched.
295
345
  console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
296
346
  }
347
+ const old = existsSync(info.path) ? readFileSync(info.path, 'utf-8') : '';
297
348
  const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
349
+ // The tests already written call the old testkit: a rebuild that fails, or
350
+ // drops something they use, is not taken — the old one stays.
351
+ const lost = old && kit !== null ? exportNames(old).filter((n) => !exportNames(kit).includes(n)) : [];
352
+ if (old && (kit === null || lost.length)) {
353
+ console.log(` kept the existing ${area} test harness: the rebuild ${kit === null ? 'failed' : `dropped ${lost.join(', ')}`}`);
354
+ exportFind(info.path);
355
+ harnessByArea.set(area, info.path);
356
+ return info.path;
357
+ }
298
358
  if (kit === null) {
299
359
  harnessByArea.set(area, null);
300
360
  return undefined;
@@ -453,6 +513,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
453
513
  const appTexts = () => framework === 'playwright'
454
514
  ? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
455
515
  : undefined;
516
+ const fixTests = (content) => trimExtraTests(content, config.maxTestsPerRule ?? 2);
456
517
  const genGate = (content, filePath) => testGate(content) ??
457
518
  checkTestGates(content, ext, {
458
519
  framework, forbidPlaceholders: true, filePath,
@@ -463,7 +524,61 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
463
524
  // to retry (a resume pass fills those in without touching real files).
464
525
  // A suite whose tests leave rules out or cannot fail is not done either:
465
526
  // it goes on, and only those tests are written again.
527
+ // Rules asked for and refused before, with this doc and this code: not asked again until one moves.
528
+ const refusedPath = join(journeyDir(journey), 'refused.json');
529
+ let refusedKeyMemo;
530
+ const refusedKey = () => (refusedKeyMemo ??= `${hash}:${journeyCodeHash(journey, sourceRoot())}`);
531
+ const readRefused = () => {
532
+ try {
533
+ const r = JSON.parse(readFileSync(refusedPath, 'utf-8'));
534
+ return r.key === refusedKey() ? (r.rules ?? {}) : {};
535
+ }
536
+ catch {
537
+ return {};
538
+ }
539
+ };
540
+ const rememberRefused = (slug, ids) => {
541
+ try {
542
+ const all = readRefused();
543
+ all[slug] = [...new Set([...(all[slug] ?? []), ...ids])];
544
+ mkdirSync(dirname(refusedPath), { recursive: true });
545
+ writeFileSync(refusedPath, JSON.stringify({ key: refusedKey(), rules: all }, null, 2), 'utf-8');
546
+ }
547
+ catch { /* best effort */ }
548
+ };
549
+ /** The file with the setup blocks that use a blank name written again, or null when that did not work. */
550
+ const redoSetup = async (content, filePath, facts, kitApi) => {
551
+ const isBlank = (n) => blankNames(`need('${n}')`).length > 0;
552
+ const blankVars = [...content.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]+)\2/g)]
553
+ .filter((m) => isBlank(m[3])).map((m) => m[1]);
554
+ const uses = (t) => blankVars.some((v) => new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(t));
555
+ const blocks = findSetupBlocks(content).filter((b) => uses(b.text));
556
+ if (!blocks.length)
557
+ return null;
558
+ const top = withoutBlankNames(content.slice(0, Math.min(...blocks.map((b) => b.start))), isBlank).slice(-6000);
559
+ let got = null;
560
+ try {
561
+ got = await rewriteSetupWithAI({ filePath, top, blocks, facts, helpers: kitApi });
562
+ noteAiSuccess();
563
+ }
564
+ catch (e) {
565
+ noteAiFailure(e);
566
+ return null;
567
+ }
568
+ if (!got)
569
+ return null;
570
+ // Each block is still what it was (the same hook or helper), and asks for nothing.
571
+ if (got.some((g, i) => !g.includes(blocks[i].name) || /\bneed\(/.test(g) || uses(g)))
572
+ return null;
573
+ let out = content;
574
+ for (let i = blocks.length - 1; i >= 0; i--)
575
+ out = out.slice(0, blocks[i].start) + got[i] + out.slice(blocks[i].end);
576
+ out = withoutNames(out);
577
+ const why = isSyntaxValid(out, ext) ? genGate(out, filePath) : syntaxReason(out, ext);
578
+ return why ? null : out;
579
+ };
466
580
  const suiteComplete = () => {
581
+ const refused = readRefused();
467
582
  const plan = readPlan(journey);
468
583
  if (!plan?.some((f) => f.rules?.length))
469
584
  return true;
@@ -475,7 +590,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
475
590
  return true;
476
591
  // Rules left without a test make a suite unfinished. A check that flags an
477
592
  // existing test does not: kept tests are reported, never rewritten.
478
- return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.length;
593
+ return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.filter((id) => !(refused[f.slug] ?? []).includes(id)).length;
479
594
  });
480
595
  };
481
596
  // Tests an earlier version appended after the outer describe go back inside
@@ -486,7 +601,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
486
601
  continue;
487
602
  const p = join(testDir, f);
488
603
  const before = readFileSync(p, 'utf-8');
489
- const after = tuckStrayTests(before);
604
+ // Both mechanical, no AI: stray tests back inside the describe, and
605
+ // kit.insert's id used as the id it is (op[ID_FIELD] → op).
606
+ const after = useInsertIds(tuckStrayTests(before));
490
607
  if (after !== before)
491
608
  writeFileSync(p, after, 'utf-8');
492
609
  }
@@ -504,6 +621,28 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
504
621
  catch {
505
622
  return false;
506
623
  } }));
624
+ // The doc is the same and every rule has its test: new code changes no
625
+ // test. Its facts are read again and the names the tests wait on are
626
+ // filled from them; the test files are not written again (that is
627
+ // --rewrite-waiting, asked for by name).
628
+ if (!force && !opts.rewriteWaiting && waitsOnCode && cache[file] === hash && !hasDeferredStubs(testDir, ext) && suiteComplete()) {
629
+ const usage0 = aiUsage();
630
+ const facts = /^status:\s*not built/im.test(markdown) ? null : await groundJourney(journey, hash, sourceRoot(), markdown, false, opts.refreshCode);
631
+ const blank = [...new Set(testFiles0.flatMap((p) => { try {
632
+ return blankNames(readFileSync(p, 'utf-8'));
633
+ }
634
+ catch {
635
+ return [];
636
+ } }))];
637
+ const res = facts && blank.length ? scanNames(blank, sourceRoot(), facts) : undefined;
638
+ const notes = [
639
+ `doc unchanged: no test written again (code facts read again${res ? `, ${res.filled.length} of ${blank.length} blank name(s) filled from them` : ''})`,
640
+ ...(res?.filled.length ? [`filled: ${res.filled.slice(0, 8).map((f) => `${f.name} = ${f.value}`).join(', ')}${res.filled.length > 8 ? ', …' : ''}`] : []),
641
+ ...(res?.stillBlank.length ? [`${res.stillBlank.length} name(s) still blank — fill them in testforge/interface.json, or run with --rewrite-waiting to have the tests written again: ${res.stillBlank.slice(0, 8).join(', ')}${res.stillBlank.length > 8 ? ', …' : ''}`] : []),
642
+ ];
643
+ results.push({ journeyName: journey.name, testDir, testFiles: [], seedPath, status: 'skipped', seedStatus: 'none', warnings: notes, usage: usageSince(usage0) });
644
+ continue;
645
+ }
507
646
  if (!force &&
508
647
  !waitsOnCode &&
509
648
  cache[file] === hash &&
@@ -574,21 +713,45 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
574
713
  // What each file really tests once written: the memory and the report come from this, not the plan.
575
714
  const written = [];
576
715
  // need() names asked so far (the project's and this run's): handed to every later file.
577
- const askedNames = new Set(Object.keys(readInterface()).filter(isThingName));
716
+ // Only names the product has given are handed on for reuse: a blank one in
717
+ // that list reads as "use this name", and the new test waits on it again.
718
+ const filledNow = readInterface();
719
+ const askedNames = new Set(Object.keys(filledNow).filter((n) => isThingName(n) && String(filledNow[n] ?? '').trim()));
578
720
  const finish = async (content, pf, filePath, kept = false) => {
579
721
  if (!memory || !pf.rules?.length)
580
722
  return content;
581
723
  const { mem, facts } = await fileMemory(pf);
724
+ // A blank name in a hook or helper makes the whole file wait. Only the
725
+ // setup is written again (one small call), never the tests; once per
726
+ // doc and code if it is refused.
727
+ if (kept && stale.has(filePath) && blankOutsideKind(content) === 'setup' && !(readRefused()[pf.slug] ?? []).includes('#setup')) {
728
+ const redone = await redoSetup(content, filePath, facts, mem?.helpers ?? helpers);
729
+ if (redone) {
730
+ warnings.push(`${pf.slug}: its setup (hooks and helpers) was written again from the code facts; its tests were kept`);
731
+ content = redone;
732
+ }
733
+ else {
734
+ warnings.push(`${pf.slug}: its setup could not be written again — the file keeps waiting on its names`);
735
+ rememberRefused(pf.slug, ['#setup']);
736
+ }
737
+ }
582
738
  const done = await completeFile(content, {
583
739
  kept, locked: passedTitles(filePath),
584
740
  redo: kept && stale.has(filePath) ? (redoAll ? () => true : (b) => usesBlankName(content, b)) : undefined,
741
+ isBlank: (n) => blankNames(`need('${n}')`).length > 0,
742
+ skip: new Set(readRefused()[pf.slug] ?? []),
743
+ fix: fixTests,
585
744
  journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
586
745
  harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
587
746
  check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
588
747
  });
589
748
  written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
749
+ // What was asked and still has no test is not asked again for this doc and code.
750
+ if (done.refusedRules.length)
751
+ rememberRefused(pf.slug, done.refusedRules);
590
752
  for (const n of neededNames(done.content))
591
- askedNames.add(n);
753
+ if (!blankNames(`need('${n}')`).length)
754
+ askedNames.add(n);
592
755
  for (const n of done.notes)
593
756
  warnings.push(`${pf.slug}: ${n}`);
594
757
  return done.content;
@@ -709,7 +872,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
709
872
  const stale = new Set();
710
873
  // Every test of a journey first written with no code is looked at again once its code is there.
711
874
  let redoAll = false;
712
- if (useAI && !notBuilt && blindBefore && testFiles0.length && (await grounding())) {
875
+ if (useAI && !notBuilt && opts.rewriteWaiting && blindBefore && testFiles0.length && (await grounding())) {
713
876
  for (const p of planned.map(pathOf))
714
877
  if (existsSync(p))
715
878
  stale.add(p);
@@ -717,7 +880,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
717
880
  if (redoAll)
718
881
  warnings.push(`its code is here now: the tests first written without it are written again from the code (tests that passed are kept)`);
719
882
  }
720
- if (useAI && !notBuilt && !redoAll) {
883
+ if (useAI && !notBuilt && !redoAll && opts.rewriteWaiting) {
721
884
  const waiting = planned.map(pathOf).filter((p) => { try {
722
885
  return existsSync(p) && blankNames(readFileSync(p, 'utf-8')).length > 0;
723
886
  }
@@ -730,12 +893,35 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
730
893
  if (waiting.length && (opts.refreshCode || codeMoved) && (await grounding())) {
731
894
  for (const p of waiting)
732
895
  stale.add(p);
733
- if (stale.size)
734
- warnings.push(`${stale.size} file(s) waiting on names the code now has: only their waiting tests are written again`);
896
+ if (stale.size) {
897
+ const kinds = [...stale].map((p) => { try {
898
+ return blankOutsideKind(readFileSync(p, 'utf-8'));
899
+ }
900
+ catch {
901
+ return undefined;
902
+ } });
903
+ const setup = kinds.filter((k) => k === 'setup').length;
904
+ const done = readRefused();
905
+ const whole = [...stale].filter((p, i) => kinds[i] === 'file' && !(done[planned.find((pf) => pathOf(pf) === p)?.slug ?? ''] ?? []).includes('#whole')).length;
906
+ warnings.push(`${stale.size} file(s) waiting on names the code now has: only their waiting tests are written again${setup ? `; the setup (hooks, helpers) of ${setup}` : ''}${whole ? `; ${whole} written again whole, once (their names sit in parametrised tables or loops)` : ''}`);
907
+ }
735
908
  }
736
909
  }
737
910
  // A doc without rule ids has no per-test redo: its waiting files are written again whole, as before.
738
- const rewriteWhole = (p) => stale.has(p) && !memory;
911
+ // Written again whole: a doc without rule ids (no per-test redo), or a file
912
+ // whose blank names sit where no test or setup block can be split out —
913
+ // once per doc and code, and never a file holding a test that passed.
914
+ const rewriteWhole = (p) => stale.has(p) && (!memory || (() => {
915
+ try {
916
+ const c = readFileSync(p, 'utf-8');
917
+ const slug = planned.find((pf) => pathOf(pf) === p)?.slug ?? '';
918
+ return blankOutsideKind(c) === 'file' && !(readRefused()[slug] ?? []).includes('#whole')
919
+ && !findTestBlocks(c).some((b) => passedTitles(p).has(b.title));
920
+ }
921
+ catch {
922
+ return false;
923
+ }
924
+ })());
739
925
  const keeps = (p) => {
740
926
  if (rewriteWhole(p))
741
927
  return false;
@@ -774,7 +960,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
774
960
  const got = await writeFilesWithAI(files);
775
961
  noteAiSuccess();
776
962
  for (const [p, c] of got)
777
- drafts.set(p, c);
963
+ drafts.set(p, fixTests(c));
778
964
  }
779
965
  catch (e) {
780
966
  noteAiFailure(e);
@@ -821,6 +1007,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
821
1007
  }
822
1008
  let content;
823
1009
  let status;
1010
+ if (memory && stale.has(filePath) && existsSync(filePath))
1011
+ rememberRefused(pf.slug, ['#whole']);
824
1012
  if (useAI) {
825
1013
  let draft = drafts.get(filePath);
826
1014
  let draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
@@ -835,7 +1023,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
835
1023
  }
836
1024
  }
837
1025
  const { mem, facts } = await fileMemory(pf);
838
- const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
1026
+ const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined, fixTests);
839
1027
  if (ai !== null) {
840
1028
  content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
841
1029
  status = 'ai';
@@ -876,6 +1064,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
876
1064
  }
877
1065
  return { slug: pf.slug, path: filePath, status };
878
1066
  });
1067
+ // The names the new tests still wait on, looked up once against this
1068
+ // journey's code facts: a field the facts show is filled now, not left
1069
+ // for the run. Kept only if the code itself has the value.
1070
+ if (useAI && !notBuilt && memory) {
1071
+ const facts = await grounding();
1072
+ const files = planned.map(pathOf).filter((p) => existsSync(p));
1073
+ const blank = [...new Set(files.flatMap((p) => { try {
1074
+ return blankNames(readFileSync(p, 'utf-8'));
1075
+ }
1076
+ catch {
1077
+ return [];
1078
+ } }))];
1079
+ if (facts && blank.length) {
1080
+ const res = scanNames(blank, sourceRoot(), facts);
1081
+ if (res.filled.length)
1082
+ warnings.push(`filled ${res.filled.length} name(s) from the code facts: ${res.filled.slice(0, 6).map((f) => `${f.name} = ${f.value}`).join(', ')}${res.filled.length > 6 ? ', …' : ''}`);
1083
+ }
1084
+ }
879
1085
  if (memory && written.length) {
880
1086
  const sum = recordWritten(journey, written);
881
1087
  console.log(` ${journey.id}: ${sum}`);
@@ -119,3 +119,90 @@ export function renameInBlocks(content, blocks, pairs) {
119
119
  });
120
120
  return { content: spliceBlocks(content, edits), replaced };
121
121
  }
122
+ /**
123
+ * Tests written when the writer took kit.insert's answer for a row
124
+ * (`const op = await kit.insert(…); op[ID_FIELD]` with ID_FIELD =
125
+ * need('inserted row id field')). kit.insert returns the id itself, so
126
+ * `op[ID_FIELD]` is `op`: rewritten here, the blank name goes, and no AI is
127
+ * asked to write the file again. Only variables assigned from kit.insert.
128
+ */
129
+ export function useInsertIds(content) {
130
+ const decl = /^[ \t]*(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]*\binsert\w*\b[^'"`\n]*\bid\b[^'"`\n]*)\2\s*\);?[ \t]*\n/gim;
131
+ let out = content;
132
+ for (const m of [...content.matchAll(decl)]) {
133
+ const [line, name] = m;
134
+ const fromInsert = new Set([...out.matchAll(/\b([A-Za-z_$][\w$]*)\s*=\s*await\s+kit\.insert\(/g)].map((x) => x[1]));
135
+ const esc = name.replace(/\$/g, '\\$');
136
+ out = out.replace(new RegExp(`\\b([A-Za-z_$][\\w$]*)\\[${esc}\\]`, 'g'), (all, v) => (fromInsert.has(v) ? v : all));
137
+ // The constant goes only when nothing uses it any more.
138
+ if (!new RegExp(`\\b${esc}\\b`).test(out.replace(line, '')))
139
+ out = out.replace(line, '');
140
+ }
141
+ return out;
142
+ }
143
+ /** Index just past the bracket closing the one at `open` ((, { or [), skipping strings, templates and comments; -1 if none. */
144
+ export function closeBracket(src, open) {
145
+ const pairs = { '(': ')', '{': '}', '[': ']' };
146
+ const stack = [];
147
+ for (let i = open; i < src.length; i++) {
148
+ const c = src[i];
149
+ if (c === '/' && src[i + 1] === '/') {
150
+ const n = src.indexOf('\n', i);
151
+ if (n < 0)
152
+ return -1;
153
+ i = n;
154
+ continue;
155
+ }
156
+ if (c === '/' && src[i + 1] === '*') {
157
+ const n = src.indexOf('*/', i + 2);
158
+ if (n < 0)
159
+ return -1;
160
+ i = n + 1;
161
+ continue;
162
+ }
163
+ if (c === '"' || c === "'" || c === '`') {
164
+ for (i += 1; i < src.length && src[i] !== c; i++)
165
+ if (src[i] === '\\')
166
+ i += 1;
167
+ continue;
168
+ }
169
+ if (pairs[c])
170
+ stack.push(pairs[c]);
171
+ else if (c === ')' || c === '}' || c === ']') {
172
+ if (stack.pop() !== c)
173
+ return -1;
174
+ if (!stack.length)
175
+ return i + 1;
176
+ }
177
+ }
178
+ return -1;
179
+ }
180
+ /**
181
+ * The file's setup outside its tests: hooks (beforeAll/beforeEach/afterAll/
182
+ * afterEach) and helper functions (function x() {…}, const x = async () => {…}).
183
+ */
184
+ export function findSetupBlocks(content) {
185
+ const tests = findTestBlocks(content);
186
+ const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
187
+ const out = [];
188
+ const START = /^[ \t]*(?:(?:test\.)?(beforeAll|beforeEach|afterAll|afterEach)\s*\(|(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(|(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)\n]*\)|[A-Za-z_$][\w$]*)\s*=>)/gm;
189
+ for (const m of content.matchAll(START)) {
190
+ const at = m.index;
191
+ if (inTest(at) || out.some((b) => at >= b.start && at < b.end))
192
+ continue;
193
+ let end = -1;
194
+ if (m[1])
195
+ end = closeBracket(content, content.indexOf('(', at));
196
+ else {
197
+ const arrow = m[3] ? content.indexOf('=>', at) : -1;
198
+ const brace = content.indexOf('{', m[2] ? closeBracket(content, content.indexOf('(', at)) : arrow);
199
+ end = brace < 0 ? -1 : closeBracket(content, brace);
200
+ }
201
+ if (end < 0)
202
+ continue;
203
+ if (content[end] === ';')
204
+ end += 1;
205
+ out.push({ start: at, end, name: m[1] ?? m[2] ?? m[3], text: content.slice(at, end) });
206
+ }
207
+ return out;
208
+ }
@@ -17,7 +17,9 @@ export function envNames(text) {
17
17
  const out = new Set();
18
18
  for (const line of text.split('\n')) {
19
19
  const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
20
- if (m && m[2].trim().replace(/^(['"])\1$/, '') !== '')
20
+ // An unquoted value ends at a comment: `KEY= # hint` (an unfilled template line) is unset.
21
+ const v = /^\s*(['"]).*\1\s*$/.test(m?.[2] ?? '') ? (m?.[2] ?? '').trim() : (m?.[2] ?? '').replace(/(^|\s)#.*$/, '').trim();
22
+ if (m && v.replace(/^(['"])\1$/, '') !== '')
21
23
  out.add(m[1]);
22
24
  }
23
25
  return out;
@@ -36,10 +36,18 @@ export async function runSetup(srcDir, opts = {}) {
36
36
  runSetupCall(loadConfig().relatedDirs ?? []);
37
37
  if (existsSync('testforge.config.json')) {
38
38
  const problems = checkSetupConfig();
39
- console.log(problems.length ? ` testforge.config.json written, with problems to fix:\n ${problems.join('\n ')}\n` : ' done — testforge.config.json');
39
+ if (problems.length) {
40
+ // Drafting docs on a config with problems spends AI calls on a project
41
+ // that cannot run yet: stop, and say what to fix.
42
+ console.log(` testforge.config.json written, with problems to fix first:\n ${problems.join('\n ')}\n`);
43
+ console.log('Fix these, then run `testforge setup` again (or `testforge discover`).');
44
+ return;
45
+ }
46
+ console.log(' done — testforge.config.json');
40
47
  }
41
48
  else {
42
- console.log(' warning: testforge.config.json was not created');
49
+ console.log(' testforge.config.json was not created — nothing else was done. Run `testforge setup` again.');
50
+ return;
43
51
  }
44
52
  console.log(existsSync('testforge.md') ? ' done — testforge.md\n' : ' warning: testforge.md was not created\n');
45
53
  // Step 3 — discover. This is where the source-of-truth barrier lives:
@@ -183,6 +183,9 @@ E2. What you touch comes from <code_facts> and <testkit>.
183
183
  - Look in ROUTES, SHAPES and DATA before need(): a name any of them shows is
184
184
  written as it is there. need() is only for a name none of them has (see
185
185
  NAMES). Never guess one.
186
+ - A path from need() may hold parameters ({booking}, :id): fill them with
187
+ fillPath(URL, value, …) from the same module as need(), never with
188
+ .replace() on a guessed placeholder name.
186
189
  - Never need() what the testkit already gives: kit.insert returns the new
187
190
  row's id; a second person of a role (another operator, another customer)
188
191
  is a row made with kit.insert; the roles are the ones <testkit> lists.
@@ -199,9 +202,11 @@ of making it.
199
202
  - Owner-scoped data ("my orders") follows the token owner, not a query
200
203
  parameter. A non-owner check needs a second token; without one, skip and
201
204
  name the missing identity.
202
- - If a precondition really cannot be met, skip at run time and state what was
203
- observed ("TEST_X is not set", "GET /x answered 404"), never a guessed
204
- cause.
205
+ - If a precondition really cannot be met (an account, an env variable or data
206
+ the environment lacks — never an element missing from the page), skip at
207
+ run time inside the test and state what was observed ("TEST_X is not set",
208
+ "GET /x answered 404"), never a guessed cause. That is the only skip: no
209
+ skipped test declarations, describe.skip or test.todo.
205
210
 
206
211
  E4. What you wait for is the event that actually happens. Wait on a
207
212
  condition, never for a fixed time.
@@ -290,8 +295,11 @@ export const WEB = `WEB (Playwright)
290
295
  title. A stubbed item carries every field the screen reads to show it,
291
296
  including the value the list filters on (its status), and the test finds it
292
297
  on screen by a field the screen renders as text, never by one it does not
293
- show. Match the stub by exact host, port, path prefix or a request
294
- predicate, never by a substring that also matches the app's own pages.
298
+ show. Finding it is not the check: assert what the product does with that
299
+ state (the badge, the message, the button it disables or hides), never only
300
+ that the text you stubbed appears. Match the stub by exact host, port, path
301
+ prefix or a request predicate, never by a substring that also matches the
302
+ app's own pages.
295
303
  - Waiting: prefer retrying assertions. Register waitForResponse or
296
304
  waitForRequest before the action that triggers it. After a login, sign-out
297
305
  or navigation, wait for the control you will use, then assert the URL the