@activate-core/test-forge 2.4.6 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +47 -11
  2. package/dist/cli/index.js +12 -12
  3. package/dist/core/docs/cache.js +1 -1
  4. package/dist/core/docs/discover.js +5 -5
  5. package/dist/core/docs/excerpts.js +139 -5
  6. package/dist/core/docs/grounding.js +78 -16
  7. package/dist/core/docs/reference-text.js +310 -52
  8. package/dist/core/gates/coverage.js +9 -0
  9. package/dist/core/gates/validate.js +1 -0
  10. package/dist/core/generate/ai-generator.js +9 -3
  11. package/dist/core/generate/ai-usage.js +11 -4
  12. package/dist/core/generate/bind.js +3 -2
  13. package/dist/core/generate/complete-file.js +45 -7
  14. package/dist/core/generate/defect-review.js +2 -1
  15. package/dist/core/generate/harness.js +34 -0
  16. package/dist/core/generate/interface-file.js +12 -0
  17. package/dist/core/generate/journey-memory.js +7 -1
  18. package/dist/core/generate/passed.js +24 -0
  19. package/dist/core/generate/review-journey.js +2 -2
  20. package/dist/core/generate/run-generate.js +117 -13
  21. package/dist/core/generate/suite-planner.js +2 -2
  22. package/dist/core/generate/test-blocks.js +34 -10
  23. package/dist/core/paths.js +41 -6
  24. package/dist/core/project/config.js +5 -0
  25. package/dist/core/project/setup.js +9 -9
  26. package/dist/core/project/update.js +6 -6
  27. package/dist/core/prompts/blocks.js +24 -3
  28. package/dist/core/run/name-scan.js +120 -14
  29. package/dist/core/run/report.js +7 -4
  30. package/dist/core/run/run-tests.js +53 -4
  31. package/package.json +3 -2
  32. package/reporters/_shared.cjs +56 -3
  33. package/reporters/interface.cjs +14 -0
  34. package/reporters/jest-reporter.cjs +7 -3
  35. package/reporters/playwright-reporter.cjs +6 -2
  36. package/reporters/run-data.cjs +6 -1
@@ -40,6 +40,11 @@ export const HARNESS_CONTRACT = [
40
40
  `- resetRateLimits(): Promise<void> (H7)`,
41
41
  `- REQUIRED_PACKAGES: string[] (H6)`,
42
42
  `- a 'run' role alongside the fixture roles (H8)`,
43
+ `- insert(table, row): Promise<id>, one row of scenario data written straight`,
44
+ ` into the test database (H10)`,
45
+ `- DERIVED and AREA (H10)`,
46
+ `- update(table, match, values): Promise<number>, columns set on the rows`,
47
+ ` matching every key of match (H10)`,
43
48
  `Jest areas only`,
44
49
  `- call(method, path, { body?, headers?, role? }):`,
45
50
  ` Promise<{ status, headers, body }>`,
@@ -177,6 +182,35 @@ export const HARNESS_CONTRACT = [
177
182
  ` 'unknown step "<name>" (steps: …)'. If there is no such flow, export`,
178
183
  ` STEPS = {} and have openStep resolve`,
179
184
  ` 'this product has no multi-step flow'.`,
185
+ ``,
186
+ `H10. Scenario data. A scenario describes people and their state (an operator`,
187
+ ` with a location, services, working hours and earlier jobs; a setting`,
188
+ ` switched on). insert() and update() let a test build exactly that:`,
189
+ ` - insert fills every NOT NULL column without a default that row leaves out`,
190
+ ` with a value of its type (read the columns from the database), returns`,
191
+ ` the new id, and remembers the row.`,
192
+ ` - update remembers the previous values of the rows it changes.`,
193
+ ` - Each test starts from a clean slate: the rows a test writes, and what it`,
194
+ ` updates, are undone right after that test (last first), so one test's`,
195
+ ` data never takes part in the next. The testkit does this itself when it`,
196
+ ` is loaded: it notes where the record stands before each test and undoes`,
197
+ ` back to that point after it (Jest: the global beforeEach/afterEach;`,
198
+ ` Playwright: test.beforeEach/afterEach). Tests do nothing for it.`,
199
+ ` - What a beforeAll writes stays for the whole file; disconnect() undoes it`,
200
+ ` and anything left, so every run leaves the database as it found it.`,
201
+ ` - Both refuse a database that is not clearly a test one, like reset() (H5).`,
202
+ ` - Some values a scenario sets are not columns: the product works them out`,
203
+ ` from other records (a standing from past jobs, an average per day, a`,
204
+ ` count). Read in the code how each is worked out. insert() accepts each as`,
205
+ ` a virtual column named _tf_<name> (lower case, underscores) and, instead`,
206
+ ` of writing it, writes the records that give exactly that value (and are`,
207
+ ` undone with the row). Export DERIVED: { _tf_<name>: 'how it is made' }`,
208
+ ` for each, so tests can use them.`,
209
+ ` - Export AREA: { latitude, longitude } (or the project's own key, when`,
210
+ ` the product picks among records by something else): a place no record`,
211
+ ` already in the database reaches, found by querying it. Scenarios where`,
212
+ ` the product picks one record among many are built there, so data that`,
213
+ ` was already in the database never takes part.`,
180
214
  ].join('\n');
181
215
  /**
182
216
  * Stamp written into every generated testkit: the contract it was built
@@ -68,3 +68,15 @@ export function recordInterface(found, path = INTERFACE_PATH) {
68
68
  }
69
69
  return blankNames(Object.keys(found), values);
70
70
  }
71
+ /** Blank these names again (a value the code no longer has): their tests wait instead of failing for the wrong reason. */
72
+ export function clearInterface(names, path = INTERFACE_PATH) {
73
+ if (!names.length)
74
+ return;
75
+ const values = readInterface(path);
76
+ for (const n of names)
77
+ if (n in values)
78
+ values[n] = '';
79
+ mkdirSync(dirname(path), { recursive: true });
80
+ const sorted = Object.fromEntries(Object.entries(values).sort(([a], [b]) => a.localeCompare(b)));
81
+ writeFileSync(path, `${JSON.stringify(sorted, null, 2)}\n`, 'utf-8');
82
+ }
@@ -206,6 +206,10 @@ const RUNTIME_HELPERS = {
206
206
  fillFrameInputs: 'async fillFrameInputs(page, framePrefix, { name: value }) — fills inputs inside an embedded iframe (card fields).',
207
207
  watchDialogs: 'watchDialogs(page) → string[] — accepts browser alerts and records their messages.',
208
208
  };
209
+ const KIT_CONTRACT = {
210
+ insert: "resolves to the new row's id itself (a number or key, not a row object): const id = await insert('users', {...})",
211
+ update: 'resolves to the number of rows changed',
212
+ };
209
213
  export function helperApi(harnessSource) {
210
214
  const names = new Set();
211
215
  const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(harnessSource);
@@ -262,7 +266,9 @@ export function helperApi(harnessSource) {
262
266
  if (text)
263
267
  doc.unshift(text);
264
268
  }
265
- out.push(`- ${sig.length > 200 ? `${sig.slice(0, 197)}…` : sig}${doc.length ? ` — ${doc.join(' ')}` : ''}`);
269
+ // The scenario-data writers' contract (H10) holds whatever comment sits above them.
270
+ const kit = KIT_CONTRACT[name] && /^(?:async\s+)?function/.test(sig) ? KIT_CONTRACT[name] : doc.join(' ');
271
+ out.push(`- ${sig.length > 200 ? `${sig.slice(0, 197)}…` : sig}${kit ? ` — ${kit}` : ''}`);
266
272
  }
267
273
  return out.join('\n');
268
274
  }
@@ -0,0 +1,24 @@
1
+ /**
2
+ * testforge — the tests that have passed, as the run recorded them.
3
+ *
4
+ * A test that passed once checked the product for real: generate never
5
+ * removes or rewrites it again. The reporters add to testforge/passed.json
6
+ * after every run ({ "<file relative to the project>": ["title", …] }); a
7
+ * later failure does not take a test off the list — that is a regression to
8
+ * report, not a test to rewrite. `generate --force` is the only way past it.
9
+ */
10
+ import { readFileSync } from 'node:fs';
11
+ import { join, relative, resolve } from 'node:path';
12
+ import { STATE_DIR } from '../paths.js';
13
+ export const PASSED_PATH = join(STATE_DIR, 'passed.json');
14
+ /** Titles of the tests in `file` that have passed in a recorded run. */
15
+ export function passedTitles(file, path = PASSED_PATH) {
16
+ try {
17
+ const all = JSON.parse(readFileSync(path, 'utf-8'));
18
+ const rel = relative(process.cwd(), resolve(file)).split('\\').join('/');
19
+ return new Set(Array.isArray(all[rel]) ? all[rel] : []);
20
+ }
21
+ catch {
22
+ return new Set();
23
+ }
24
+ }
@@ -10,7 +10,7 @@
10
10
  */
11
11
  import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
12
12
  import { basename, extname, join } from 'node:path';
13
- import { RUNS_DIR } from '../paths.js';
13
+ import { RUNS_DIR, DOCS_DIR } from '../paths.js';
14
14
  import { findMarkdownFiles } from '../find-files.js';
15
15
  import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
16
16
  import { loadConfig, testDirForType } from '../project/config.js';
@@ -59,7 +59,7 @@ const testFiles = (dir) => {
59
59
  };
60
60
  export async function runReview(opts = {}) {
61
61
  const config = loadConfig();
62
- const docsDir = opts.docsDir ?? 'docs';
62
+ const docsDir = opts.docsDir ?? DOCS_DIR;
63
63
  const out = [];
64
64
  for (const file of findMarkdownFiles(docsDir)) {
65
65
  const markdown = readFileSync(join(docsDir, file), 'utf-8');
@@ -16,7 +16,7 @@
16
16
  * so tests/seeds land where the project expects them. Defaults to
17
17
  * generated-tests/ when not configured.
18
18
  */
19
- import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
19
+ import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, } from 'node:fs';
20
20
  import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
21
21
  import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored, journeyDir } from '../paths.js';
22
22
  import { lintGenerated } from '../gates/lint.js';
@@ -27,13 +27,34 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
29
  import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
30
- import { completeFile, fileIsComplete } from './complete-file.js';
30
+ import { completeFile } from './complete-file.js';
31
+ import { tuckStrayTests } from './test-blocks.js';
32
+ import { passedTitles } from './passed.js';
31
33
  import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
32
- import { harnessSteps } from '../gates/test-quality.js';
34
+ /** Names the file asks for that testforge/interface.json still has blank. */
35
+ /** Does this test use a name the product has not given yet (directly, or through the constant it was put in)? */
36
+ function usesBlankName(content, block) {
37
+ const text = content.slice(block.start, block.end);
38
+ return blankNames(content).some((n) => {
39
+ if (text.includes(`'${n}'`) || text.includes(`"${n}"`) || text.includes(`\`${n}\``))
40
+ return true;
41
+ const esc = n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
42
+ const v = new RegExp(`\\b(?:const|let|var)\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*need\\(\\s*(['"\`])${esc}\\2`).exec(content)?.[1];
43
+ return !!v && new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(text);
44
+ });
45
+ }
46
+ function blankNames(content) {
47
+ const iface = readInterface();
48
+ return [...content.matchAll(/\bneed\(\s*(['"`])([^'"`]+)\1\s*\)/g)].map((m) => m[2]).filter((n) => {
49
+ const v = iface[n];
50
+ return typeof v === 'string' ? !v.trim() : v === undefined || v === null;
51
+ });
52
+ }
53
+ import { harnessSteps, ruleCoverage } from '../gates/test-quality.js';
33
54
  import { normalisePending } from './pending.js';
34
55
  import { packageRoot } from '../run/run-tests.js';
35
56
  import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
36
- import { groundJourney } from '../docs/grounding.js';
57
+ import { factsCodeMoved, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
37
58
  import { generateSeed, seedProblem } from './seed-generator.js';
38
59
  import { dataAccess, writeRequiredData } from './required-data.js';
39
60
  import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
@@ -452,10 +473,39 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
452
473
  const p = join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`);
453
474
  if (!f.rules?.length || !existsSync(p))
454
475
  return true;
455
- return fileIsComplete(readFileSync(p, 'utf-8'), f.rules, steps, appTexts());
476
+ // Rules left without a test make a suite unfinished. A check that flags an
477
+ // existing test does not: kept tests are reported, never rewritten.
478
+ return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.length;
456
479
  });
457
480
  };
481
+ // Tests an earlier version appended after the outer describe go back inside
482
+ // it, so they get the file's setup. No AI, also for an unchanged journey.
483
+ if (existsSync(testDir)) {
484
+ for (const f of readdirSync(testDir)) {
485
+ if (!f.endsWith(ext))
486
+ continue;
487
+ const p = join(testDir, f);
488
+ const before = readFileSync(p, 'utf-8');
489
+ const after = tuckStrayTests(before);
490
+ if (after !== before)
491
+ writeFileSync(p, after, 'utf-8');
492
+ }
493
+ }
494
+ // Code that arrived after the tests were written: files still asking for
495
+ // names the product did not have are written again from the new code.
496
+ const codeMoved = useAI && (opts.refreshCode || factsCodeMoved(journey, sourceRoot()));
497
+ const testFiles0 = existsSync(testDir) ? readdirSync(testDir).filter((f) => f.endsWith(ext)).map((f) => join(testDir, f)) : [];
498
+ // Tests written while the feature had no code: their "how" was a guess, so
499
+ // the code arriving is a reason to look again even with every name filled.
500
+ const blindBefore = writtenWithoutCode(journey);
501
+ const waitsOnCode = codeMoved && (blindBefore || testFiles0.some((p) => { try {
502
+ return blankNames(readFileSync(p, 'utf-8')).length > 0;
503
+ }
504
+ catch {
505
+ return false;
506
+ } }));
458
507
  if (!force &&
508
+ !waitsOnCode &&
459
509
  cache[file] === hash &&
460
510
  !hasDeferredStubs(testDir, ext) &&
461
511
  suiteComplete()) {
@@ -474,6 +524,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
474
524
  // A feature written up before it is built: its tests come from the doc
475
525
  // alone. There is no code to read for facts and no app to run them on.
476
526
  const notBuilt = /^status:\s*not built/im.test(markdown);
527
+ if (notBuilt && useAI)
528
+ markWrittenWithoutCode(journey, sourceRoot());
477
529
  // Not live yet (not built, or behind a flag): every file is stamped, and
478
530
  // `testforge run` holds it back until the doc's Status line goes.
479
531
  const pending = pendingReason(markdown);
@@ -523,11 +575,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
523
575
  const written = [];
524
576
  // need() names asked so far (the project's and this run's): handed to every later file.
525
577
  const askedNames = new Set(Object.keys(readInterface()).filter(isThingName));
526
- const finish = async (content, pf, filePath) => {
578
+ const finish = async (content, pf, filePath, kept = false) => {
527
579
  if (!memory || !pf.rules?.length)
528
580
  return content;
529
581
  const { mem, facts } = await fileMemory(pf);
530
582
  const done = await completeFile(content, {
583
+ kept, locked: passedTitles(filePath),
584
+ redo: kept && stale.has(filePath) ? (redoAll ? () => true : (b) => usesBlankName(content, b)) : undefined,
531
585
  journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
532
586
  harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
533
587
  check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
@@ -635,7 +689,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
635
689
  // on first use: re-checking an existing suite costs no grounding at all.
636
690
  let groundingP;
637
691
  const grounding = () => (groundingP ??= useAI && !notBuilt
638
- ? groundJourney(journey, hash, sourceRoot(), markdown).then((g) => g ?? undefined)
692
+ ? groundJourney(journey, hash, sourceRoot(), markdown, false, opts.refreshCode).then((g) => g ?? undefined)
639
693
  : Promise.resolve(undefined));
640
694
  const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
641
695
  if (staleRemoved.length) {
@@ -651,7 +705,40 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
651
705
  // once. A drafted file that fails its checks is written again on its own,
652
706
  // told why; a file the batch left out is written on its own.
653
707
  const pathOf = (pf) => join(testDir, `${flat ? `${journey.id}-${pf.slug}` : pf.slug}${ext}`);
708
+ // A file written from older code facts that still waits on names: written again.
709
+ const stale = new Set();
710
+ // Every test of a journey first written with no code is looked at again once its code is there.
711
+ let redoAll = false;
712
+ if (useAI && !notBuilt && blindBefore && testFiles0.length && (await grounding())) {
713
+ for (const p of planned.map(pathOf))
714
+ if (existsSync(p))
715
+ stale.add(p);
716
+ redoAll = stale.size > 0;
717
+ if (redoAll)
718
+ warnings.push(`its code is here now: the tests first written without it are written again from the code (tests that passed are kept)`);
719
+ }
720
+ if (useAI && !notBuilt && !redoAll) {
721
+ const waiting = planned.map(pathOf).filter((p) => { try {
722
+ return existsSync(p) && blankNames(readFileSync(p, 'utf-8')).length > 0;
723
+ }
724
+ catch {
725
+ return false;
726
+ } });
727
+ // Only new code (or --refresh-code) rewrites a waiting file: facts read
728
+ // again from the same code may word things differently, and that alone
729
+ // must never cost a rewrite.
730
+ if (waiting.length && (opts.refreshCode || codeMoved) && (await grounding())) {
731
+ for (const p of waiting)
732
+ stale.add(p);
733
+ if (stale.size)
734
+ warnings.push(`${stale.size} file(s) waiting on names the code now has: only their waiting tests are written again`);
735
+ }
736
+ }
737
+ // A doc without rule ids has no per-test redo: its waiting files are written again whole, as before.
738
+ const rewriteWhole = (p) => stale.has(p) && !memory;
654
739
  const keeps = (p) => {
740
+ if (rewriteWhole(p))
741
+ return false;
655
742
  if ((force && !redone.has(i)) || !existsSync(p))
656
743
  return false;
657
744
  try {
@@ -711,7 +798,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
711
798
  // midway resumes from exactly the files it hadn't written yet.
712
799
  // (--force ignores this and regenerates.)
713
800
  // A redo after stubs fills in only the stubs: --force already rewrote the rest once this run.
714
- if ((!force || redone.has(i)) && existsSync(filePath)) {
801
+ if ((!force || redone.has(i)) && existsSync(filePath) && !rewriteWhole(filePath)) {
715
802
  try {
716
803
  const existing = readFileSync(filePath, 'utf-8');
717
804
  // Keep a real, parsing file — but a DEFERRED stub is not real
@@ -722,7 +809,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
722
809
  const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
723
810
  // A kept file that leaves rules out or holds tests that cannot fail:
724
811
  // only those tests are written again.
725
- const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath) : normalised, pending);
812
+ const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath, true) : normalised, pending);
726
813
  if (finished !== existing)
727
814
  writeFileSync(filePath, finished, 'utf-8');
728
815
  return { slug: pf.slug, path: filePath, status: 'ai' };
@@ -735,8 +822,18 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
735
822
  let content;
736
823
  let status;
737
824
  if (useAI) {
738
- const draft = drafts.get(filePath);
739
- const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
825
+ let draft = drafts.get(filePath);
826
+ let draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
827
+ // A draft that parses but holds a few refused tests: only those tests are
828
+ // written again (the way a kept file is finished), not the whole file.
829
+ if (draft !== undefined && draftRefusal && isSyntaxValid(draft, ext)) {
830
+ const finished = await finish(framework === 'playwright' ? normalisePending(draft) : draft, pf, filePath);
831
+ const still = isSyntaxValid(finished, ext) ? genGate(finished, filePath) : syntaxReason(finished, ext);
832
+ if (!still) {
833
+ draft = finished;
834
+ draftRefusal = undefined;
835
+ }
836
+ }
740
837
  const { mem, facts } = await fileMemory(pf);
741
838
  const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
742
839
  if (ai !== null) {
@@ -744,6 +841,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
744
841
  status = 'ai';
745
842
  }
746
843
  else {
844
+ // The AI could not write it: a real file already there stays as it is,
845
+ // rather than being replaced by a placeholder.
846
+ const existing = existsSync(filePath) ? readFileSync(filePath, 'utf-8') : '';
847
+ if (existing && isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
848
+ warnings.push(`${pf.slug}: could not be written again — the earlier file is kept`);
849
+ return { slug: pf.slug, path: filePath, status: 'ai' };
850
+ }
747
851
  const built = buildTest(journey, false, ext, filePath, framework);
748
852
  content = built.content;
749
853
  status = 'basic';
@@ -809,8 +913,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
809
913
  if (useAI && dataAccess(config, packageRoot(areaDir)) === 'api') {
810
914
  writeRequiredData(seedDir, journey.id, journey.name, markdown);
811
915
  }
812
- else if (useAI && !force && existsSync(seedPath)) {
813
- seedStatus = 'ok'; // kept, like an existing test file
916
+ else if (useAI && !force && existsSync(seedPath) && !redoAll) {
917
+ seedStatus = 'ok'; // kept, like an existing test file — unless it was written before the code existed
814
918
  }
815
919
  else if (useAI && pending) {
816
920
  // Not live yet: there is nothing to seed for, and its tests make their own data.
@@ -18,7 +18,7 @@ import { readFileSync, existsSync, readdirSync } from 'node:fs';
18
18
  import { join } from 'node:path';
19
19
  import { loadConfig, splitCommand, withNoTools, withReadOnlyTools } from '../project/config.js';
20
20
  import { ruleIndex } from './journey-memory.js';
21
- import { MEMORY_DIR } from '../paths.js';
21
+ import { MEMORY_DIR, DOCS_DIR } from '../paths.js';
22
22
  import { runAiSync } from './ai-usage.js';
23
23
  const MAP_PATH = 'testforge.md';
24
24
  /**
@@ -28,7 +28,7 @@ const MAP_PATH = 'testforge.md';
28
28
  * well over half of all tests. With the siblings in view, each behaviour is
29
29
  * planned once, in the journey that owns it.
30
30
  */
31
- export function siblingJourneys(journey, docsDir = 'docs') {
31
+ export function siblingJourneys(journey, docsDir = DOCS_DIR) {
32
32
  const lines = [];
33
33
  const planned = new Set();
34
34
  const dir = join(MEMORY_DIR, journey.area);
@@ -66,18 +66,42 @@ export function spliceBlocks(content, edits) {
66
66
  }
67
67
  /** Add tests inside the file's outermost describe, before it closes; at the end when there is none. */
68
68
  export function appendTests(content, tests) {
69
- const d = /\btest\.describe(?:\.\w+)?\s*\(/.exec(content);
70
- if (d) {
71
- const close = matchParen(content, content.indexOf('(', d.index));
72
- if (close !== -1) {
73
- // The describe's body closes with "}" just before ")".
74
- const brace = content.lastIndexOf('}', close);
75
- if (brace > d.index)
76
- return `${content.slice(0, brace).trimEnd()}\n\n${indent(tests.trim(), ' ')}\n${content.slice(brace)}`;
77
- }
78
- }
69
+ const d = outerDescribe(content);
70
+ if (d)
71
+ return `${content.slice(0, d.brace).trimEnd()}\n\n${indent(tests.trim(), ' ')}\n${content.slice(d.brace)}`;
79
72
  return `${content.trimEnd()}\n\n${tests.trim()}\n`;
80
73
  }
74
+ /**
75
+ * The file's outer describe (Playwright's test.describe or Jest's describe):
76
+ * where it starts, the "}" closing its body and the ")" closing the call.
77
+ */
78
+ function outerDescribe(content) {
79
+ const d = /^(?:test\.)?describe(?:\.\w+)?\s*\(/m.exec(content) ?? /\b(?:test\.)?describe(?:\.\w+)?\s*\(/.exec(content);
80
+ if (!d)
81
+ return null;
82
+ const close = matchParen(content, content.indexOf('(', d.index));
83
+ if (close === -1)
84
+ return null;
85
+ // The describe's body closes with "}" just before ")".
86
+ const brace = content.lastIndexOf('}', close);
87
+ return brace > d.index ? { start: d.index, brace, close } : null;
88
+ }
89
+ /**
90
+ * Tests that ended up after the outer describe (earlier versions appended
91
+ * them there) are moved inside it, so they get the file's setup and cleanup.
92
+ * No AI; a file with nothing outside is returned as it is.
93
+ */
94
+ export function tuckStrayTests(content) {
95
+ const d = outerDescribe(content);
96
+ if (!d)
97
+ return content;
98
+ const stray = findTestBlocks(content).filter((b) => b.start > d.close);
99
+ if (!stray.length)
100
+ return content;
101
+ const moved = stray.map((b) => content.slice(b.start, b.end).trim()).join('\n\n');
102
+ const rest = spliceBlocks(content, stray.map((block) => ({ block, text: '' }))).replace(/\n{3,}/g, '\n\n');
103
+ return appendTests(rest, moved);
104
+ }
81
105
  const indent = (t, pad) => t.split('\n').map((l) => (l ? pad + l : l)).join('\n');
82
106
  /** Literal replacement of each old text with the new one, only inside the given blocks. */
83
107
  export function renameInBlocks(content, blocks, pairs) {
@@ -16,13 +16,15 @@ export const OLD_STATE_DIR = 'test-artifacts';
16
16
  export const CACHE_PATH = join(STATE_DIR, 'cache.json');
17
17
  export const SOURCE_PATH = join(STATE_DIR, 'source.json');
18
18
  export const COVERAGE_PATH = join(STATE_DIR, 'coverage.json');
19
- /** Everything TestForge remembers about each journey: <MEMORY_DIR>/<area>/<id>/ */
20
- export const MEMORY_DIR = join(STATE_DIR, 'memory');
19
+ /** The journey docs: <DOCS_DIR>/<area>/<journey>.md, and shared notes under _reference/. */
20
+ export const DOCS_DIR = join(STATE_DIR, 'docs');
21
+ /** Everything TestForge keeps about each journey: <MEMORY_DIR>/<area>/<id>/ */
22
+ export const MEMORY_DIR = join(STATE_DIR, 'journeys');
21
23
  /** One journey's folder: rules.json, plan.json, plan-check.md, facts.json. */
22
24
  export function journeyDir(journey) {
23
25
  return join(MEMORY_DIR, (journey.area || 'misc').replace(/[^a-z0-9-]/gi, '-'), journey.id);
24
26
  }
25
- /** The references read (Notion pages, ticket bundles) and the comparison with the map. */
27
+ /** Ticket bundles and the comparison with the map. Reference pages are read fresh, never kept here. */
26
28
  export const REFERENCE_DIR = join(STATE_DIR, 'reference');
27
29
  /** What one run leaves behind, kept on this machine only. */
28
30
  export const RUNS_DIR = join(STATE_DIR, 'runs');
@@ -43,6 +45,13 @@ export function migrateState() {
43
45
  moved.push(`${from} → ${to}`);
44
46
  };
45
47
  move('testforge.diff.md', join(REFERENCE_DIR, 'diff.md'));
48
+ move(join(STATE_DIR, 'memory'), MEMORY_DIR);
49
+ // Reference pages were kept as <12 hex>.md; they are read fresh now.
50
+ for (const f of readdirSafe(REFERENCE_DIR)) {
51
+ if (/^[0-9a-f]{12}\.md$/.test(f))
52
+ move(join(REFERENCE_DIR, f), join(RUNS_DIR, 'reference', f));
53
+ }
54
+ moveDocs(moved);
46
55
  const old = OLD_STATE_DIR;
47
56
  if (!existsSync(old) || !statSync(old).isDirectory())
48
57
  return moved;
@@ -72,6 +81,32 @@ export function migrateState() {
72
81
  removeIfEmpty(old);
73
82
  return moved;
74
83
  }
84
+ /**
85
+ * Journey docs from the project's docs/ into testforge/docs/: only what
86
+ * TestForge wrote (an <area>/ file opening with "# Journey:", and _reference/),
87
+ * never the project's own documents. Nothing already there is overwritten.
88
+ */
89
+ function moveDocs(moved) {
90
+ const from = 'docs';
91
+ for (const area of readdirSafe(from)) {
92
+ const dir = join(from, area);
93
+ if (!statSync(dir).isDirectory())
94
+ continue;
95
+ for (const f of readdirSafe(dir)) {
96
+ const src = join(dir, f);
97
+ const to = join(DOCS_DIR, area, f);
98
+ if (existsSync(to) || !statSync(src).isFile() || !f.endsWith('.md'))
99
+ continue;
100
+ if (area !== '_reference' && !/^#\s*Journey:/m.test(readFileSync(src, 'utf-8')))
101
+ continue;
102
+ mkdirSync(dirname(to), { recursive: true });
103
+ renameSync(src, to);
104
+ moved.push(`${src} → ${to}`);
105
+ }
106
+ removeIfEmpty(dir);
107
+ }
108
+ removeIfEmpty(from);
109
+ }
75
110
  function readdirSafe(dir) {
76
111
  try {
77
112
  return readdirSync(dir);
@@ -92,9 +127,9 @@ function removeIfEmpty(dir) {
92
127
  /** The ignore file TestForge used to write: everything local. Replaced when found. */
93
128
  const OLD_IGNORE = '# testforge local state: regenerated as needed, so not committed.\n*\n!.gitignore\n!source.json\n';
94
129
  const IGNORE = [
95
- '# testforge state. Commit it: each journey\'s memory (rules, plan, code facts) and',
96
- '# the references read cost AI to rebuild, and a teammate who pulls them starts',
97
- '# from where you left off. Only runs/ (run output, AI logs) stays on this machine.',
130
+ '# testforge state. Commit it: the journey docs and each journey\'s rules, plan and',
131
+ '# code facts cost AI to rebuild, and a teammate who pulls them starts from where',
132
+ '# you left off. Only runs/ (run output, AI logs) stays on this machine.',
98
133
  'runs/',
99
134
  '',
100
135
  ].join('\n');
@@ -282,6 +282,11 @@ export function loadEnvFile(path) {
282
282
  (value.startsWith("'") && value.endsWith("'"))) {
283
283
  value = value.slice(1, -1);
284
284
  }
285
+ else {
286
+ // An unquoted value ends at a comment: `KEY= # hint` (the template's
287
+ // unfilled line) is empty, not the hint.
288
+ value = value.replace(/(^|\s)#.*$/, '').trim();
289
+ }
285
290
  out[key] = value;
286
291
  }
287
292
  return out;
@@ -4,14 +4,14 @@
4
4
  * Runs the one-time project preparation in order, with progress so the
5
5
  * user can see what is happening at each stage:
6
6
  * 1. config + map — one call writes testforge.config.json and testforge.md
7
- * 2. discover — draft a doc per journey into docs/
7
+ * 2. discover — draft a doc per journey into testforge/docs/
8
8
  *
9
9
  * Each step is also available as its own command for re-running a
10
10
  * single stage; setup just chains them for a smooth first run.
11
11
  */
12
12
  import { existsSync } from 'node:fs';
13
13
  import { hashProjectSource, loadCache, saveCache } from '../docs/cache.js';
14
- import { CACHE_PATH } from '../paths.js';
14
+ import { CACHE_PATH, DOCS_DIR } from '../paths.js';
15
15
  import { checkSetupConfig, runSetupCall } from './init.js';
16
16
  import { loadConfig } from './config.js';
17
17
  import { runDiscover } from '../docs/discover.js';
@@ -21,13 +21,13 @@ export async function runSetup(srcDir, opts = {}) {
21
21
  // to `update` instead, which only refreshes what actually changed.
22
22
  const alreadySetUp = existsSync('testforge.config.json') &&
23
23
  existsSync('testforge.md') &&
24
- existsSync('docs');
24
+ existsSync(DOCS_DIR);
25
25
  if (alreadySetUp) {
26
26
  console.log('This project is already set up for testforge.');
27
- console.log(' - testforge.config.json, testforge.md and docs/ exist.\n');
27
+ console.log(' - testforge.config.json, testforge.md and testforge/docs/ exist.\n');
28
28
  console.log("You don't need to run setup again. Instead:");
29
29
  console.log(' testforge update refresh only what changed');
30
- console.log(' testforge generate (re)generate tests from docs/');
30
+ console.log(' testforge generate (re)generate tests from testforge/docs/');
31
31
  return;
32
32
  }
33
33
  console.log('Setting up testforge for this project.\n');
@@ -45,8 +45,8 @@ export async function runSetup(srcDir, opts = {}) {
45
45
  // Step 3 — discover. This is where the source-of-truth barrier lives:
46
46
  // the docs may be based on testforge.md or on an external reference.
47
47
  console.log('Step 2/2: Drafting a doc for each journey...');
48
- await runDiscover(srcDir, 'docs', opts);
49
- console.log(' done — docs/\n');
48
+ await runDiscover(srcDir, DOCS_DIR, opts);
49
+ console.log(' done — testforge/docs/\n');
50
50
  // Remember the source state so the first `update` can detect "no change".
51
51
  const cache = loadCache(CACHE_PATH);
52
52
  cache['__source__'] = hashProjectSource(srcDir);
@@ -54,12 +54,12 @@ export async function runSetup(srcDir, opts = {}) {
54
54
  // Build an env template so the developer knows which credentials each
55
55
  // area/journey needs for running the tests.
56
56
  console.log('Building env template...');
57
- runEnvTemplate('docs', '.env.testing.example', srcDir);
57
+ runEnvTemplate(DOCS_DIR, '.env.testing.example', srcDir);
58
58
  console.log('');
59
59
  console.log('Setup complete.');
60
60
  if (!loadConfig().dashboardUrl) {
61
61
  console.log('To publish runs to the TestForge dashboard, set "dashboardUrl" in testforge.config.json and TESTFORGE_KEY in the env file.');
62
62
  }
63
- console.log('Review the drafts in docs/, fix anything the AI got wrong,');
63
+ console.log('Review the drafts in testforge/docs/, fix anything the AI got wrong,');
64
64
  console.log('then run `testforge generate` to create tests and seed files.');
65
65
  }
@@ -15,7 +15,7 @@ import { runGenerate } from '../generate/run-generate.js';
15
15
  import { hashProjectSource, hashReference, loadCache, saveCache } from '../docs/cache.js';
16
16
  import { loadSourceChoice } from '../docs/source-of-truth.js';
17
17
  import { runEnvTemplate } from './env-template.js';
18
- import { CACHE_PATH } from '../paths.js';
18
+ import { CACHE_PATH, DOCS_DIR } from '../paths.js';
19
19
  const SOURCE_KEY = '__source__';
20
20
  const REFERENCE_KEY = '__reference__';
21
21
  export async function runUpdate(srcDir, opts = {}) {
@@ -69,7 +69,7 @@ export async function runUpdate(srcDir, opts = {}) {
69
69
  // Reuse the source of truth chosen at setup (map or a reference)
70
70
  // without prompting — an update is a refresh, not a re-decision.
71
71
  console.log('Updating journey docs...');
72
- await runDiscover(srcDir, 'docs', { noPrompt: true });
72
+ await runDiscover(srcDir, DOCS_DIR, { noPrompt: true });
73
73
  // Journeys that arrived on their own document — a ticket's attached spec —
74
74
  // are refreshed from THAT document. The project specification does not
75
75
  // describe them, so refreshing them from it would quietly replace a written
@@ -78,11 +78,11 @@ export async function runUpdate(srcDir, opts = {}) {
78
78
  for (const [doc, ref] of ownRefs) {
79
79
  const name = doc.replace(/\.md$/, '').split('/').pop();
80
80
  console.log(` refreshing ${name} from its own reference: ${ref}`);
81
- await runDiscover(srcDir, 'docs', { reference: ref, noPrompt: true }, name);
81
+ await runDiscover(srcDir, DOCS_DIR, { reference: ref, noPrompt: true }, name);
82
82
  }
83
- console.log(' done — docs/\n');
83
+ console.log(' done — testforge/docs/\n');
84
84
  console.log('Regenerating tests for changed journeys...');
85
- const results = await runGenerate('docs', true, false);
85
+ const results = await runGenerate(DOCS_DIR, true, false);
86
86
  const made = results.filter((r) => r.status !== 'skipped').length;
87
87
  const skipped = results.filter((r) => r.status === 'skipped').length;
88
88
  console.log(` done — ${made} updated, ${skipped} unchanged.\n`);
@@ -91,7 +91,7 @@ export async function runUpdate(srcDir, opts = {}) {
91
91
  cache[REFERENCE_KEY] = refState.hash;
92
92
  saveCache(CACHE_PATH, cache);
93
93
  console.log('Refreshing env template...');
94
- runEnvTemplate('docs', '.env.testing.example', srcDir);
94
+ runEnvTemplate(DOCS_DIR, '.env.testing.example', srcDir);
95
95
  console.log('');
96
96
  console.log('Update complete.');
97
97
  }