@activate-core/test-forge 2.7.1 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,8 +28,10 @@ import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
29
  import { rewriteSetupWithAI, buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
30
30
  import { completeFile, withoutBlankNames, withoutNames } from './complete-file.js';
31
- import { findSetupBlocks, findTestBlocks, tuckStrayTests, useInsertIds } from './test-blocks.js';
31
+ import { findSetupBlocks, findTestBlocks, tuckStrayTests, useInsertIds, withoutDuplicateTests } from './test-blocks.js';
32
32
  import { passedTitles } from './passed.js';
33
+ import { actionsPath, actionsStale, clearActionFailures, exportedNames, generateActionsAsync, journeysWith, stampActions } from './actions.js';
34
+ import { draftedByTestForge } from '../docs/doc-origin.js';
33
35
  import { scanNames } from '../run/name-scan.js';
34
36
  import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
35
37
  /** Names the file asks for that testforge/interface.json still has blank. */
@@ -114,7 +116,7 @@ import { updateJourney } from './update-journey.js';
114
116
  import { sourceRoot, sourceText } from '../docs/excerpts.js';
115
117
  import { isSyntaxValid, checkTestGates, trimExtraTests, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
116
118
  import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
117
- import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
119
+ import { harnessInfo, generateHarnessAsync, harnessGate, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
118
120
  import { hashContent, loadCache, saveCache } from '../docs/cache.js';
119
121
  import { findMarkdownFiles } from '../find-files.js';
120
122
  import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
@@ -345,7 +347,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
345
347
  console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
346
348
  }
347
349
  const old = existsSync(info.path) ? readFileSync(info.path, 'utf-8') : '';
348
- const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
350
+ const kit = await generateValidatedAsync((refusal) => generateHarnessAsync(info.path, framework, refusal), ext.endsWith('.js') ? '.js' : '.ts', (c) => harnessGate(c, framework));
349
351
  // The tests already written call the old testkit: a rebuild that fails, or
350
352
  // drops something they use, is not taken — the old one stays.
351
353
  const lost = old && kit !== null ? exportNames(old).filter((n) => !exportNames(kit).includes(n)) : [];
@@ -369,6 +371,28 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
369
371
  harnessByArea.set(area, info.path);
370
372
  return info.path;
371
373
  }
374
+ // The actions a journey's people take (book, pay, accept…), built once from the
375
+ // handler and the product's own client, so no test guesses a request's shape.
376
+ // Jest areas only. Built again when its contract, this journey's doc, or a
377
+ // refusal in the last run says so; a build that fails or drops an export the
378
+ // tests use is not taken.
379
+ async function ensureActions(harnessPath, journeyId, docPath, doc) {
380
+ const path = actionsPath(harnessPath);
381
+ const why = actionsStale(path, journeyId, doc);
382
+ if (!why)
383
+ return { path, built: false };
384
+ console.log(` actions: building (${why})`);
385
+ const old = existsSync(path) ? readFileSync(path, 'utf-8') : '';
386
+ const src = await generateValidatedAsync(() => generateActionsAsync(path, harnessPath, [docPath]), ext.endsWith('.ts') ? '.ts' : '.js');
387
+ const lost = old && src !== null ? exportedNames(old).filter((n) => !exportedNames(src).includes(n)) : [];
388
+ if (src === null || lost.length) {
389
+ console.log(` kept the existing actions: the build ${src === null ? 'failed' : `dropped ${lost.join(', ')}`}`);
390
+ return { path: old ? path : undefined, built: false };
391
+ }
392
+ writeFileSync(path, stampActions(src, journeysWith(path, journeyId, doc)), 'utf-8');
393
+ clearActionFailures();
394
+ return { path, built: true };
395
+ }
372
396
  const waitForLimit = opts.waitForLimit ?? config.waitForLimit ?? true;
373
397
  let notAttempted = 0;
374
398
  const redone = new Set();
@@ -513,7 +537,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
513
537
  const appTexts = () => framework === 'playwright'
514
538
  ? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
515
539
  : undefined;
516
- const fixTests = (content) => trimExtraTests(content, config.maxTestsPerRule ?? 2);
540
+ const fixTests = (content) => trimExtraTests(withoutDuplicateTests(content), config.maxTestsPerRule ?? 2);
517
541
  const genGate = (content, filePath) => testGate(content) ??
518
542
  checkTestGates(content, ext, {
519
543
  framework, forbidPlaceholders: true, filePath,
@@ -708,7 +732,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
708
732
  const parsed = useAI ? readRules(markdown) : undefined;
709
733
  const tested = testedTags();
710
734
  let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
711
- const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
735
+ const actions = useAI && harnessPath && framework === 'jest' ? await ensureActions(harnessPath, journey.id, join(docsDir, file), markdown) : { built: false };
736
+ const actionsFile = actions.path;
737
+ const actionsApi = actionsFile && existsSync(actionsFile)
738
+ ? `\n\nACTIONS, the module at ${actionsFile} (import it like the testkit, as act):\n${helperApi(readFileSync(actionsFile, 'utf-8'))}`
739
+ : '';
740
+ const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) + actionsApi : '';
712
741
  const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
713
742
  // What each file really tests once written: the memory and the report come from this, not the plan.
714
743
  const written = [];
@@ -745,6 +774,20 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
745
774
  harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
746
775
  check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
747
776
  });
777
+ // A file written again whole is taken only if it still tests every rule the
778
+ // earlier one did: a waiting test beats no test.
779
+ const previous = !kept && stale.has(filePath) && existsSync(filePath) ? readFileSync(filePath, 'utf-8') : undefined;
780
+ if (previous) {
781
+ const rules = pf.rules ?? [];
782
+ const had = rules.filter((r) => !ruleCoverage(previous, rules).missing.includes(r) && findTestBlocks(previous).some((t) => t.rules.includes(r)));
783
+ const lost = had.filter((r) => !done.tested.includes(r));
784
+ if (lost.length) {
785
+ warnings.push(`${pf.slug}: kept the earlier file — written again whole it lost the test for ${lost.join(', ')}`);
786
+ const cov = ruleCoverage(previous, rules);
787
+ written.push({ slug: pf.slug, tested: had, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [] });
788
+ return previous;
789
+ }
790
+ }
748
791
  written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
749
792
  // What was asked and still has no test is not asked again for this doc and code.
750
793
  if (done.refusedRules.length)
@@ -781,6 +824,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
781
824
  journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
782
825
  pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
783
826
  check: (c, p) => (isSyntaxValid(c, ext) ? genGate(c, p) : syntaxReason(c, ext)),
827
+ locked: (p) => passedTitles(p),
828
+ // Only a person's edit to the doc (or --force) may change a test that passed.
829
+ protectPassed: !force && draftedByTestForge(file, markdown),
784
830
  });
785
831
  for (const n of up.notes)
786
832
  console.log(` ${n}`);
@@ -887,10 +933,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
887
933
  catch {
888
934
  return false;
889
935
  } });
890
- // Only new code (or --refresh-code) rewrites a waiting file: facts read
891
- // again from the same code may word things differently, and that alone
892
- // must never cost a rewrite.
893
- if (waiting.length && (opts.refreshCode || codeMoved) && (await grounding())) {
936
+ // Only new code (or --refresh-code), or actions built this run, rewrite a
937
+ // waiting file: facts read again from the same code may word things
938
+ // differently, and that alone must never cost a rewrite.
939
+ if (waiting.length && (opts.refreshCode || codeMoved || actions.built) && (await grounding())) {
894
940
  for (const p of waiting)
895
941
  stale.add(p);
896
942
  if (stale.size) {
@@ -903,7 +949,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
903
949
  const setup = kinds.filter((k) => k === 'setup').length;
904
950
  const done = readRefused();
905
951
  const whole = [...stale].filter((p, i) => kinds[i] === 'file' && !(done[planned.find((pf) => pathOf(pf) === p)?.slug ?? ''] ?? []).includes('#whole')).length;
906
- warnings.push(`${stale.size} file(s) waiting on names the code now has: only their waiting tests are written again${setup ? `; the setup (hooks, helpers) of ${setup}` : ''}${whole ? `; ${whole} written again whole, once (their names sit in parametrised tables or loops)` : ''}`);
952
+ warnings.push(`${stale.size} file(s) waiting on names${actions.built ? ' (the actions module is new or rebuilt)' : ' the code now has'}: only their waiting tests are written again${setup ? `; the setup (hooks, helpers) of ${setup}` : ''}${whole ? `; ${whole} written again whole, once (their names sit in parametrised tables or loops)` : ''}`);
907
953
  }
908
954
  }
909
955
  }
@@ -992,7 +1038,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
992
1038
  if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
993
1039
  // A suite written before pending tests carried their reason gets
994
1040
  // it now, deterministically — no regeneration, no AI.
995
- const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
1041
+ // The same test appended twice (an earlier version did not see test.each) goes, without AI.
1042
+ const normalised = withoutDuplicateTests(framework === 'playwright' ? normalisePending(existing) : existing);
996
1043
  // A kept file that leaves rules out or holds tests that cannot fail:
997
1044
  // only those tests are written again.
998
1045
  const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath, true) : normalised, pending);
@@ -25,9 +25,27 @@ export function findTestBlocks(content) {
25
25
  const calls = aliases.length
26
26
  ? new RegExp(`(^|[^\\w.])(?:test|it|${aliases.join('|')})(?:\\.(?:only|fixme|skip|slow|fail))?\\s*\\(\\s*(['"\`])((?:(?!\\2).)*)\\2`, 'gm')
27
27
  : TEST_CALL;
28
+ // A parametrised test, test.each(table)('title %s', fn): one block from the
29
+ // table to the call's close. Missed, its rule read as untested and was asked
30
+ // for (and appended) again on every generate.
31
+ const names = ['test', 'it', ...aliases].join('|');
32
+ const each = new RegExp(`(^|[^\\w.])(?:${names})(?:\\.(?:only|skip|concurrent))?\\.each\\s*\\(`, 'gm');
33
+ const hits = [];
34
+ for (const m of content.matchAll(each)) {
35
+ const tableOpen = m.index + m[0].length - 1;
36
+ const tableClose = matchParen(content, tableOpen);
37
+ if (tableClose === -1)
38
+ continue;
39
+ const t = /^\s*\(\s*(['"`])((?:(?!\1).)*)\1/.exec(content.slice(tableClose + 1));
40
+ if (!t)
41
+ continue;
42
+ hits.push({ callAt: m.index + m[1].length, open: content.indexOf('(', tableClose + 1), title: t[2] });
43
+ }
28
44
  for (const m of content.matchAll(calls)) {
29
45
  const callAt = m.index + m[1].length;
30
- const open = content.indexOf('(', callAt);
46
+ hits.push({ callAt, open: content.indexOf('(', callAt), title: m[3] });
47
+ }
48
+ for (const { callAt, open, title } of hits) {
31
49
  const close = matchParen(content, open);
32
50
  if (close === -1)
33
51
  continue;
@@ -47,10 +65,26 @@ export function findTestBlocks(content) {
47
65
  }
48
66
  const text = content.slice(start, end);
49
67
  const rules = [...text.matchAll(RULE_ID)].flatMap((r) => r[1].match(/R\d+/g) ?? []);
50
- out.push({ start, end, title: m[3], rules: [...new Set(rules)] });
68
+ out.push({ start, end, title, rules: [...new Set(rules)] });
51
69
  }
70
+ out.sort((a, b) => a.start - b.start);
52
71
  return out.filter((b) => !out.some((o) => o !== b && o.start <= b.start && o.end >= b.end && o.end - o.start > b.end - b.start));
53
72
  }
73
+ /**
74
+ * The file with one test per title: a later test under a title an earlier one
75
+ * already has goes (the same test appended again). No AI.
76
+ */
77
+ export function withoutDuplicateTests(content) {
78
+ const seen = new Set();
79
+ const extra = findTestBlocks(content).filter((b) => {
80
+ const key = `${b.title}|${b.rules.join(',')}`;
81
+ if (seen.has(key))
82
+ return true;
83
+ seen.add(key);
84
+ return false;
85
+ });
86
+ return extra.length ? spliceBlocks(content, extra.map((block) => ({ block, text: '' }))) : content;
87
+ }
54
88
  /** Replace (or, with '', remove) the given blocks; later ones first so offsets stay valid. */
55
89
  export function spliceBlocks(content, edits) {
56
90
  let out = content;
@@ -44,23 +44,41 @@ export async function updateJourney(ctx) {
44
44
  const before = readFileSync(path, 'utf-8');
45
45
  let content = before;
46
46
  const mine = (ids) => ids.filter((id) => ctx.plan.find((p) => p.slug === f.slug)?.rules?.includes(id));
47
+ const locked = ctx.protectPassed && ctx.locked ? ctx.locked(path) : new Set();
48
+ const free = (blocks) => blocks.filter((b) => !locked.has(b.title));
49
+ const kept = new Set();
47
50
  // Renames: literal replacement inside the renamed rule's tests only. No AI.
48
51
  // A file written before tests named their rule gets the rename in every test.
49
52
  const legacy = !findTestBlocks(content).some((b) => b.rules.length);
50
53
  for (const r of diff.renamed.filter((x) => legacy || mine([x.id]).length)) {
51
- const res = renameInBlocks(content, legacy ? findTestBlocks(content) : blocksFor(content, [r.id]), r.pairs);
54
+ const targets = legacy ? findTestBlocks(content) : blocksFor(content, [r.id]);
55
+ for (const b of targets)
56
+ if (locked.has(b.title))
57
+ kept.add(b.title);
58
+ const res = renameInBlocks(content, free(targets), r.pairs);
52
59
  content = res.content;
53
60
  if (res.replaced)
54
61
  notes.push(`${f.slug}: ${r.id} renamed ${r.pairs.map(([a, b]) => `"${a}" → "${b}"`).join(', ')} (${res.replaced} place(s))`);
55
62
  }
56
63
  // Removed rules: their tests go, unless a test also checks a rule that stays.
57
- const gone = findTestBlocks(content).filter((b) => b.rules.length && b.rules.every((id) => removed.has(id)));
64
+ const goneAll = findTestBlocks(content).filter((b) => b.rules.length && b.rules.every((id) => removed.has(id)));
65
+ for (const b of goneAll)
66
+ if (locked.has(b.title))
67
+ kept.add(b.title);
68
+ const gone = free(goneAll);
58
69
  if (gone.length) {
59
70
  content = spliceBlocks(content, gone.map((block) => ({ block, text: '' })));
60
71
  notes.push(`${f.slug}: removed ${gone.length} test(s) of rules no longer in the doc`);
61
72
  }
62
73
  // Reworded rules and added rules: one small AI call for just those.
63
- const changed = mine(diff.changed).filter(testedId);
74
+ // A reworded rule whose test passed keeps that test when the rewording was
75
+ // TestForge's own: the passing test is the evidence, a redraft is not.
76
+ const changed = mine(diff.changed).filter(testedId).filter((id) => {
77
+ const passedOnes = blocksFor(content, [id]).filter((b) => locked.has(b.title));
78
+ for (const b of passedOnes)
79
+ kept.add(b.title);
80
+ return !passedOnes.length;
81
+ });
64
82
  const extra = toFile.get(f) ?? [];
65
83
  const ids = [...changed, ...extra];
66
84
  if (ids.length) {
@@ -106,6 +124,8 @@ export async function updateJourney(ctx) {
106
124
  notes.push(`${f.slug}: tests for ${ids.join(', ')} were refused (${refusal}) — the file keeps its old tests`);
107
125
  }
108
126
  }
127
+ if (kept.size)
128
+ notes.push(`${f.slug}: kept ${kept.size} test(s) that passed — the doc change came from TestForge's drafting, not a person (edit the doc yourself or use --force to change them)`);
109
129
  if (content !== before)
110
130
  writeFileSync(path, content, 'utf-8');
111
131
  }
@@ -0,0 +1,69 @@
1
+ /**
2
+ * testforge — is the AI CLI there before a command spends time on it?
3
+ *
4
+ * Checked with no AI call (no tokens): the configured program is on PATH and
5
+ * runs `--version`. Whether it is signed in can only be known by asking it
6
+ * something, so that shows up on the first real call; `friendlyAiError` turns
7
+ * that and a missing program into a plain message instead of a raw process
8
+ * error.
9
+ */
10
+ import { execFileSync } from 'node:child_process';
11
+ import { accessSync, constants } from 'node:fs';
12
+ import { delimiter, isAbsolute, join } from 'node:path';
13
+ import { loadConfig, splitCommand } from './config.js';
14
+ export class AiCliError extends Error {
15
+ }
16
+ /** The program's full path when it is runnable from here, else undefined. */
17
+ export function findOnPath(program, path = process.env.PATH ?? '') {
18
+ const runnable = (p) => { try {
19
+ accessSync(p, constants.X_OK);
20
+ return true;
21
+ }
22
+ catch {
23
+ return false;
24
+ } };
25
+ if (program.includes('/') || isAbsolute(program))
26
+ return runnable(program) ? program : undefined;
27
+ const exts = process.platform === 'win32' ? (process.env.PATHEXT ?? '.EXE;.CMD;.BAT').split(';') : [''];
28
+ for (const dir of path.split(delimiter).filter(Boolean)) {
29
+ for (const ext of exts) {
30
+ const p = join(dir, program + ext);
31
+ if (runnable(p))
32
+ return p;
33
+ }
34
+ }
35
+ return undefined;
36
+ }
37
+ /** Throws an AiCliError with what to do when the configured AI CLI cannot be run. */
38
+ export function preflightAi() {
39
+ const { program } = splitCommand(loadConfig().aiCommand);
40
+ if (!findOnPath(program)) {
41
+ throw new AiCliError(`Error: the AI CLI "${program}" was not found on PATH.\n` +
42
+ ` Install it (for Claude: npm i -g @anthropic-ai/claude-code, then run \`claude\` once to sign in),\n` +
43
+ ` or set "aiCommand" in testforge.config.json to the AI CLI you use.`);
44
+ }
45
+ try {
46
+ execFileSync(program, ['--version'], { stdio: 'pipe', timeout: 20_000 });
47
+ }
48
+ catch (e) {
49
+ const msg = String(e.stderr ?? e.message).split('\n')[0].slice(0, 200);
50
+ throw new AiCliError(`Error: the AI CLI "${program}" is installed but did not run (${msg}).`);
51
+ }
52
+ }
53
+ /** A plain message for an AI CLI failure, or undefined when the error is something else. */
54
+ export function friendlyAiError(err) {
55
+ const text = err instanceof Error ? `${err.message} ${String(err.stderr ?? '')}` : String(err);
56
+ const program = (() => { try {
57
+ return splitCommand(loadConfig().aiCommand).program;
58
+ }
59
+ catch {
60
+ return 'the AI CLI';
61
+ } })();
62
+ if (/spawn(?:Sync)? \S+ ENOENT/.test(text)) {
63
+ return `Error: the AI CLI "${program}" was not found. Install it, or set "aiCommand" in testforge.config.json.`;
64
+ }
65
+ if (/not logged in|please log ?in|unauthori[sz]ed|invalid api key|authentication/i.test(text)) {
66
+ return `Error: the AI CLI "${program}" is not signed in. Run \`${program}\` once and sign in (for Claude Code: /login), then try again.`;
67
+ }
68
+ return undefined;
69
+ }
@@ -1,12 +1,19 @@
1
1
  /**
2
2
  * testforge — update command.
3
3
  *
4
- * Refreshes a project that is already set up, doing only the work that
5
- * is actually needed. It decides whether anything changed by hashing
6
- * the project's SOURCE CODE (not the AI-generated map, which isn't
7
- * deterministic). If the code is unchanged, it does nothing — no AI
8
- * calls at all. If the code changed, it re-maps, re-discovers, and
9
- * regenerates (generate still skips individual docs that didn't change).
4
+ * Refreshes a project that is already set up, doing only the work that is
5
+ * actually needed. The journey docs are the source of truth for what the
6
+ * tests assert, so the code never rewrites them:
7
+ *
8
+ * - Code changed: the project map is refreshed, the names the tests wait on
9
+ * are looked up again (testforge/interface.json), and actions that waited
10
+ * for the product (or were refused in the last run) are written from the
11
+ * code. The docs and the tests are not touched.
12
+ * - The reference changed: reported, not acted on — `update --refresh-docs`
13
+ * redrafts the docs from it when a person decides to.
14
+ * - `--refresh-docs` or `--reference <x>`: the docs are redrafted from the
15
+ * reference, then the tests of what changed are regenerated (tests that
16
+ * passed are kept unless a person edits the doc or --force is given).
10
17
  */
11
18
  import { existsSync } from 'node:fs';
12
19
  import { runMap } from './map.js';
@@ -16,8 +23,87 @@ import { hashProjectSource, hashReference, loadCache, saveCache } from '../docs/
16
23
  import { loadSourceChoice } from '../docs/source-of-truth.js';
17
24
  import { runEnvTemplate } from './env-template.js';
18
25
  import { CACHE_PATH, DOCS_DIR } from '../paths.js';
26
+ import { readFileSync } from 'node:fs';
27
+ import { loadConfig, testDirForType } from './config.js';
28
+ import { testFilesIn } from '../run/readiness.js';
29
+ import { blankNames, neededNames } from '../generate/interface-file.js';
30
+ import { recheckFilled, scanNames } from '../run/name-scan.js';
31
+ import { basename, dirname, join } from 'node:path';
32
+ import { writeFileSync } from 'node:fs';
33
+ import { findMarkdownFiles } from '../find-files.js';
34
+ import { isSyntaxValid } from '../gates/validate.js';
35
+ import { clearActionFailures, exportedNames, generateActionsAsync, hasWaitingActions, readActionFailures, readStamp, stampActions } from '../generate/actions.js';
19
36
  const SOURCE_KEY = '__source__';
20
37
  const REFERENCE_KEY = '__reference__';
38
+ /**
39
+ * The names the tests wait on, looked up again in the code (a value the code
40
+ * no longer has is blanked first). Only testforge/interface.json changes.
41
+ */
42
+ function refreshNames(root) {
43
+ const config = loadConfig();
44
+ const files = Object.keys(config.testOutput ?? {}).flatMap((area) => {
45
+ const dir = testDirForType(config, area);
46
+ return existsSync(dir) ? testFilesIn(dir) : [];
47
+ });
48
+ for (const n of recheckFilled(root))
49
+ console.log(` no longer in the code: ${n} — blanked; its tests wait until the new name is found`);
50
+ const blank = blankNames([...new Set(files.flatMap((f) => { try {
51
+ return neededNames(readFileSync(f, 'utf-8'));
52
+ }
53
+ catch {
54
+ return [];
55
+ } }))]);
56
+ if (!blank.length) {
57
+ console.log(' no test waits on a name.');
58
+ return;
59
+ }
60
+ const res = scanNames(blank, root);
61
+ for (const f of res.filled)
62
+ console.log(` found in the code: ${f.name} = ${f.value}`);
63
+ console.log(` ${res.stillBlank.length} name(s) still blank (see testforge/interface.json).`);
64
+ }
65
+ /**
66
+ * The actions modules, once the code moved: an action that was waiting for the
67
+ * product, or that the last run had refused, is written from the code now.
68
+ * Only the actions module changes; the tests call the same functions and stop
69
+ * waiting by themselves.
70
+ */
71
+ async function refreshActions() {
72
+ const config = loadConfig();
73
+ const docs = existsSync(DOCS_DIR) ? findMarkdownFiles(DOCS_DIR) : [];
74
+ for (const area of Object.keys(config.testOutput ?? {})) {
75
+ const helpers = join(dirname(testDirForType(config, area)), 'helpers');
76
+ for (const ext of ['.js', '.ts']) {
77
+ const path = join(helpers, `actions${ext}`);
78
+ if (!existsSync(path))
79
+ continue;
80
+ const old = readFileSync(path, 'utf-8');
81
+ const failed = Object.keys(readActionFailures()).filter((n) => exportedNames(old).includes(n));
82
+ if (!hasWaitingActions(old) && !failed.length)
83
+ continue;
84
+ const stamp = readStamp(old);
85
+ const docPaths = docs.filter((d) => Object.keys(stamp.journeys).includes(basename(d).replace(/\.md$/, '')));
86
+ console.log(` actions (${area}): writing ${failed.length ? `${failed.join(', ')} again and ` : ''}the waiting actions from the code…`);
87
+ let src;
88
+ try {
89
+ src = await generateActionsAsync(path, join(helpers, `testkit${ext}`), docPaths);
90
+ }
91
+ catch (e) {
92
+ console.log(` not written — ${e.message.split('\n')[0].slice(0, 160)}; the actions stay as they were`);
93
+ continue;
94
+ }
95
+ const lost = exportedNames(old).filter((n) => !exportedNames(src).includes(n));
96
+ if (!isSyntaxValid(src, ext) || lost.length) {
97
+ console.log(` not taken — ${lost.length ? `it dropped ${lost.join(', ')}` : 'not valid code'}; the actions stay as they were`);
98
+ continue;
99
+ }
100
+ writeFileSync(path, stampActions(src, stamp.journeys), 'utf-8');
101
+ clearActionFailures();
102
+ const still = [...src.matchAll(/\bwaiting\(\s*['"`](\w+)/g)].map((m) => m[1]);
103
+ console.log(` done — the tests are unchanged${still.length ? `; still not in the product: ${[...new Set(still)].join(', ')}` : ''}`);
104
+ }
105
+ }
106
+ }
21
107
  export async function runUpdate(srcDir, opts = {}) {
22
108
  if (!existsSync('testforge.md') || !existsSync('testforge.config.json')) {
23
109
  console.log("This project isn't set up yet.");
@@ -39,7 +125,31 @@ export async function runUpdate(srcDir, opts = {}) {
39
125
  : { unreadable: 'docs are derived from the code map, not a reference' };
40
126
  const refChanged = Boolean(refState.hash) && cache[REFERENCE_KEY] !== undefined && cache[REFERENCE_KEY] !== refState.hash;
41
127
  const refFirstSeen = Boolean(refState.hash) && cache[REFERENCE_KEY] === undefined;
42
- if (!codeChanged && !refChanged && !opts.refreshDocs) {
128
+ const redraft = Boolean(opts.refreshDocs || opts.reference);
129
+ // Code moved, no redraft asked for: the map and the names, nothing else.
130
+ // A doc is a decision about behaviour; the code is not allowed to make it.
131
+ if (!redraft && (codeChanged || refChanged)) {
132
+ if (codeChanged) {
133
+ console.log('Source code changed — refreshing the project map and the names the tests wait on.');
134
+ console.log('(The journey docs and the tests are not changed: they follow the specification, not the code.)\n');
135
+ runMap(srcDir);
136
+ console.log(' done — testforge.md');
137
+ refreshNames('.');
138
+ await refreshActions();
139
+ cache[SOURCE_KEY] = newHash;
140
+ }
141
+ if (refChanged) {
142
+ console.log(`\nThe reference has changed: ${src.reference}`);
143
+ console.log(' Nothing was redrafted. Review the change, then run `testforge update --refresh-docs`');
144
+ console.log(' to redraft the docs from it (the tests of what changed are then regenerated).');
145
+ }
146
+ if (refFirstSeen)
147
+ cache[REFERENCE_KEY] = refState.hash;
148
+ saveCache(CACHE_PATH, cache);
149
+ console.log('\nUpdate complete.');
150
+ return;
151
+ }
152
+ if (!codeChanged && !refChanged && !opts.refreshDocs && !opts.reference) {
43
153
  console.log(`No source changes detected.${refState.hash ? ' The reference is unchanged too.' : ''}`);
44
154
  if (refState.unreadable && src.mode === 'reference') {
45
155
  // Say what was NOT checked. Claiming "up to date" while the specification
@@ -57,11 +167,7 @@ export async function runUpdate(srcDir, opts = {}) {
57
167
  }
58
168
  return;
59
169
  }
60
- const why = [
61
- codeChanged ? 'source code changed' : null,
62
- refChanged ? 'the reference changed' : null,
63
- opts.refreshDocs && !codeChanged && !refChanged ? 'refresh requested' : null,
64
- ].filter(Boolean).join(' and ');
170
+ const why = opts.reference ? `redrafting the docs from ${opts.reference}` : 'redrafting the docs (--refresh-docs)';
65
171
  console.log(`Updating the project — ${why}.\n`);
66
172
  console.log('Re-mapping the project...');
67
173
  runMap(srcDir);
@@ -69,12 +175,12 @@ export async function runUpdate(srcDir, opts = {}) {
69
175
  // Reuse the source of truth chosen at setup (map or a reference)
70
176
  // without prompting — an update is a refresh, not a re-decision.
71
177
  console.log('Updating journey docs...');
72
- await runDiscover(srcDir, DOCS_DIR, { noPrompt: true });
178
+ await runDiscover(srcDir, DOCS_DIR, opts.reference ? { reference: opts.reference, noPrompt: true, replace: true } : { noPrompt: true });
73
179
  // Journeys that arrived on their own document — a ticket's attached spec —
74
180
  // are refreshed from THAT document. The project specification does not
75
181
  // describe them, so refreshing them from it would quietly replace a written
76
182
  // requirement with whatever the code currently does.
77
- const ownRefs = Object.entries(src.journeys || {}).filter(([, ref]) => ref !== src.reference);
183
+ const ownRefs = opts.reference ? [] : Object.entries(src.journeys || {}).filter(([, ref]) => ref !== src.reference);
78
184
  for (const [doc, ref] of ownRefs) {
79
185
  const name = doc.replace(/\.md$/, '').split('/').pop();
80
186
  console.log(` refreshing ${name} from its own reference: ${ref}`);
@@ -191,6 +191,15 @@ E2. What you touch comes from <code_facts> and <testkit>.
191
191
  is a row made with kit.insert; the roles are the ones <testkit> lists.
192
192
  - Call testkit helpers exactly as <testkit> lists them, and never write your
193
193
  own copy.
194
+ - A step a person takes through the product (book, pay, accept, cancel…) is
195
+ the ACTIONS function <testkit> lists for it, never a request you write: pass
196
+ the business values it takes, and put what the rule varies (a bad or
197
+ missing field) in its { body } overrides. A test of a refusal passes
198
+ { allowRefusal: true } and asserts on the result's status and body. Assert
199
+ on the named results an action lists (orderId, orderStatus…), not on raw
200
+ body fields, so the test holds when the code arrives or moves. An action
201
+ "not in the product yet" is called all the same: the test waits until it
202
+ is. A request no action covers is written from <code_facts> as above.
194
203
 
195
204
  E3. What you need first comes from the testkit, or from the product's own way
196
205
  of making it.
@@ -199,6 +208,8 @@ of making it.
199
208
  type of each id.
200
209
  - Each role uses its own credential. Treat DUMMY, replace_me, an empty value
201
210
  or a <bracketed> value as absent.
211
+ - A request with no login passes role: null. A request with an old or
212
+ forged token passes that Authorization header: it is sent as given.
202
213
  - Owner-scoped data ("my orders") follows the token owner, not a query
203
214
  parameter. A non-owner check needs a second token; without one, skip and
204
215
  name the missing identity.
@@ -270,6 +281,12 @@ export const WEB = `WEB (Playwright)
270
281
  wording only when <code_facts> lack the element. Always await kit.find
271
282
  before using its result. Use page.getBy* only for what kit.find cannot
272
283
  express. Never use a data-testid that <code_facts> do not show.
284
+ - Never select by markup: no CSS class (page.locator('.btn')), no tag paths
285
+ or positions ('div > span', ':nth-child'), no XPath ('//button'). They break
286
+ when the page is restyled. When kit.find cannot express it, use
287
+ page.getByRole, getByLabel, getByText or getByTestId; an id or name
288
+ attribute that <code_facts> show ('#email', 'input[name="phone"]') is the
289
+ last resort, for an input with no label.
273
290
  - When a label also appears in an always-mounted part of the page (a header,
274
291
  a login pop-up), scope to the journey's own container. The SCREENS lines say
275
292
  which container renders each element.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.7.1",
3
+ "version": "2.9.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -429,16 +429,47 @@ function environmentFor(raw) {
429
429
  // by the tests of that endpoint, not by every test that signs in.
430
430
  const t = /testkit: token check (\w+ \S+) answered (5\d\d|429)\b/.exec(text);
431
431
  if (t) return `Environment: the server answered ${t[2]} to the testkit's sign-in check (${t[1]}), so this test could not start`;
432
+ // The API's rate limit stopped a testkit call: the run went too fast for the
433
+ // limiter, the product's rule was never reached.
434
+ const rl = /testkit: (\w+ \S+) answered 429\b/.exec(text);
435
+ if (rl) return `Environment: the API's rate limit answered 429 to the testkit (${rl[1]}) — clear the limiter or run fewer tests at once`;
436
+ // Nothing listening where the test connects (a server, database or cache that is down or not reachable).
437
+ const cr = /\bconnect ECONNREFUSED ([\w.:\[\]-]+)/.exec(text);
438
+ if (cr) return `Environment: nothing answered at ${cr[1]} (a server, database or cache that is down or not reachable from here), so this test could not run`;
432
439
  return undefined;
433
440
  }
434
441
 
435
- // Why a failed test is not the product's: it waits for a name, or the environment lacks something.
442
+ /** An action whose own request the product refused (refused() in interface.cjs): { action, detail }. */
443
+ function actionRefused(raw) {
444
+ const m = /\[testforge:action\] (\w+): ([^\n]*)/.exec(strip(String(raw || '')));
445
+ return m ? { action: m[1], detail: m[2].trim() } : undefined;
446
+ }
447
+
448
+ // Why a failed test is not the product's: it waits for a name, the environment
449
+ // lacks something, or a step before the rule failed.
436
450
  function notTheProduct(raw) {
437
451
  const waiting = waitingFor(raw);
438
452
  if (waiting) return `Waiting for the product: "${waiting}" is not in it yet`;
453
+ const a = actionRefused(raw);
454
+ if (a) return `Step failed before the rule was checked: action "${a.action}" was refused (${a.detail}). The next generate builds the action again from the code; refused again, the product is at fault`;
439
455
  return environmentFor(raw);
440
456
  }
441
457
 
458
+ /**
459
+ * The actions refused in this run (testforge/actions-failed.json), for generate
460
+ * to build again. Replaced each run: an action not refused this time is fine.
461
+ */
462
+ function recordActionFailures(root, raws) {
463
+ const found = {};
464
+ for (const raw of raws) { const a = actionRefused(raw); if (a && !found[a.action]) found[a.action] = a.detail; }
465
+ const file = path.join(root, 'testforge', 'actions-failed.json');
466
+ try {
467
+ if (!Object.keys(found).length) { fs.rmSync(file, { force: true }); return; }
468
+ fs.mkdirSync(path.dirname(file), { recursive: true });
469
+ fs.writeFileSync(file, JSON.stringify(found, null, 2) + '\n');
470
+ } catch { /* the run's own output still names them */ }
471
+ }
472
+
442
473
  /**
443
474
  * Remember the tests that passed (testforge/passed.json): generate never removes
444
475
  * or rewrites a test that has passed. Add-only — a later failure is a regression
@@ -484,4 +515,4 @@ function recordSummary(root, category, suites) {
484
515
  } catch { /* the run's own output still has the counts */ }
485
516
  }
486
517
 
487
- module.exports = { recordSummary, recordPassed, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
518
+ module.exports = { recordSummary, recordPassed, recordActionFailures, actionRefused, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };