@activate-core/test-forge 2.7.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +21 -0
- package/dist/cli/index.js +10 -3
- package/dist/core/docs/discover.js +15 -0
- package/dist/core/docs/doc-origin.js +42 -0
- package/dist/core/docs/reference.js +18 -1
- package/dist/core/gates/validate.js +8 -0
- package/dist/core/generate/actions.js +200 -0
- package/dist/core/generate/complete-file.js +43 -0
- package/dist/core/generate/harness.js +57 -14
- package/dist/core/generate/run-generate.js +58 -11
- package/dist/core/generate/test-blocks.js +36 -2
- package/dist/core/generate/update-journey.js +23 -3
- package/dist/core/project/preflight.js +69 -0
- package/dist/core/project/update.js +120 -14
- package/dist/core/prompts/blocks.js +17 -0
- package/package.json +1 -1
- package/reporters/_shared.cjs +33 -2
- package/reporters/http.cjs +96 -0
- package/reporters/interface.cjs +25 -2
- package/reporters/jest-reporter.cjs +3 -1
|
@@ -28,8 +28,10 @@ import { generateTestCases } from './generator.js';
|
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
29
|
import { rewriteSetupWithAI, buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
30
|
import { completeFile, withoutBlankNames, withoutNames } from './complete-file.js';
|
|
31
|
-
import { findSetupBlocks, findTestBlocks, tuckStrayTests, useInsertIds } from './test-blocks.js';
|
|
31
|
+
import { findSetupBlocks, findTestBlocks, tuckStrayTests, useInsertIds, withoutDuplicateTests } from './test-blocks.js';
|
|
32
32
|
import { passedTitles } from './passed.js';
|
|
33
|
+
import { actionsPath, actionsStale, clearActionFailures, exportedNames, generateActionsAsync, journeysWith, stampActions } from './actions.js';
|
|
34
|
+
import { draftedByTestForge } from '../docs/doc-origin.js';
|
|
33
35
|
import { scanNames } from '../run/name-scan.js';
|
|
34
36
|
import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
|
|
35
37
|
/** Names the file asks for that testforge/interface.json still has blank. */
|
|
@@ -114,7 +116,7 @@ import { updateJourney } from './update-journey.js';
|
|
|
114
116
|
import { sourceRoot, sourceText } from '../docs/excerpts.js';
|
|
115
117
|
import { isSyntaxValid, checkTestGates, trimExtraTests, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
116
118
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
117
|
-
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
119
|
+
import { harnessInfo, generateHarnessAsync, harnessGate, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
118
120
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
119
121
|
import { findMarkdownFiles } from '../find-files.js';
|
|
120
122
|
import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
|
|
@@ -345,7 +347,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
345
347
|
console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
|
|
346
348
|
}
|
|
347
349
|
const old = existsSync(info.path) ? readFileSync(info.path, 'utf-8') : '';
|
|
348
|
-
const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
|
|
350
|
+
const kit = await generateValidatedAsync((refusal) => generateHarnessAsync(info.path, framework, refusal), ext.endsWith('.js') ? '.js' : '.ts', (c) => harnessGate(c, framework));
|
|
349
351
|
// The tests already written call the old testkit: a rebuild that fails, or
|
|
350
352
|
// drops something they use, is not taken — the old one stays.
|
|
351
353
|
const lost = old && kit !== null ? exportNames(old).filter((n) => !exportNames(kit).includes(n)) : [];
|
|
@@ -369,6 +371,28 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
369
371
|
harnessByArea.set(area, info.path);
|
|
370
372
|
return info.path;
|
|
371
373
|
}
|
|
374
|
+
// The actions a journey's people take (book, pay, accept…), built once from the
|
|
375
|
+
// handler and the product's own client, so no test guesses a request's shape.
|
|
376
|
+
// Jest areas only. Built again when its contract, this journey's doc, or a
|
|
377
|
+
// refusal in the last run says so; a build that fails or drops an export the
|
|
378
|
+
// tests use is not taken.
|
|
379
|
+
async function ensureActions(harnessPath, journeyId, docPath, doc) {
|
|
380
|
+
const path = actionsPath(harnessPath);
|
|
381
|
+
const why = actionsStale(path, journeyId, doc);
|
|
382
|
+
if (!why)
|
|
383
|
+
return { path, built: false };
|
|
384
|
+
console.log(` actions: building (${why})`);
|
|
385
|
+
const old = existsSync(path) ? readFileSync(path, 'utf-8') : '';
|
|
386
|
+
const src = await generateValidatedAsync(() => generateActionsAsync(path, harnessPath, [docPath]), ext.endsWith('.ts') ? '.ts' : '.js');
|
|
387
|
+
const lost = old && src !== null ? exportedNames(old).filter((n) => !exportedNames(src).includes(n)) : [];
|
|
388
|
+
if (src === null || lost.length) {
|
|
389
|
+
console.log(` kept the existing actions: the build ${src === null ? 'failed' : `dropped ${lost.join(', ')}`}`);
|
|
390
|
+
return { path: old ? path : undefined, built: false };
|
|
391
|
+
}
|
|
392
|
+
writeFileSync(path, stampActions(src, journeysWith(path, journeyId, doc)), 'utf-8');
|
|
393
|
+
clearActionFailures();
|
|
394
|
+
return { path, built: true };
|
|
395
|
+
}
|
|
372
396
|
const waitForLimit = opts.waitForLimit ?? config.waitForLimit ?? true;
|
|
373
397
|
let notAttempted = 0;
|
|
374
398
|
const redone = new Set();
|
|
@@ -513,7 +537,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
513
537
|
const appTexts = () => framework === 'playwright'
|
|
514
538
|
? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
515
539
|
: undefined;
|
|
516
|
-
const fixTests = (content) => trimExtraTests(content, config.maxTestsPerRule ?? 2);
|
|
540
|
+
const fixTests = (content) => trimExtraTests(withoutDuplicateTests(content), config.maxTestsPerRule ?? 2);
|
|
517
541
|
const genGate = (content, filePath) => testGate(content) ??
|
|
518
542
|
checkTestGates(content, ext, {
|
|
519
543
|
framework, forbidPlaceholders: true, filePath,
|
|
@@ -708,7 +732,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
708
732
|
const parsed = useAI ? readRules(markdown) : undefined;
|
|
709
733
|
const tested = testedTags();
|
|
710
734
|
let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
|
|
711
|
-
const
|
|
735
|
+
const actions = useAI && harnessPath && framework === 'jest' ? await ensureActions(harnessPath, journey.id, join(docsDir, file), markdown) : { built: false };
|
|
736
|
+
const actionsFile = actions.path;
|
|
737
|
+
const actionsApi = actionsFile && existsSync(actionsFile)
|
|
738
|
+
? `\n\nACTIONS, the module at ${actionsFile} (import it like the testkit, as act):\n${helperApi(readFileSync(actionsFile, 'utf-8'))}`
|
|
739
|
+
: '';
|
|
740
|
+
const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) + actionsApi : '';
|
|
712
741
|
const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
|
|
713
742
|
// What each file really tests once written: the memory and the report come from this, not the plan.
|
|
714
743
|
const written = [];
|
|
@@ -745,6 +774,20 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
745
774
|
harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
|
|
746
775
|
check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
|
|
747
776
|
});
|
|
777
|
+
// A file written again whole is taken only if it still tests every rule the
|
|
778
|
+
// earlier one did: a waiting test beats no test.
|
|
779
|
+
const previous = !kept && stale.has(filePath) && existsSync(filePath) ? readFileSync(filePath, 'utf-8') : undefined;
|
|
780
|
+
if (previous) {
|
|
781
|
+
const rules = pf.rules ?? [];
|
|
782
|
+
const had = rules.filter((r) => !ruleCoverage(previous, rules).missing.includes(r) && findTestBlocks(previous).some((t) => t.rules.includes(r)));
|
|
783
|
+
const lost = had.filter((r) => !done.tested.includes(r));
|
|
784
|
+
if (lost.length) {
|
|
785
|
+
warnings.push(`${pf.slug}: kept the earlier file — written again whole it lost the test for ${lost.join(', ')}`);
|
|
786
|
+
const cov = ruleCoverage(previous, rules);
|
|
787
|
+
written.push({ slug: pf.slug, tested: had, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [] });
|
|
788
|
+
return previous;
|
|
789
|
+
}
|
|
790
|
+
}
|
|
748
791
|
written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
|
|
749
792
|
// What was asked and still has no test is not asked again for this doc and code.
|
|
750
793
|
if (done.refusedRules.length)
|
|
@@ -781,6 +824,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
781
824
|
journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
|
|
782
825
|
pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
|
|
783
826
|
check: (c, p) => (isSyntaxValid(c, ext) ? genGate(c, p) : syntaxReason(c, ext)),
|
|
827
|
+
locked: (p) => passedTitles(p),
|
|
828
|
+
// Only a person's edit to the doc (or --force) may change a test that passed.
|
|
829
|
+
protectPassed: !force && draftedByTestForge(file, markdown),
|
|
784
830
|
});
|
|
785
831
|
for (const n of up.notes)
|
|
786
832
|
console.log(` ${n}`);
|
|
@@ -887,10 +933,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
887
933
|
catch {
|
|
888
934
|
return false;
|
|
889
935
|
} });
|
|
890
|
-
// Only new code (or --refresh-code)
|
|
891
|
-
// again from the same code may word things
|
|
892
|
-
// must never cost a rewrite.
|
|
893
|
-
if (waiting.length && (opts.refreshCode || codeMoved) && (await grounding())) {
|
|
936
|
+
// Only new code (or --refresh-code), or actions built this run, rewrite a
|
|
937
|
+
// waiting file: facts read again from the same code may word things
|
|
938
|
+
// differently, and that alone must never cost a rewrite.
|
|
939
|
+
if (waiting.length && (opts.refreshCode || codeMoved || actions.built) && (await grounding())) {
|
|
894
940
|
for (const p of waiting)
|
|
895
941
|
stale.add(p);
|
|
896
942
|
if (stale.size) {
|
|
@@ -903,7 +949,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
903
949
|
const setup = kinds.filter((k) => k === 'setup').length;
|
|
904
950
|
const done = readRefused();
|
|
905
951
|
const whole = [...stale].filter((p, i) => kinds[i] === 'file' && !(done[planned.find((pf) => pathOf(pf) === p)?.slug ?? ''] ?? []).includes('#whole')).length;
|
|
906
|
-
warnings.push(`${stale.size} file(s) waiting on names the code now has: only their waiting tests are written again${setup ? `; the setup (hooks, helpers) of ${setup}` : ''}${whole ? `; ${whole} written again whole, once (their names sit in parametrised tables or loops)` : ''}`);
|
|
952
|
+
warnings.push(`${stale.size} file(s) waiting on names${actions.built ? ' (the actions module is new or rebuilt)' : ' the code now has'}: only their waiting tests are written again${setup ? `; the setup (hooks, helpers) of ${setup}` : ''}${whole ? `; ${whole} written again whole, once (their names sit in parametrised tables or loops)` : ''}`);
|
|
907
953
|
}
|
|
908
954
|
}
|
|
909
955
|
}
|
|
@@ -992,7 +1038,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
992
1038
|
if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
|
|
993
1039
|
// A suite written before pending tests carried their reason gets
|
|
994
1040
|
// it now, deterministically — no regeneration, no AI.
|
|
995
|
-
|
|
1041
|
+
// The same test appended twice (an earlier version did not see test.each) goes, without AI.
|
|
1042
|
+
const normalised = withoutDuplicateTests(framework === 'playwright' ? normalisePending(existing) : existing);
|
|
996
1043
|
// A kept file that leaves rules out or holds tests that cannot fail:
|
|
997
1044
|
// only those tests are written again.
|
|
998
1045
|
const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath, true) : normalised, pending);
|
|
@@ -25,9 +25,27 @@ export function findTestBlocks(content) {
|
|
|
25
25
|
const calls = aliases.length
|
|
26
26
|
? new RegExp(`(^|[^\\w.])(?:test|it|${aliases.join('|')})(?:\\.(?:only|fixme|skip|slow|fail))?\\s*\\(\\s*(['"\`])((?:(?!\\2).)*)\\2`, 'gm')
|
|
27
27
|
: TEST_CALL;
|
|
28
|
+
// A parametrised test, test.each(table)('title %s', fn): one block from the
|
|
29
|
+
// table to the call's close. Missed, its rule read as untested and was asked
|
|
30
|
+
// for (and appended) again on every generate.
|
|
31
|
+
const names = ['test', 'it', ...aliases].join('|');
|
|
32
|
+
const each = new RegExp(`(^|[^\\w.])(?:${names})(?:\\.(?:only|skip|concurrent))?\\.each\\s*\\(`, 'gm');
|
|
33
|
+
const hits = [];
|
|
34
|
+
for (const m of content.matchAll(each)) {
|
|
35
|
+
const tableOpen = m.index + m[0].length - 1;
|
|
36
|
+
const tableClose = matchParen(content, tableOpen);
|
|
37
|
+
if (tableClose === -1)
|
|
38
|
+
continue;
|
|
39
|
+
const t = /^\s*\(\s*(['"`])((?:(?!\1).)*)\1/.exec(content.slice(tableClose + 1));
|
|
40
|
+
if (!t)
|
|
41
|
+
continue;
|
|
42
|
+
hits.push({ callAt: m.index + m[1].length, open: content.indexOf('(', tableClose + 1), title: t[2] });
|
|
43
|
+
}
|
|
28
44
|
for (const m of content.matchAll(calls)) {
|
|
29
45
|
const callAt = m.index + m[1].length;
|
|
30
|
-
|
|
46
|
+
hits.push({ callAt, open: content.indexOf('(', callAt), title: m[3] });
|
|
47
|
+
}
|
|
48
|
+
for (const { callAt, open, title } of hits) {
|
|
31
49
|
const close = matchParen(content, open);
|
|
32
50
|
if (close === -1)
|
|
33
51
|
continue;
|
|
@@ -47,10 +65,26 @@ export function findTestBlocks(content) {
|
|
|
47
65
|
}
|
|
48
66
|
const text = content.slice(start, end);
|
|
49
67
|
const rules = [...text.matchAll(RULE_ID)].flatMap((r) => r[1].match(/R\d+/g) ?? []);
|
|
50
|
-
out.push({ start, end, title
|
|
68
|
+
out.push({ start, end, title, rules: [...new Set(rules)] });
|
|
51
69
|
}
|
|
70
|
+
out.sort((a, b) => a.start - b.start);
|
|
52
71
|
return out.filter((b) => !out.some((o) => o !== b && o.start <= b.start && o.end >= b.end && o.end - o.start > b.end - b.start));
|
|
53
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* The file with one test per title: a later test under a title an earlier one
|
|
75
|
+
* already has goes (the same test appended again). No AI.
|
|
76
|
+
*/
|
|
77
|
+
export function withoutDuplicateTests(content) {
|
|
78
|
+
const seen = new Set();
|
|
79
|
+
const extra = findTestBlocks(content).filter((b) => {
|
|
80
|
+
const key = `${b.title}|${b.rules.join(',')}`;
|
|
81
|
+
if (seen.has(key))
|
|
82
|
+
return true;
|
|
83
|
+
seen.add(key);
|
|
84
|
+
return false;
|
|
85
|
+
});
|
|
86
|
+
return extra.length ? spliceBlocks(content, extra.map((block) => ({ block, text: '' }))) : content;
|
|
87
|
+
}
|
|
54
88
|
/** Replace (or, with '', remove) the given blocks; later ones first so offsets stay valid. */
|
|
55
89
|
export function spliceBlocks(content, edits) {
|
|
56
90
|
let out = content;
|
|
@@ -44,23 +44,41 @@ export async function updateJourney(ctx) {
|
|
|
44
44
|
const before = readFileSync(path, 'utf-8');
|
|
45
45
|
let content = before;
|
|
46
46
|
const mine = (ids) => ids.filter((id) => ctx.plan.find((p) => p.slug === f.slug)?.rules?.includes(id));
|
|
47
|
+
const locked = ctx.protectPassed && ctx.locked ? ctx.locked(path) : new Set();
|
|
48
|
+
const free = (blocks) => blocks.filter((b) => !locked.has(b.title));
|
|
49
|
+
const kept = new Set();
|
|
47
50
|
// Renames: literal replacement inside the renamed rule's tests only. No AI.
|
|
48
51
|
// A file written before tests named their rule gets the rename in every test.
|
|
49
52
|
const legacy = !findTestBlocks(content).some((b) => b.rules.length);
|
|
50
53
|
for (const r of diff.renamed.filter((x) => legacy || mine([x.id]).length)) {
|
|
51
|
-
const
|
|
54
|
+
const targets = legacy ? findTestBlocks(content) : blocksFor(content, [r.id]);
|
|
55
|
+
for (const b of targets)
|
|
56
|
+
if (locked.has(b.title))
|
|
57
|
+
kept.add(b.title);
|
|
58
|
+
const res = renameInBlocks(content, free(targets), r.pairs);
|
|
52
59
|
content = res.content;
|
|
53
60
|
if (res.replaced)
|
|
54
61
|
notes.push(`${f.slug}: ${r.id} renamed ${r.pairs.map(([a, b]) => `"${a}" → "${b}"`).join(', ')} (${res.replaced} place(s))`);
|
|
55
62
|
}
|
|
56
63
|
// Removed rules: their tests go, unless a test also checks a rule that stays.
|
|
57
|
-
const
|
|
64
|
+
const goneAll = findTestBlocks(content).filter((b) => b.rules.length && b.rules.every((id) => removed.has(id)));
|
|
65
|
+
for (const b of goneAll)
|
|
66
|
+
if (locked.has(b.title))
|
|
67
|
+
kept.add(b.title);
|
|
68
|
+
const gone = free(goneAll);
|
|
58
69
|
if (gone.length) {
|
|
59
70
|
content = spliceBlocks(content, gone.map((block) => ({ block, text: '' })));
|
|
60
71
|
notes.push(`${f.slug}: removed ${gone.length} test(s) of rules no longer in the doc`);
|
|
61
72
|
}
|
|
62
73
|
// Reworded rules and added rules: one small AI call for just those.
|
|
63
|
-
|
|
74
|
+
// A reworded rule whose test passed keeps that test when the rewording was
|
|
75
|
+
// TestForge's own: the passing test is the evidence, a redraft is not.
|
|
76
|
+
const changed = mine(diff.changed).filter(testedId).filter((id) => {
|
|
77
|
+
const passedOnes = blocksFor(content, [id]).filter((b) => locked.has(b.title));
|
|
78
|
+
for (const b of passedOnes)
|
|
79
|
+
kept.add(b.title);
|
|
80
|
+
return !passedOnes.length;
|
|
81
|
+
});
|
|
64
82
|
const extra = toFile.get(f) ?? [];
|
|
65
83
|
const ids = [...changed, ...extra];
|
|
66
84
|
if (ids.length) {
|
|
@@ -106,6 +124,8 @@ export async function updateJourney(ctx) {
|
|
|
106
124
|
notes.push(`${f.slug}: tests for ${ids.join(', ')} were refused (${refusal}) — the file keeps its old tests`);
|
|
107
125
|
}
|
|
108
126
|
}
|
|
127
|
+
if (kept.size)
|
|
128
|
+
notes.push(`${f.slug}: kept ${kept.size} test(s) that passed — the doc change came from TestForge's drafting, not a person (edit the doc yourself or use --force to change them)`);
|
|
109
129
|
if (content !== before)
|
|
110
130
|
writeFileSync(path, content, 'utf-8');
|
|
111
131
|
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — is the AI CLI there before a command spends time on it?
|
|
3
|
+
*
|
|
4
|
+
* Checked with no AI call (no tokens): the configured program is on PATH and
|
|
5
|
+
* runs `--version`. Whether it is signed in can only be known by asking it
|
|
6
|
+
* something, so that shows up on the first real call; `friendlyAiError` turns
|
|
7
|
+
* that and a missing program into a plain message instead of a raw process
|
|
8
|
+
* error.
|
|
9
|
+
*/
|
|
10
|
+
import { execFileSync } from 'node:child_process';
|
|
11
|
+
import { accessSync, constants } from 'node:fs';
|
|
12
|
+
import { delimiter, isAbsolute, join } from 'node:path';
|
|
13
|
+
import { loadConfig, splitCommand } from './config.js';
|
|
14
|
+
export class AiCliError extends Error {
|
|
15
|
+
}
|
|
16
|
+
/** The program's full path when it is runnable from here, else undefined. */
|
|
17
|
+
export function findOnPath(program, path = process.env.PATH ?? '') {
|
|
18
|
+
const runnable = (p) => { try {
|
|
19
|
+
accessSync(p, constants.X_OK);
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return false;
|
|
24
|
+
} };
|
|
25
|
+
if (program.includes('/') || isAbsolute(program))
|
|
26
|
+
return runnable(program) ? program : undefined;
|
|
27
|
+
const exts = process.platform === 'win32' ? (process.env.PATHEXT ?? '.EXE;.CMD;.BAT').split(';') : [''];
|
|
28
|
+
for (const dir of path.split(delimiter).filter(Boolean)) {
|
|
29
|
+
for (const ext of exts) {
|
|
30
|
+
const p = join(dir, program + ext);
|
|
31
|
+
if (runnable(p))
|
|
32
|
+
return p;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
/** Throws an AiCliError with what to do when the configured AI CLI cannot be run. */
|
|
38
|
+
export function preflightAi() {
|
|
39
|
+
const { program } = splitCommand(loadConfig().aiCommand);
|
|
40
|
+
if (!findOnPath(program)) {
|
|
41
|
+
throw new AiCliError(`Error: the AI CLI "${program}" was not found on PATH.\n` +
|
|
42
|
+
` Install it (for Claude: npm i -g @anthropic-ai/claude-code, then run \`claude\` once to sign in),\n` +
|
|
43
|
+
` or set "aiCommand" in testforge.config.json to the AI CLI you use.`);
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
execFileSync(program, ['--version'], { stdio: 'pipe', timeout: 20_000 });
|
|
47
|
+
}
|
|
48
|
+
catch (e) {
|
|
49
|
+
const msg = String(e.stderr ?? e.message).split('\n')[0].slice(0, 200);
|
|
50
|
+
throw new AiCliError(`Error: the AI CLI "${program}" is installed but did not run (${msg}).`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** A plain message for an AI CLI failure, or undefined when the error is something else. */
|
|
54
|
+
export function friendlyAiError(err) {
|
|
55
|
+
const text = err instanceof Error ? `${err.message} ${String(err.stderr ?? '')}` : String(err);
|
|
56
|
+
const program = (() => { try {
|
|
57
|
+
return splitCommand(loadConfig().aiCommand).program;
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return 'the AI CLI';
|
|
61
|
+
} })();
|
|
62
|
+
if (/spawn(?:Sync)? \S+ ENOENT/.test(text)) {
|
|
63
|
+
return `Error: the AI CLI "${program}" was not found. Install it, or set "aiCommand" in testforge.config.json.`;
|
|
64
|
+
}
|
|
65
|
+
if (/not logged in|please log ?in|unauthori[sz]ed|invalid api key|authentication/i.test(text)) {
|
|
66
|
+
return `Error: the AI CLI "${program}" is not signed in. Run \`${program}\` once and sign in (for Claude Code: /login), then try again.`;
|
|
67
|
+
}
|
|
68
|
+
return undefined;
|
|
69
|
+
}
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* testforge — update command.
|
|
3
3
|
*
|
|
4
|
-
* Refreshes a project that is already set up, doing only the work that
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* Refreshes a project that is already set up, doing only the work that is
|
|
5
|
+
* actually needed. The journey docs are the source of truth for what the
|
|
6
|
+
* tests assert, so the code never rewrites them:
|
|
7
|
+
*
|
|
8
|
+
* - Code changed: the project map is refreshed, the names the tests wait on
|
|
9
|
+
* are looked up again (testforge/interface.json), and actions that waited
|
|
10
|
+
* for the product (or were refused in the last run) are written from the
|
|
11
|
+
* code. The docs and the tests are not touched.
|
|
12
|
+
* - The reference changed: reported, not acted on — `update --refresh-docs`
|
|
13
|
+
* redrafts the docs from it when a person decides to.
|
|
14
|
+
* - `--refresh-docs` or `--reference <x>`: the docs are redrafted from the
|
|
15
|
+
* reference, then the tests of what changed are regenerated (tests that
|
|
16
|
+
* passed are kept unless a person edits the doc or --force is given).
|
|
10
17
|
*/
|
|
11
18
|
import { existsSync } from 'node:fs';
|
|
12
19
|
import { runMap } from './map.js';
|
|
@@ -16,8 +23,87 @@ import { hashProjectSource, hashReference, loadCache, saveCache } from '../docs/
|
|
|
16
23
|
import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
17
24
|
import { runEnvTemplate } from './env-template.js';
|
|
18
25
|
import { CACHE_PATH, DOCS_DIR } from '../paths.js';
|
|
26
|
+
import { readFileSync } from 'node:fs';
|
|
27
|
+
import { loadConfig, testDirForType } from './config.js';
|
|
28
|
+
import { testFilesIn } from '../run/readiness.js';
|
|
29
|
+
import { blankNames, neededNames } from '../generate/interface-file.js';
|
|
30
|
+
import { recheckFilled, scanNames } from '../run/name-scan.js';
|
|
31
|
+
import { basename, dirname, join } from 'node:path';
|
|
32
|
+
import { writeFileSync } from 'node:fs';
|
|
33
|
+
import { findMarkdownFiles } from '../find-files.js';
|
|
34
|
+
import { isSyntaxValid } from '../gates/validate.js';
|
|
35
|
+
import { clearActionFailures, exportedNames, generateActionsAsync, hasWaitingActions, readActionFailures, readStamp, stampActions } from '../generate/actions.js';
|
|
19
36
|
const SOURCE_KEY = '__source__';
|
|
20
37
|
const REFERENCE_KEY = '__reference__';
|
|
38
|
+
/**
|
|
39
|
+
* The names the tests wait on, looked up again in the code (a value the code
|
|
40
|
+
* no longer has is blanked first). Only testforge/interface.json changes.
|
|
41
|
+
*/
|
|
42
|
+
function refreshNames(root) {
|
|
43
|
+
const config = loadConfig();
|
|
44
|
+
const files = Object.keys(config.testOutput ?? {}).flatMap((area) => {
|
|
45
|
+
const dir = testDirForType(config, area);
|
|
46
|
+
return existsSync(dir) ? testFilesIn(dir) : [];
|
|
47
|
+
});
|
|
48
|
+
for (const n of recheckFilled(root))
|
|
49
|
+
console.log(` no longer in the code: ${n} — blanked; its tests wait until the new name is found`);
|
|
50
|
+
const blank = blankNames([...new Set(files.flatMap((f) => { try {
|
|
51
|
+
return neededNames(readFileSync(f, 'utf-8'));
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return [];
|
|
55
|
+
} }))]);
|
|
56
|
+
if (!blank.length) {
|
|
57
|
+
console.log(' no test waits on a name.');
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const res = scanNames(blank, root);
|
|
61
|
+
for (const f of res.filled)
|
|
62
|
+
console.log(` found in the code: ${f.name} = ${f.value}`);
|
|
63
|
+
console.log(` ${res.stillBlank.length} name(s) still blank (see testforge/interface.json).`);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* The actions modules, once the code moved: an action that was waiting for the
|
|
67
|
+
* product, or that the last run had refused, is written from the code now.
|
|
68
|
+
* Only the actions module changes; the tests call the same functions and stop
|
|
69
|
+
* waiting by themselves.
|
|
70
|
+
*/
|
|
71
|
+
async function refreshActions() {
|
|
72
|
+
const config = loadConfig();
|
|
73
|
+
const docs = existsSync(DOCS_DIR) ? findMarkdownFiles(DOCS_DIR) : [];
|
|
74
|
+
for (const area of Object.keys(config.testOutput ?? {})) {
|
|
75
|
+
const helpers = join(dirname(testDirForType(config, area)), 'helpers');
|
|
76
|
+
for (const ext of ['.js', '.ts']) {
|
|
77
|
+
const path = join(helpers, `actions${ext}`);
|
|
78
|
+
if (!existsSync(path))
|
|
79
|
+
continue;
|
|
80
|
+
const old = readFileSync(path, 'utf-8');
|
|
81
|
+
const failed = Object.keys(readActionFailures()).filter((n) => exportedNames(old).includes(n));
|
|
82
|
+
if (!hasWaitingActions(old) && !failed.length)
|
|
83
|
+
continue;
|
|
84
|
+
const stamp = readStamp(old);
|
|
85
|
+
const docPaths = docs.filter((d) => Object.keys(stamp.journeys).includes(basename(d).replace(/\.md$/, '')));
|
|
86
|
+
console.log(` actions (${area}): writing ${failed.length ? `${failed.join(', ')} again and ` : ''}the waiting actions from the code…`);
|
|
87
|
+
let src;
|
|
88
|
+
try {
|
|
89
|
+
src = await generateActionsAsync(path, join(helpers, `testkit${ext}`), docPaths);
|
|
90
|
+
}
|
|
91
|
+
catch (e) {
|
|
92
|
+
console.log(` not written — ${e.message.split('\n')[0].slice(0, 160)}; the actions stay as they were`);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const lost = exportedNames(old).filter((n) => !exportedNames(src).includes(n));
|
|
96
|
+
if (!isSyntaxValid(src, ext) || lost.length) {
|
|
97
|
+
console.log(` not taken — ${lost.length ? `it dropped ${lost.join(', ')}` : 'not valid code'}; the actions stay as they were`);
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
writeFileSync(path, stampActions(src, stamp.journeys), 'utf-8');
|
|
101
|
+
clearActionFailures();
|
|
102
|
+
const still = [...src.matchAll(/\bwaiting\(\s*['"`](\w+)/g)].map((m) => m[1]);
|
|
103
|
+
console.log(` done — the tests are unchanged${still.length ? `; still not in the product: ${[...new Set(still)].join(', ')}` : ''}`);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
21
107
|
export async function runUpdate(srcDir, opts = {}) {
|
|
22
108
|
if (!existsSync('testforge.md') || !existsSync('testforge.config.json')) {
|
|
23
109
|
console.log("This project isn't set up yet.");
|
|
@@ -39,7 +125,31 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
39
125
|
: { unreadable: 'docs are derived from the code map, not a reference' };
|
|
40
126
|
const refChanged = Boolean(refState.hash) && cache[REFERENCE_KEY] !== undefined && cache[REFERENCE_KEY] !== refState.hash;
|
|
41
127
|
const refFirstSeen = Boolean(refState.hash) && cache[REFERENCE_KEY] === undefined;
|
|
42
|
-
|
|
128
|
+
const redraft = Boolean(opts.refreshDocs || opts.reference);
|
|
129
|
+
// Code moved, no redraft asked for: the map and the names, nothing else.
|
|
130
|
+
// A doc is a decision about behaviour; the code is not allowed to make it.
|
|
131
|
+
if (!redraft && (codeChanged || refChanged)) {
|
|
132
|
+
if (codeChanged) {
|
|
133
|
+
console.log('Source code changed — refreshing the project map and the names the tests wait on.');
|
|
134
|
+
console.log('(The journey docs and the tests are not changed: they follow the specification, not the code.)\n');
|
|
135
|
+
runMap(srcDir);
|
|
136
|
+
console.log(' done — testforge.md');
|
|
137
|
+
refreshNames('.');
|
|
138
|
+
await refreshActions();
|
|
139
|
+
cache[SOURCE_KEY] = newHash;
|
|
140
|
+
}
|
|
141
|
+
if (refChanged) {
|
|
142
|
+
console.log(`\nThe reference has changed: ${src.reference}`);
|
|
143
|
+
console.log(' Nothing was redrafted. Review the change, then run `testforge update --refresh-docs`');
|
|
144
|
+
console.log(' to redraft the docs from it (the tests of what changed are then regenerated).');
|
|
145
|
+
}
|
|
146
|
+
if (refFirstSeen)
|
|
147
|
+
cache[REFERENCE_KEY] = refState.hash;
|
|
148
|
+
saveCache(CACHE_PATH, cache);
|
|
149
|
+
console.log('\nUpdate complete.');
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
if (!codeChanged && !refChanged && !opts.refreshDocs && !opts.reference) {
|
|
43
153
|
console.log(`No source changes detected.${refState.hash ? ' The reference is unchanged too.' : ''}`);
|
|
44
154
|
if (refState.unreadable && src.mode === 'reference') {
|
|
45
155
|
// Say what was NOT checked. Claiming "up to date" while the specification
|
|
@@ -57,11 +167,7 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
57
167
|
}
|
|
58
168
|
return;
|
|
59
169
|
}
|
|
60
|
-
const why =
|
|
61
|
-
codeChanged ? 'source code changed' : null,
|
|
62
|
-
refChanged ? 'the reference changed' : null,
|
|
63
|
-
opts.refreshDocs && !codeChanged && !refChanged ? 'refresh requested' : null,
|
|
64
|
-
].filter(Boolean).join(' and ');
|
|
170
|
+
const why = opts.reference ? `redrafting the docs from ${opts.reference}` : 'redrafting the docs (--refresh-docs)';
|
|
65
171
|
console.log(`Updating the project — ${why}.\n`);
|
|
66
172
|
console.log('Re-mapping the project...');
|
|
67
173
|
runMap(srcDir);
|
|
@@ -69,12 +175,12 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
69
175
|
// Reuse the source of truth chosen at setup (map or a reference)
|
|
70
176
|
// without prompting — an update is a refresh, not a re-decision.
|
|
71
177
|
console.log('Updating journey docs...');
|
|
72
|
-
await runDiscover(srcDir, DOCS_DIR, { noPrompt: true });
|
|
178
|
+
await runDiscover(srcDir, DOCS_DIR, opts.reference ? { reference: opts.reference, noPrompt: true, replace: true } : { noPrompt: true });
|
|
73
179
|
// Journeys that arrived on their own document — a ticket's attached spec —
|
|
74
180
|
// are refreshed from THAT document. The project specification does not
|
|
75
181
|
// describe them, so refreshing them from it would quietly replace a written
|
|
76
182
|
// requirement with whatever the code currently does.
|
|
77
|
-
const ownRefs = Object.entries(src.journeys || {}).filter(([, ref]) => ref !== src.reference);
|
|
183
|
+
const ownRefs = opts.reference ? [] : Object.entries(src.journeys || {}).filter(([, ref]) => ref !== src.reference);
|
|
78
184
|
for (const [doc, ref] of ownRefs) {
|
|
79
185
|
const name = doc.replace(/\.md$/, '').split('/').pop();
|
|
80
186
|
console.log(` refreshing ${name} from its own reference: ${ref}`);
|
|
@@ -191,6 +191,15 @@ E2. What you touch comes from <code_facts> and <testkit>.
|
|
|
191
191
|
is a row made with kit.insert; the roles are the ones <testkit> lists.
|
|
192
192
|
- Call testkit helpers exactly as <testkit> lists them, and never write your
|
|
193
193
|
own copy.
|
|
194
|
+
- A step a person takes through the product (book, pay, accept, cancel…) is
|
|
195
|
+
the ACTIONS function <testkit> lists for it, never a request you write: pass
|
|
196
|
+
the business values it takes, and put what the rule varies (a bad or
|
|
197
|
+
missing field) in its { body } overrides. A test of a refusal passes
|
|
198
|
+
{ allowRefusal: true } and asserts on the result's status and body. Assert
|
|
199
|
+
on the named results an action lists (orderId, orderStatus…), not on raw
|
|
200
|
+
body fields, so the test holds when the code arrives or moves. An action
|
|
201
|
+
"not in the product yet" is called all the same: the test waits until it
|
|
202
|
+
is. A request no action covers is written from <code_facts> as above.
|
|
194
203
|
|
|
195
204
|
E3. What you need first comes from the testkit, or from the product's own way
|
|
196
205
|
of making it.
|
|
@@ -199,6 +208,8 @@ of making it.
|
|
|
199
208
|
type of each id.
|
|
200
209
|
- Each role uses its own credential. Treat DUMMY, replace_me, an empty value
|
|
201
210
|
or a <bracketed> value as absent.
|
|
211
|
+
- A request with no login passes role: null. A request with an old or
|
|
212
|
+
forged token passes that Authorization header: it is sent as given.
|
|
202
213
|
- Owner-scoped data ("my orders") follows the token owner, not a query
|
|
203
214
|
parameter. A non-owner check needs a second token; without one, skip and
|
|
204
215
|
name the missing identity.
|
|
@@ -270,6 +281,12 @@ export const WEB = `WEB (Playwright)
|
|
|
270
281
|
wording only when <code_facts> lack the element. Always await kit.find
|
|
271
282
|
before using its result. Use page.getBy* only for what kit.find cannot
|
|
272
283
|
express. Never use a data-testid that <code_facts> do not show.
|
|
284
|
+
- Never select by markup: no CSS class (page.locator('.btn')), no tag paths
|
|
285
|
+
or positions ('div > span', ':nth-child'), no XPath ('//button'). They break
|
|
286
|
+
when the page is restyled. When kit.find cannot express it, use
|
|
287
|
+
page.getByRole, getByLabel, getByText or getByTestId; an id or name
|
|
288
|
+
attribute that <code_facts> show ('#email', 'input[name="phone"]') is the
|
|
289
|
+
last resort, for an input with no label.
|
|
273
290
|
- When a label also appears in an always-mounted part of the page (a header,
|
|
274
291
|
a login pop-up), scope to the journey's own container. The SCREENS lines say
|
|
275
292
|
which container renders each element.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.9.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -429,16 +429,47 @@ function environmentFor(raw) {
|
|
|
429
429
|
// by the tests of that endpoint, not by every test that signs in.
|
|
430
430
|
const t = /testkit: token check (\w+ \S+) answered (5\d\d|429)\b/.exec(text);
|
|
431
431
|
if (t) return `Environment: the server answered ${t[2]} to the testkit's sign-in check (${t[1]}), so this test could not start`;
|
|
432
|
+
// The API's rate limit stopped a testkit call: the run went too fast for the
|
|
433
|
+
// limiter, the product's rule was never reached.
|
|
434
|
+
const rl = /testkit: (\w+ \S+) answered 429\b/.exec(text);
|
|
435
|
+
if (rl) return `Environment: the API's rate limit answered 429 to the testkit (${rl[1]}) — clear the limiter or run fewer tests at once`;
|
|
436
|
+
// Nothing listening where the test connects (a server, database or cache that is down or not reachable).
|
|
437
|
+
const cr = /\bconnect ECONNREFUSED ([\w.:\[\]-]+)/.exec(text);
|
|
438
|
+
if (cr) return `Environment: nothing answered at ${cr[1]} (a server, database or cache that is down or not reachable from here), so this test could not run`;
|
|
432
439
|
return undefined;
|
|
433
440
|
}
|
|
434
441
|
|
|
435
|
-
|
|
442
|
+
/** An action whose own request the product refused (refused() in interface.cjs): { action, detail }. */
|
|
443
|
+
function actionRefused(raw) {
|
|
444
|
+
const m = /\[testforge:action\] (\w+): ([^\n]*)/.exec(strip(String(raw || '')));
|
|
445
|
+
return m ? { action: m[1], detail: m[2].trim() } : undefined;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Why a failed test is not the product's: it waits for a name, the environment
|
|
449
|
+
// lacks something, or a step before the rule failed.
|
|
436
450
|
function notTheProduct(raw) {
|
|
437
451
|
const waiting = waitingFor(raw);
|
|
438
452
|
if (waiting) return `Waiting for the product: "${waiting}" is not in it yet`;
|
|
453
|
+
const a = actionRefused(raw);
|
|
454
|
+
if (a) return `Step failed before the rule was checked: action "${a.action}" was refused (${a.detail}). The next generate builds the action again from the code; refused again, the product is at fault`;
|
|
439
455
|
return environmentFor(raw);
|
|
440
456
|
}
|
|
441
457
|
|
|
458
|
+
/**
|
|
459
|
+
* The actions refused in this run (testforge/actions-failed.json), for generate
|
|
460
|
+
* to build again. Replaced each run: an action not refused this time is fine.
|
|
461
|
+
*/
|
|
462
|
+
function recordActionFailures(root, raws) {
|
|
463
|
+
const found = {};
|
|
464
|
+
for (const raw of raws) { const a = actionRefused(raw); if (a && !found[a.action]) found[a.action] = a.detail; }
|
|
465
|
+
const file = path.join(root, 'testforge', 'actions-failed.json');
|
|
466
|
+
try {
|
|
467
|
+
if (!Object.keys(found).length) { fs.rmSync(file, { force: true }); return; }
|
|
468
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
469
|
+
fs.writeFileSync(file, JSON.stringify(found, null, 2) + '\n');
|
|
470
|
+
} catch { /* the run's own output still names them */ }
|
|
471
|
+
}
|
|
472
|
+
|
|
442
473
|
/**
|
|
443
474
|
* Remember the tests that passed (testforge/passed.json): generate never removes
|
|
444
475
|
* or rewrites a test that has passed. Add-only — a later failure is a regression
|
|
@@ -484,4 +515,4 @@ function recordSummary(root, category, suites) {
|
|
|
484
515
|
} catch { /* the run's own output still has the counts */ }
|
|
485
516
|
}
|
|
486
517
|
|
|
487
|
-
module.exports = { recordSummary, recordPassed, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
|
|
518
|
+
module.exports = { recordSummary, recordPassed, recordActionFailures, actionRefused, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
|