@activate-core/test-forge 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +21 -0
- package/dist/cli/index.js +14 -4
- package/dist/core/docs/discover.js +15 -0
- package/dist/core/docs/doc-origin.js +42 -0
- package/dist/core/docs/reference.js +18 -1
- package/dist/core/gates/validate.js +8 -0
- package/dist/core/generate/actions.js +200 -0
- package/dist/core/generate/ai-generator.js +41 -0
- package/dist/core/generate/ai-usage.js +1 -0
- package/dist/core/generate/complete-file.js +62 -2
- package/dist/core/generate/harness.js +57 -14
- package/dist/core/generate/run-generate.js +203 -27
- package/dist/core/generate/test-blocks.js +123 -2
- package/dist/core/generate/update-journey.js +23 -3
- package/dist/core/project/preflight.js +69 -0
- package/dist/core/project/update.js +120 -14
- package/dist/core/prompts/blocks.js +17 -0
- package/package.json +1 -1
- package/reporters/_shared.cjs +33 -2
- package/reporters/http.cjs +96 -0
- package/reporters/interface.cjs +25 -2
- package/reporters/jest-reporter.cjs +3 -1
|
@@ -44,23 +44,41 @@ export async function updateJourney(ctx) {
|
|
|
44
44
|
const before = readFileSync(path, 'utf-8');
|
|
45
45
|
let content = before;
|
|
46
46
|
const mine = (ids) => ids.filter((id) => ctx.plan.find((p) => p.slug === f.slug)?.rules?.includes(id));
|
|
47
|
+
const locked = ctx.protectPassed && ctx.locked ? ctx.locked(path) : new Set();
|
|
48
|
+
const free = (blocks) => blocks.filter((b) => !locked.has(b.title));
|
|
49
|
+
const kept = new Set();
|
|
47
50
|
// Renames: literal replacement inside the renamed rule's tests only. No AI.
|
|
48
51
|
// A file written before tests named their rule gets the rename in every test.
|
|
49
52
|
const legacy = !findTestBlocks(content).some((b) => b.rules.length);
|
|
50
53
|
for (const r of diff.renamed.filter((x) => legacy || mine([x.id]).length)) {
|
|
51
|
-
const
|
|
54
|
+
const targets = legacy ? findTestBlocks(content) : blocksFor(content, [r.id]);
|
|
55
|
+
for (const b of targets)
|
|
56
|
+
if (locked.has(b.title))
|
|
57
|
+
kept.add(b.title);
|
|
58
|
+
const res = renameInBlocks(content, free(targets), r.pairs);
|
|
52
59
|
content = res.content;
|
|
53
60
|
if (res.replaced)
|
|
54
61
|
notes.push(`${f.slug}: ${r.id} renamed ${r.pairs.map(([a, b]) => `"${a}" → "${b}"`).join(', ')} (${res.replaced} place(s))`);
|
|
55
62
|
}
|
|
56
63
|
// Removed rules: their tests go, unless a test also checks a rule that stays.
|
|
57
|
-
const
|
|
64
|
+
const goneAll = findTestBlocks(content).filter((b) => b.rules.length && b.rules.every((id) => removed.has(id)));
|
|
65
|
+
for (const b of goneAll)
|
|
66
|
+
if (locked.has(b.title))
|
|
67
|
+
kept.add(b.title);
|
|
68
|
+
const gone = free(goneAll);
|
|
58
69
|
if (gone.length) {
|
|
59
70
|
content = spliceBlocks(content, gone.map((block) => ({ block, text: '' })));
|
|
60
71
|
notes.push(`${f.slug}: removed ${gone.length} test(s) of rules no longer in the doc`);
|
|
61
72
|
}
|
|
62
73
|
// Reworded rules and added rules: one small AI call for just those.
|
|
63
|
-
|
|
74
|
+
// A reworded rule whose test passed keeps that test when the rewording was
|
|
75
|
+
// TestForge's own: the passing test is the evidence, a redraft is not.
|
|
76
|
+
const changed = mine(diff.changed).filter(testedId).filter((id) => {
|
|
77
|
+
const passedOnes = blocksFor(content, [id]).filter((b) => locked.has(b.title));
|
|
78
|
+
for (const b of passedOnes)
|
|
79
|
+
kept.add(b.title);
|
|
80
|
+
return !passedOnes.length;
|
|
81
|
+
});
|
|
64
82
|
const extra = toFile.get(f) ?? [];
|
|
65
83
|
const ids = [...changed, ...extra];
|
|
66
84
|
if (ids.length) {
|
|
@@ -106,6 +124,8 @@ export async function updateJourney(ctx) {
|
|
|
106
124
|
notes.push(`${f.slug}: tests for ${ids.join(', ')} were refused (${refusal}) — the file keeps its old tests`);
|
|
107
125
|
}
|
|
108
126
|
}
|
|
127
|
+
if (kept.size)
|
|
128
|
+
notes.push(`${f.slug}: kept ${kept.size} test(s) that passed — the doc change came from TestForge's drafting, not a person (edit the doc yourself or use --force to change them)`);
|
|
109
129
|
if (content !== before)
|
|
110
130
|
writeFileSync(path, content, 'utf-8');
|
|
111
131
|
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — is the AI CLI there before a command spends time on it?
|
|
3
|
+
*
|
|
4
|
+
* Checked with no AI call (no tokens): the configured program is on PATH and
|
|
5
|
+
* runs `--version`. Whether it is signed in can only be known by asking it
|
|
6
|
+
* something, so that shows up on the first real call; `friendlyAiError` turns
|
|
7
|
+
* that and a missing program into a plain message instead of a raw process
|
|
8
|
+
* error.
|
|
9
|
+
*/
|
|
10
|
+
import { execFileSync } from 'node:child_process';
|
|
11
|
+
import { accessSync, constants } from 'node:fs';
|
|
12
|
+
import { delimiter, isAbsolute, join } from 'node:path';
|
|
13
|
+
import { loadConfig, splitCommand } from './config.js';
|
|
14
|
+
export class AiCliError extends Error {
|
|
15
|
+
}
|
|
16
|
+
/** The program's full path when it is runnable from here, else undefined. */
|
|
17
|
+
export function findOnPath(program, path = process.env.PATH ?? '') {
|
|
18
|
+
const runnable = (p) => { try {
|
|
19
|
+
accessSync(p, constants.X_OK);
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return false;
|
|
24
|
+
} };
|
|
25
|
+
if (program.includes('/') || isAbsolute(program))
|
|
26
|
+
return runnable(program) ? program : undefined;
|
|
27
|
+
const exts = process.platform === 'win32' ? (process.env.PATHEXT ?? '.EXE;.CMD;.BAT').split(';') : [''];
|
|
28
|
+
for (const dir of path.split(delimiter).filter(Boolean)) {
|
|
29
|
+
for (const ext of exts) {
|
|
30
|
+
const p = join(dir, program + ext);
|
|
31
|
+
if (runnable(p))
|
|
32
|
+
return p;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
/** Throws an AiCliError with what to do when the configured AI CLI cannot be run. */
|
|
38
|
+
export function preflightAi() {
|
|
39
|
+
const { program } = splitCommand(loadConfig().aiCommand);
|
|
40
|
+
if (!findOnPath(program)) {
|
|
41
|
+
throw new AiCliError(`Error: the AI CLI "${program}" was not found on PATH.\n` +
|
|
42
|
+
` Install it (for Claude: npm i -g @anthropic-ai/claude-code, then run \`claude\` once to sign in),\n` +
|
|
43
|
+
` or set "aiCommand" in testforge.config.json to the AI CLI you use.`);
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
execFileSync(program, ['--version'], { stdio: 'pipe', timeout: 20_000 });
|
|
47
|
+
}
|
|
48
|
+
catch (e) {
|
|
49
|
+
const msg = String(e.stderr ?? e.message).split('\n')[0].slice(0, 200);
|
|
50
|
+
throw new AiCliError(`Error: the AI CLI "${program}" is installed but did not run (${msg}).`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** A plain message for an AI CLI failure, or undefined when the error is something else. */
|
|
54
|
+
export function friendlyAiError(err) {
|
|
55
|
+
const text = err instanceof Error ? `${err.message} ${String(err.stderr ?? '')}` : String(err);
|
|
56
|
+
const program = (() => { try {
|
|
57
|
+
return splitCommand(loadConfig().aiCommand).program;
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return 'the AI CLI';
|
|
61
|
+
} })();
|
|
62
|
+
if (/spawn(?:Sync)? \S+ ENOENT/.test(text)) {
|
|
63
|
+
return `Error: the AI CLI "${program}" was not found. Install it, or set "aiCommand" in testforge.config.json.`;
|
|
64
|
+
}
|
|
65
|
+
if (/not logged in|please log ?in|unauthori[sz]ed|invalid api key|authentication/i.test(text)) {
|
|
66
|
+
return `Error: the AI CLI "${program}" is not signed in. Run \`${program}\` once and sign in (for Claude Code: /login), then try again.`;
|
|
67
|
+
}
|
|
68
|
+
return undefined;
|
|
69
|
+
}
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* testforge — update command.
|
|
3
3
|
*
|
|
4
|
-
* Refreshes a project that is already set up, doing only the work that
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* Refreshes a project that is already set up, doing only the work that is
|
|
5
|
+
* actually needed. The journey docs are the source of truth for what the
|
|
6
|
+
* tests assert, so the code never rewrites them:
|
|
7
|
+
*
|
|
8
|
+
* - Code changed: the project map is refreshed, the names the tests wait on
|
|
9
|
+
* are looked up again (testforge/interface.json), and actions that waited
|
|
10
|
+
* for the product (or were refused in the last run) are written from the
|
|
11
|
+
* code. The docs and the tests are not touched.
|
|
12
|
+
* - The reference changed: reported, not acted on — `update --refresh-docs`
|
|
13
|
+
* redrafts the docs from it when a person decides to.
|
|
14
|
+
* - `--refresh-docs` or `--reference <x>`: the docs are redrafted from the
|
|
15
|
+
* reference, then the tests of what changed are regenerated (tests that
|
|
16
|
+
* passed are kept unless a person edits the doc or --force is given).
|
|
10
17
|
*/
|
|
11
18
|
import { existsSync } from 'node:fs';
|
|
12
19
|
import { runMap } from './map.js';
|
|
@@ -16,8 +23,87 @@ import { hashProjectSource, hashReference, loadCache, saveCache } from '../docs/
|
|
|
16
23
|
import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
17
24
|
import { runEnvTemplate } from './env-template.js';
|
|
18
25
|
import { CACHE_PATH, DOCS_DIR } from '../paths.js';
|
|
26
|
+
import { readFileSync } from 'node:fs';
|
|
27
|
+
import { loadConfig, testDirForType } from './config.js';
|
|
28
|
+
import { testFilesIn } from '../run/readiness.js';
|
|
29
|
+
import { blankNames, neededNames } from '../generate/interface-file.js';
|
|
30
|
+
import { recheckFilled, scanNames } from '../run/name-scan.js';
|
|
31
|
+
import { basename, dirname, join } from 'node:path';
|
|
32
|
+
import { writeFileSync } from 'node:fs';
|
|
33
|
+
import { findMarkdownFiles } from '../find-files.js';
|
|
34
|
+
import { isSyntaxValid } from '../gates/validate.js';
|
|
35
|
+
import { clearActionFailures, exportedNames, generateActionsAsync, hasWaitingActions, readActionFailures, readStamp, stampActions } from '../generate/actions.js';
|
|
19
36
|
const SOURCE_KEY = '__source__';
|
|
20
37
|
const REFERENCE_KEY = '__reference__';
|
|
38
|
+
/**
|
|
39
|
+
* The names the tests wait on, looked up again in the code (a value the code
|
|
40
|
+
* no longer has is blanked first). Only testforge/interface.json changes.
|
|
41
|
+
*/
|
|
42
|
+
function refreshNames(root) {
|
|
43
|
+
const config = loadConfig();
|
|
44
|
+
const files = Object.keys(config.testOutput ?? {}).flatMap((area) => {
|
|
45
|
+
const dir = testDirForType(config, area);
|
|
46
|
+
return existsSync(dir) ? testFilesIn(dir) : [];
|
|
47
|
+
});
|
|
48
|
+
for (const n of recheckFilled(root))
|
|
49
|
+
console.log(` no longer in the code: ${n} — blanked; its tests wait until the new name is found`);
|
|
50
|
+
const blank = blankNames([...new Set(files.flatMap((f) => { try {
|
|
51
|
+
return neededNames(readFileSync(f, 'utf-8'));
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return [];
|
|
55
|
+
} }))]);
|
|
56
|
+
if (!blank.length) {
|
|
57
|
+
console.log(' no test waits on a name.');
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const res = scanNames(blank, root);
|
|
61
|
+
for (const f of res.filled)
|
|
62
|
+
console.log(` found in the code: ${f.name} = ${f.value}`);
|
|
63
|
+
console.log(` ${res.stillBlank.length} name(s) still blank (see testforge/interface.json).`);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* The actions modules, once the code moved: an action that was waiting for the
|
|
67
|
+
* product, or that the last run had refused, is written from the code now.
|
|
68
|
+
* Only the actions module changes; the tests call the same functions and stop
|
|
69
|
+
* waiting by themselves.
|
|
70
|
+
*/
|
|
71
|
+
async function refreshActions() {
|
|
72
|
+
const config = loadConfig();
|
|
73
|
+
const docs = existsSync(DOCS_DIR) ? findMarkdownFiles(DOCS_DIR) : [];
|
|
74
|
+
for (const area of Object.keys(config.testOutput ?? {})) {
|
|
75
|
+
const helpers = join(dirname(testDirForType(config, area)), 'helpers');
|
|
76
|
+
for (const ext of ['.js', '.ts']) {
|
|
77
|
+
const path = join(helpers, `actions${ext}`);
|
|
78
|
+
if (!existsSync(path))
|
|
79
|
+
continue;
|
|
80
|
+
const old = readFileSync(path, 'utf-8');
|
|
81
|
+
const failed = Object.keys(readActionFailures()).filter((n) => exportedNames(old).includes(n));
|
|
82
|
+
if (!hasWaitingActions(old) && !failed.length)
|
|
83
|
+
continue;
|
|
84
|
+
const stamp = readStamp(old);
|
|
85
|
+
const docPaths = docs.filter((d) => Object.keys(stamp.journeys).includes(basename(d).replace(/\.md$/, '')));
|
|
86
|
+
console.log(` actions (${area}): writing ${failed.length ? `${failed.join(', ')} again and ` : ''}the waiting actions from the code…`);
|
|
87
|
+
let src;
|
|
88
|
+
try {
|
|
89
|
+
src = await generateActionsAsync(path, join(helpers, `testkit${ext}`), docPaths);
|
|
90
|
+
}
|
|
91
|
+
catch (e) {
|
|
92
|
+
console.log(` not written — ${e.message.split('\n')[0].slice(0, 160)}; the actions stay as they were`);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const lost = exportedNames(old).filter((n) => !exportedNames(src).includes(n));
|
|
96
|
+
if (!isSyntaxValid(src, ext) || lost.length) {
|
|
97
|
+
console.log(` not taken — ${lost.length ? `it dropped ${lost.join(', ')}` : 'not valid code'}; the actions stay as they were`);
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
writeFileSync(path, stampActions(src, stamp.journeys), 'utf-8');
|
|
101
|
+
clearActionFailures();
|
|
102
|
+
const still = [...src.matchAll(/\bwaiting\(\s*['"`](\w+)/g)].map((m) => m[1]);
|
|
103
|
+
console.log(` done — the tests are unchanged${still.length ? `; still not in the product: ${[...new Set(still)].join(', ')}` : ''}`);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
21
107
|
export async function runUpdate(srcDir, opts = {}) {
|
|
22
108
|
if (!existsSync('testforge.md') || !existsSync('testforge.config.json')) {
|
|
23
109
|
console.log("This project isn't set up yet.");
|
|
@@ -39,7 +125,31 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
39
125
|
: { unreadable: 'docs are derived from the code map, not a reference' };
|
|
40
126
|
const refChanged = Boolean(refState.hash) && cache[REFERENCE_KEY] !== undefined && cache[REFERENCE_KEY] !== refState.hash;
|
|
41
127
|
const refFirstSeen = Boolean(refState.hash) && cache[REFERENCE_KEY] === undefined;
|
|
42
|
-
|
|
128
|
+
const redraft = Boolean(opts.refreshDocs || opts.reference);
|
|
129
|
+
// Code moved, no redraft asked for: the map and the names, nothing else.
|
|
130
|
+
// A doc is a decision about behaviour; the code is not allowed to make it.
|
|
131
|
+
if (!redraft && (codeChanged || refChanged)) {
|
|
132
|
+
if (codeChanged) {
|
|
133
|
+
console.log('Source code changed — refreshing the project map and the names the tests wait on.');
|
|
134
|
+
console.log('(The journey docs and the tests are not changed: they follow the specification, not the code.)\n');
|
|
135
|
+
runMap(srcDir);
|
|
136
|
+
console.log(' done — testforge.md');
|
|
137
|
+
refreshNames('.');
|
|
138
|
+
await refreshActions();
|
|
139
|
+
cache[SOURCE_KEY] = newHash;
|
|
140
|
+
}
|
|
141
|
+
if (refChanged) {
|
|
142
|
+
console.log(`\nThe reference has changed: ${src.reference}`);
|
|
143
|
+
console.log(' Nothing was redrafted. Review the change, then run `testforge update --refresh-docs`');
|
|
144
|
+
console.log(' to redraft the docs from it (the tests of what changed are then regenerated).');
|
|
145
|
+
}
|
|
146
|
+
if (refFirstSeen)
|
|
147
|
+
cache[REFERENCE_KEY] = refState.hash;
|
|
148
|
+
saveCache(CACHE_PATH, cache);
|
|
149
|
+
console.log('\nUpdate complete.');
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
if (!codeChanged && !refChanged && !opts.refreshDocs && !opts.reference) {
|
|
43
153
|
console.log(`No source changes detected.${refState.hash ? ' The reference is unchanged too.' : ''}`);
|
|
44
154
|
if (refState.unreadable && src.mode === 'reference') {
|
|
45
155
|
// Say what was NOT checked. Claiming "up to date" while the specification
|
|
@@ -57,11 +167,7 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
57
167
|
}
|
|
58
168
|
return;
|
|
59
169
|
}
|
|
60
|
-
const why =
|
|
61
|
-
codeChanged ? 'source code changed' : null,
|
|
62
|
-
refChanged ? 'the reference changed' : null,
|
|
63
|
-
opts.refreshDocs && !codeChanged && !refChanged ? 'refresh requested' : null,
|
|
64
|
-
].filter(Boolean).join(' and ');
|
|
170
|
+
const why = opts.reference ? `redrafting the docs from ${opts.reference}` : 'redrafting the docs (--refresh-docs)';
|
|
65
171
|
console.log(`Updating the project — ${why}.\n`);
|
|
66
172
|
console.log('Re-mapping the project...');
|
|
67
173
|
runMap(srcDir);
|
|
@@ -69,12 +175,12 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
69
175
|
// Reuse the source of truth chosen at setup (map or a reference)
|
|
70
176
|
// without prompting — an update is a refresh, not a re-decision.
|
|
71
177
|
console.log('Updating journey docs...');
|
|
72
|
-
await runDiscover(srcDir, DOCS_DIR, { noPrompt: true });
|
|
178
|
+
await runDiscover(srcDir, DOCS_DIR, opts.reference ? { reference: opts.reference, noPrompt: true, replace: true } : { noPrompt: true });
|
|
73
179
|
// Journeys that arrived on their own document — a ticket's attached spec —
|
|
74
180
|
// are refreshed from THAT document. The project specification does not
|
|
75
181
|
// describe them, so refreshing them from it would quietly replace a written
|
|
76
182
|
// requirement with whatever the code currently does.
|
|
77
|
-
const ownRefs = Object.entries(src.journeys || {}).filter(([, ref]) => ref !== src.reference);
|
|
183
|
+
const ownRefs = opts.reference ? [] : Object.entries(src.journeys || {}).filter(([, ref]) => ref !== src.reference);
|
|
78
184
|
for (const [doc, ref] of ownRefs) {
|
|
79
185
|
const name = doc.replace(/\.md$/, '').split('/').pop();
|
|
80
186
|
console.log(` refreshing ${name} from its own reference: ${ref}`);
|
|
@@ -191,6 +191,15 @@ E2. What you touch comes from <code_facts> and <testkit>.
|
|
|
191
191
|
is a row made with kit.insert; the roles are the ones <testkit> lists.
|
|
192
192
|
- Call testkit helpers exactly as <testkit> lists them, and never write your
|
|
193
193
|
own copy.
|
|
194
|
+
- A step a person takes through the product (book, pay, accept, cancel…) is
|
|
195
|
+
the ACTIONS function <testkit> lists for it, never a request you write: pass
|
|
196
|
+
the business values it takes, and put what the rule varies (a bad or
|
|
197
|
+
missing field) in its { body } overrides. A test of a refusal passes
|
|
198
|
+
{ allowRefusal: true } and asserts on the result's status and body. Assert
|
|
199
|
+
on the named results an action lists (orderId, orderStatus…), not on raw
|
|
200
|
+
body fields, so the test holds when the code arrives or moves. An action
|
|
201
|
+
"not in the product yet" is called all the same: the test waits until it
|
|
202
|
+
is. A request no action covers is written from <code_facts> as above.
|
|
194
203
|
|
|
195
204
|
E3. What you need first comes from the testkit, or from the product's own way
|
|
196
205
|
of making it.
|
|
@@ -199,6 +208,8 @@ of making it.
|
|
|
199
208
|
type of each id.
|
|
200
209
|
- Each role uses its own credential. Treat DUMMY, replace_me, an empty value
|
|
201
210
|
or a <bracketed> value as absent.
|
|
211
|
+
- A request with no login passes role: null. A request with an old or
|
|
212
|
+
forged token passes that Authorization header: it is sent as given.
|
|
202
213
|
- Owner-scoped data ("my orders") follows the token owner, not a query
|
|
203
214
|
parameter. A non-owner check needs a second token; without one, skip and
|
|
204
215
|
name the missing identity.
|
|
@@ -270,6 +281,12 @@ export const WEB = `WEB (Playwright)
|
|
|
270
281
|
wording only when <code_facts> lack the element. Always await kit.find
|
|
271
282
|
before using its result. Use page.getBy* only for what kit.find cannot
|
|
272
283
|
express. Never use a data-testid that <code_facts> do not show.
|
|
284
|
+
- Never select by markup: no CSS class (page.locator('.btn')), no tag paths
|
|
285
|
+
or positions ('div > span', ':nth-child'), no XPath ('//button'). They break
|
|
286
|
+
when the page is restyled. When kit.find cannot express it, use
|
|
287
|
+
page.getByRole, getByLabel, getByText or getByTestId; an id or name
|
|
288
|
+
attribute that <code_facts> show ('#email', 'input[name="phone"]') is the
|
|
289
|
+
last resort, for an input with no label.
|
|
273
290
|
- When a label also appears in an always-mounted part of the page (a header,
|
|
274
291
|
a login pop-up), scope to the journey's own container. The SCREENS lines say
|
|
275
292
|
which container renders each element.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.9.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -429,16 +429,47 @@ function environmentFor(raw) {
|
|
|
429
429
|
// by the tests of that endpoint, not by every test that signs in.
|
|
430
430
|
const t = /testkit: token check (\w+ \S+) answered (5\d\d|429)\b/.exec(text);
|
|
431
431
|
if (t) return `Environment: the server answered ${t[2]} to the testkit's sign-in check (${t[1]}), so this test could not start`;
|
|
432
|
+
// The API's rate limit stopped a testkit call: the run went too fast for the
|
|
433
|
+
// limiter, the product's rule was never reached.
|
|
434
|
+
const rl = /testkit: (\w+ \S+) answered 429\b/.exec(text);
|
|
435
|
+
if (rl) return `Environment: the API's rate limit answered 429 to the testkit (${rl[1]}) — clear the limiter or run fewer tests at once`;
|
|
436
|
+
// Nothing listening where the test connects (a server, database or cache that is down or not reachable).
|
|
437
|
+
const cr = /\bconnect ECONNREFUSED ([\w.:\[\]-]+)/.exec(text);
|
|
438
|
+
if (cr) return `Environment: nothing answered at ${cr[1]} (a server, database or cache that is down or not reachable from here), so this test could not run`;
|
|
432
439
|
return undefined;
|
|
433
440
|
}
|
|
434
441
|
|
|
435
|
-
|
|
442
|
+
/** An action whose own request the product refused (refused() in interface.cjs): { action, detail }. */
|
|
443
|
+
function actionRefused(raw) {
|
|
444
|
+
const m = /\[testforge:action\] (\w+): ([^\n]*)/.exec(strip(String(raw || '')));
|
|
445
|
+
return m ? { action: m[1], detail: m[2].trim() } : undefined;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Why a failed test is not the product's: it waits for a name, the environment
|
|
449
|
+
// lacks something, or a step before the rule failed.
|
|
436
450
|
function notTheProduct(raw) {
|
|
437
451
|
const waiting = waitingFor(raw);
|
|
438
452
|
if (waiting) return `Waiting for the product: "${waiting}" is not in it yet`;
|
|
453
|
+
const a = actionRefused(raw);
|
|
454
|
+
if (a) return `Step failed before the rule was checked: action "${a.action}" was refused (${a.detail}). The next generate builds the action again from the code; refused again, the product is at fault`;
|
|
439
455
|
return environmentFor(raw);
|
|
440
456
|
}
|
|
441
457
|
|
|
458
|
+
/**
|
|
459
|
+
* The actions refused in this run (testforge/actions-failed.json), for generate
|
|
460
|
+
* to build again. Replaced each run: an action not refused this time is fine.
|
|
461
|
+
*/
|
|
462
|
+
function recordActionFailures(root, raws) {
|
|
463
|
+
const found = {};
|
|
464
|
+
for (const raw of raws) { const a = actionRefused(raw); if (a && !found[a.action]) found[a.action] = a.detail; }
|
|
465
|
+
const file = path.join(root, 'testforge', 'actions-failed.json');
|
|
466
|
+
try {
|
|
467
|
+
if (!Object.keys(found).length) { fs.rmSync(file, { force: true }); return; }
|
|
468
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
469
|
+
fs.writeFileSync(file, JSON.stringify(found, null, 2) + '\n');
|
|
470
|
+
} catch { /* the run's own output still names them */ }
|
|
471
|
+
}
|
|
472
|
+
|
|
442
473
|
/**
|
|
443
474
|
* Remember the tests that passed (testforge/passed.json): generate never removes
|
|
444
475
|
* or rewrites a test that has passed. Add-only — a later failure is a regression
|
|
@@ -484,4 +515,4 @@ function recordSummary(root, category, suites) {
|
|
|
484
515
|
} catch { /* the run's own output still has the counts */ }
|
|
485
516
|
}
|
|
486
517
|
|
|
487
|
-
module.exports = { recordSummary, recordPassed, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
|
|
518
|
+
module.exports = { recordSummary, recordPassed, recordActionFailures, actionRefused, strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, waitingFor, environmentFor, notTheProduct, plainTitle };
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// TestForge runtime — the testkit's call(), the same in every project.
|
|
2
|
+
//
|
|
3
|
+
// What is the same everywhere lives here as code, not in each AI-written
|
|
4
|
+
// testkit: a header the test gives wins over the role's token, and a 429 from
|
|
5
|
+
// the API's rate limiter is cleared or waited out instead of failing the test.
|
|
6
|
+
// The testkit supplies only what is the product's own: where the server is,
|
|
7
|
+
// how a role gets a token, how to clear the limiter, and (for a Node server it
|
|
8
|
+
// drives in-process) how to send a request.
|
|
9
|
+
'use strict';
|
|
10
|
+
const http = require('http');
|
|
11
|
+
const https = require('https');
|
|
12
|
+
|
|
13
|
+
/** Plain HTTP with Node built-ins; JSON in and out when it is JSON. */
|
|
14
|
+
function httpSend(method, url, { headers = {}, body } = {}) {
|
|
15
|
+
return new Promise((resolve, reject) => {
|
|
16
|
+
const u = new URL(url);
|
|
17
|
+
const lib = u.protocol === 'https:' ? https : http;
|
|
18
|
+
const payload = body === undefined ? undefined : (typeof body === 'string' ? body : JSON.stringify(body));
|
|
19
|
+
const h = { Accept: 'application/json', ...headers };
|
|
20
|
+
if (payload !== undefined) {
|
|
21
|
+
if (!Object.keys(h).some((k) => k.toLowerCase() === 'content-type')) h['Content-Type'] = 'application/json';
|
|
22
|
+
h['Content-Length'] = Buffer.byteLength(payload);
|
|
23
|
+
}
|
|
24
|
+
const req = lib.request(u, { method, headers: h, timeout: 30000 }, (res) => {
|
|
25
|
+
const chunks = [];
|
|
26
|
+
res.on('data', (c) => chunks.push(c));
|
|
27
|
+
res.on('end', () => {
|
|
28
|
+
const text = Buffer.concat(chunks).toString('utf8');
|
|
29
|
+
let json;
|
|
30
|
+
try { json = text ? JSON.parse(text) : undefined; } catch (_) { json = undefined; }
|
|
31
|
+
resolve({ status: res.statusCode, headers: res.headers, body: json !== undefined ? json : text });
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
req.on('timeout', () => req.destroy(new Error(`testkit: ${method} ${url} timed out`)));
|
|
35
|
+
req.on('error', reject);
|
|
36
|
+
if (payload !== undefined) req.write(payload);
|
|
37
|
+
req.end();
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* call(method, path, { body, headers, role }) → { status, headers, body }.
|
|
45
|
+
* url(path) the full URL of a path (the product's prefix rules)
|
|
46
|
+
* ensureToken(role, { force }) a token the product accepts for that role
|
|
47
|
+
* resetRateLimits() clears the limiter, resolves how many entries it cleared (optional)
|
|
48
|
+
* send(method, url, { headers, body }) the transport (default: HTTP)
|
|
49
|
+
* refused(status) a status that means "token not accepted" (default: 401)
|
|
50
|
+
*/
|
|
51
|
+
function makeCall({ url, ensureToken, resetRateLimits, send = httpSend, refused = (s) => s === 401, limitMs = 60000 }) {
|
|
52
|
+
if (typeof url !== 'function' || typeof ensureToken !== 'function') throw new Error('makeCall: url and ensureToken are required');
|
|
53
|
+
return async function call(method, path, { body, headers = {}, role = 'customer' } = {}) {
|
|
54
|
+
const m = String(method).toUpperCase();
|
|
55
|
+
const target = url(path);
|
|
56
|
+
// A header the test gives is sent exactly as given ('' = none) and never
|
|
57
|
+
// replaced, retried or re-minted: an old or forged token is what it tests.
|
|
58
|
+
const authKey = Object.keys(headers).find((k) => k.toLowerCase() === 'authorization');
|
|
59
|
+
const own = authKey !== undefined || role === null;
|
|
60
|
+
const given = { ...headers };
|
|
61
|
+
if (authKey !== undefined && !given[authKey]) delete given[authKey];
|
|
62
|
+
const once = async (force) => {
|
|
63
|
+
if (own) return send(m, target, { headers: given, body });
|
|
64
|
+
const tok = await ensureToken(role, { force });
|
|
65
|
+
return send(m, target, { headers: tok ? { ...given, Authorization: `Bearer ${tok}` } : given, body });
|
|
66
|
+
};
|
|
67
|
+
let res = await once(false);
|
|
68
|
+
if (!own && refused(res.status)) res = await once(true);
|
|
69
|
+
const until = Date.now() + limitMs;
|
|
70
|
+
while (res.status === 429 && Date.now() < until) {
|
|
71
|
+
let cleared = 0;
|
|
72
|
+
try { cleared = resetRateLimits ? Number(await resetRateLimits()) || 0 : 0; } catch (_) { cleared = 0; }
|
|
73
|
+
if (!cleared) {
|
|
74
|
+
const after = Number((res.headers || {})['retry-after']);
|
|
75
|
+
await sleep(Math.max(0, Math.min(until - Date.now(), 1000 * (after > 0 ? after : 2))));
|
|
76
|
+
}
|
|
77
|
+
res = await once(false);
|
|
78
|
+
}
|
|
79
|
+
return { status: res.status, headers: res.headers, body: res.body };
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The tables DERIVED_STATE names really exist: a wrong name (a model's class
|
|
85
|
+
* name instead of its table) would quietly derive nothing. query(sql) runs SQL.
|
|
86
|
+
*/
|
|
87
|
+
async function verifyDerivedState(derived, query) {
|
|
88
|
+
const missing = [];
|
|
89
|
+
for (const t of Object.keys(derived || {})) {
|
|
90
|
+
if (!/^[A-Za-z_][\w.]*$/.test(t)) { missing.push(t); continue; }
|
|
91
|
+
try { await query(`SELECT 1 FROM ${t} WHERE 1 = 0`); } catch (_) { missing.push(t); }
|
|
92
|
+
}
|
|
93
|
+
if (missing.length) throw new Error(`testkit: DERIVED_STATE names ${missing.join(', ')}, not a table in the database — the derived rows are never made`);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
module.exports = { makeCall, httpSend, verifyDerivedState };
|
package/reporters/interface.cjs
CHANGED
|
@@ -105,7 +105,11 @@ function usersOf(file, name) {
|
|
|
105
105
|
});
|
|
106
106
|
const word = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
|
|
107
107
|
// A name inside a string ('redeem points URL') is not a use of the constant URL.
|
|
108
|
-
|
|
108
|
+
// A `${URL}` inside a template literal is a use: keep the interpolations.
|
|
109
|
+
const code = (t) => String(t)
|
|
110
|
+
.replace(/`(?:\\.|[^`\\])*`/g, (m) => (m.match(/\$\{[^}]*\}/g) || []).join(' '))
|
|
111
|
+
.replace(/(['"])(?:\\.|(?!\1)[^\\])*\1/g, '""');
|
|
112
|
+
const uses = { test: (t) => word.test(code(t)) };
|
|
109
113
|
const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
|
|
110
114
|
if (lines.some((l, i) => i !== declLine && !inTest(i) && uses.test(l))) return undefined;
|
|
111
115
|
return tests.filter((t) => uses.test(t.text));
|
|
@@ -192,4 +196,23 @@ function fillPath(template, ...values) {
|
|
|
192
196
|
return String(template).replace(/\{[^}/]+\}|(?<=\/):[A-Za-z_]\w*/g, (m) => (i < values.length ? encodeURIComponent(String(values[i++])) : m));
|
|
193
197
|
}
|
|
194
198
|
|
|
195
|
-
|
|
199
|
+
/**
|
|
200
|
+
* An action the product does not offer yet (actions module, A7): calling it
|
|
201
|
+
* makes the test wait, saying what was looked for.
|
|
202
|
+
*/
|
|
203
|
+
function waiting(action, why) {
|
|
204
|
+
const message = `[testforge:waiting] "${action}" (action) is not in the product yet${why ? ` — ${why}` : ''}: this test runs for real once it is there.`;
|
|
205
|
+
return async () => { throw new Error(message); };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* The product refused an action's own request (actions module, A6): either the
|
|
210
|
+
* action sends the wrong thing or the product is at fault. The reporters file it
|
|
211
|
+
* apart from a rule the test checks, and generate builds the action again.
|
|
212
|
+
*/
|
|
213
|
+
function refused(action, method, path, res) {
|
|
214
|
+
const body = res && typeof res.body === 'string' ? res.body : JSON.stringify(res && res.body);
|
|
215
|
+
return new Error(`[testforge:action] ${action}: ${method} ${path} answered ${res && res.status}: ${String(body).slice(0, 300)}`);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
module.exports = { need, fillPath, waiting, refused, _usersOf: usersOf };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to jest.config.js:
|
|
3
3
|
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle, recordPassed, recordSummary } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts, notTheProduct, plainTitle, recordPassed, recordSummary, recordActionFailures } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgeJestReporter {
|
|
7
7
|
constructor(_globalConfig, options) {
|
|
@@ -38,6 +38,8 @@ module.exports = class TestForgeJestReporter {
|
|
|
38
38
|
}
|
|
39
39
|
recordPassed(this.root, (results.testResults || []).flatMap((f) =>
|
|
40
40
|
(f.testResults || f.assertionResults || []).filter((a) => a.status === 'passed').map((a) => ({ file: f.testFilePath, title: a.title }))));
|
|
41
|
+
recordActionFailures(this.root, (results.testResults || []).flatMap((f) =>
|
|
42
|
+
(f.testResults || f.assertionResults || []).filter((a) => a.status === 'failed').map((a) => (a.failureMessages || [])[0])));
|
|
41
43
|
const durationSec = Math.round((results.startTime ? Date.now() - results.startTime : 0) / 1000);
|
|
42
44
|
const list = Object.values(suites);
|
|
43
45
|
recordSummary(this.root, this.category, list);
|