@activate-core/test-forge 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/docs/doc-guidance.js +15 -5
- package/dist/core/gates/validate.js +6 -1
- package/dist/core/generate/ai-generator.js +43 -28
- package/dist/core/generate/pending.js +4 -0
- package/dist/core/generate/run-generate.js +5 -1
- package/dist/core/generate/suite-planner.js +52 -3
- package/package.json +1 -1
|
@@ -87,6 +87,10 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
87
87
|
` middleware that trims, normalises or converts empty strings, and the`,
|
|
88
88
|
` input mutators the handler applies. A row whose verdict is wrong`,
|
|
89
89
|
` becomes a test that fails on a correct product.`,
|
|
90
|
+
` An [observed] criterion is about what the customer sees or what the`,
|
|
91
|
+
` server accepts or refuses — never an implementation detail (a storage`,
|
|
92
|
+
` key name, a CSS class, an internal flag, a request field name) unless the`,
|
|
93
|
+
` source itself is about it.`,
|
|
90
94
|
` Only what a test can drive is a criterion. Behaviour no screen or`,
|
|
91
95
|
` endpoint can reach ("not tested through the screen") is not tagged —`,
|
|
92
96
|
` leave it out, or mention it in Test setup if the generator needs to know.`,
|
|
@@ -100,8 +104,11 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
100
104
|
` [baseline] item that fits to EACH row, not just to the main form.`,
|
|
101
105
|
`2. COMBINATIONS. When the source states a rule across alternatives and`,
|
|
102
106
|
` conditions ("for login and sign-up, also on a slow connection and on`,
|
|
103
|
-
` a retry"),
|
|
104
|
-
`
|
|
107
|
+
` a retry"), check in the code which of them take a different path. Write`,
|
|
108
|
+
` one criterion per combination that takes its own path; alternatives`,
|
|
109
|
+
` handled by the same code share ONE criterion that names them all. A`,
|
|
110
|
+
` criterion per opener, per page or per input that all reach the same`,
|
|
111
|
+
` code becomes one test per item — volume, not coverage.`,
|
|
105
112
|
`3. CODE BRANCHES. Read the code the journey runs — screens, components,`,
|
|
106
113
|
` client API calls, and the server handlers they reach (related repos`,
|
|
107
114
|
` included) — and walk EVERY validation rule, error response and`,
|
|
@@ -114,9 +121,12 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
114
121
|
`4. QUOTE [spec]. End every [spec] criterion with the source sentence it`,
|
|
115
122
|
` rests on, quoted: (source: "..."). A criterion you cannot quote is`,
|
|
116
123
|
` NOT [spec] — make it [baseline] or [observed], or a Question.`,
|
|
117
|
-
`5. SHARED RULES. A rule shared by several
|
|
118
|
-
`
|
|
119
|
-
`
|
|
124
|
+
`5. SHARED RULES AND SHARED CONTROLS. A rule or control shared by several`,
|
|
125
|
+
` journeys (a password rule, a code-entry step, the login form, sign-out)`,
|
|
126
|
+
` is specified in full ONCE, in the journey whose subject it is. Other`,
|
|
127
|
+
` journeys that pass through it state only their own part: how the`,
|
|
128
|
+
` customer reaches it from there and the outcome specific to that journey,`,
|
|
129
|
+
` and name the journey that holds the full rule.`,
|
|
120
130
|
`6. COVERAGE. After the numbered sections, add "## Coverage": one row per`,
|
|
121
131
|
` control from step 1, naming the section that covers each [baseline]`,
|
|
122
132
|
` item that fits it. A blank cell is a gap — fill it before finishing.`,
|
|
@@ -20,7 +20,7 @@ import { execFileSync } from 'node:child_process';
|
|
|
20
20
|
import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
|
21
21
|
import { join } from 'node:path';
|
|
22
22
|
import { tmpdir } from 'node:os';
|
|
23
|
-
import { reasonlessPending } from '../generate/pending.js';
|
|
23
|
+
import { reasonlessPending, pendingDeclarations } from '../generate/pending.js';
|
|
24
24
|
/**
|
|
25
25
|
* Third-party network SDKs common enough to be worth gating everywhere:
|
|
26
26
|
* a backend test that imports one without mocking it is talking to a real
|
|
@@ -164,6 +164,11 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
164
164
|
'too; scope the stub to one of those, or name the ones you ruled out');
|
|
165
165
|
}
|
|
166
166
|
}
|
|
167
|
+
// F16 — no placeholders in a freshly written file: a placeholder never runs,
|
|
168
|
+
// and a suite full of them reads as coverage it does not have.
|
|
169
|
+
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
170
|
+
return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
|
|
171
|
+
}
|
|
167
172
|
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
168
173
|
// as a bare "fixme", indistinguishable from the tool giving up. Runs on
|
|
169
174
|
// content normalisePending() has already given every reason it could find.
|
|
@@ -77,11 +77,6 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
77
77
|
const framework = isApi
|
|
78
78
|
? 'Jest for an API/backend test'
|
|
79
79
|
: 'Playwright for a web/UI test';
|
|
80
|
-
// Pending-test syntax differs by runner: Jest has test.todo/it.skip;
|
|
81
|
-
// Playwright has NEITHER (test.todo throws) — it uses test.fixme/skip.
|
|
82
|
-
const pending = isApi
|
|
83
|
-
? 'test.todo(...) or it.skip(...)'
|
|
84
|
-
: "test.fixme(title, { annotation: { type: 'fixme', description: '<why it is pending>' } }, async () => {}) — the reason goes in the annotation, where the report shows it; Playwright has NO test.todo/it.*";
|
|
85
80
|
const ext = config.testExtension ?? '.test.ts';
|
|
86
81
|
const isJs = ext.endsWith('.js');
|
|
87
82
|
const language = isJs
|
|
@@ -134,13 +129,14 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
134
129
|
` - Assert the real end state (the response body and status, the`,
|
|
135
130
|
` persisted record, the screen) — not merely that nothing threw, and`,
|
|
136
131
|
` never a value your own stub hard-coded.`,
|
|
137
|
-
` - No line, no assertion:
|
|
132
|
+
` - No line, no assertion: a case the document fixes no outcome for gets`,
|
|
133
|
+
` NO test — not a placeholder either. Neither does an item under`,
|
|
138
134
|
` "Questions for product (not asserted)" or "Suggested", or one the`,
|
|
139
|
-
` document says is not tested through the screen
|
|
140
|
-
`
|
|
135
|
+
` document says is not tested through the screen: it is the product's`,
|
|
136
|
+
` to answer.`,
|
|
141
137
|
` - A security probe asserts the SECURE outcome (refused, rejected,`,
|
|
142
138
|
` idempotent) unless the document records the weakness as accepted; when`,
|
|
143
|
-
` you cannot tell whether a defence exists, the case is
|
|
139
|
+
` you cannot tell whether a defence exists, the case is left out.`,
|
|
144
140
|
`E2. WHAT YOU TOUCH — evidence: this project's source.`,
|
|
145
141
|
` - Every locator, route, request field, response field, storage key and id`,
|
|
146
142
|
` is something you found in THIS project's code (or the CODE FACTS), never`,
|
|
@@ -227,8 +223,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
227
223
|
` - A stub intercepts exactly the dependency — a different host; the same`,
|
|
228
224
|
` host on another port; a path prefix; or a predicate on the request —`,
|
|
229
225
|
` never a URL substring that also matches the app's own pages; then`,
|
|
230
|
-
` confirm the page under test still loads.
|
|
231
|
-
`
|
|
226
|
+
` confirm the page under test still loads. A dependency you truly cannot`,
|
|
227
|
+
` separate is left untested; say which of these you checked in a comment.`,
|
|
232
228
|
` - Real side effects (a message, an email, a charge) sit behind the opt-in`,
|
|
233
229
|
` gate the sibling files use, with one identity per rate-limited send.`,
|
|
234
230
|
` - A login elsewhere can end a cached session: get tokens through the`,
|
|
@@ -238,10 +234,32 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
238
234
|
` (change decoded bytes, not an encoded character), and the valid control`,
|
|
239
235
|
` is asserted too where you can.`,
|
|
240
236
|
``,
|
|
237
|
+
`COVER EACH OUTCOME ONCE. A suite is judged by the defects it can catch, not`,
|
|
238
|
+
`by its size; every redundant test costs run time and buries the ones that`,
|
|
239
|
+
`matter.`,
|
|
240
|
+
` - One test per distinct outcome: each criterion, and each branch or`,
|
|
241
|
+
` boundary the code takes for it (just inside, just outside). Inputs that`,
|
|
242
|
+
` reach the same branch and give the same outcome are ONE case: thirty`,
|
|
243
|
+
` attack strings against a field that only checks "not empty" prove`,
|
|
244
|
+
` nothing the first did not. A parametrised test lists only rows that`,
|
|
245
|
+
` reach different branches, each with a short comment naming its branch.`,
|
|
246
|
+
` The same goes for repeating one check per field, per page, per opener`,
|
|
247
|
+
` or per error code when the same code handles them all.`,
|
|
248
|
+
` - A behaviour the "covers" line of this file says another journey owns is`,
|
|
249
|
+
` not re-tested here: check only how the customer reaches it from this`,
|
|
250
|
+
` journey, and the outcome specific to this journey.`,
|
|
251
|
+
` - Do not assert implementation details — storage key names, CSS classes,`,
|
|
252
|
+
` internal flags, request field names — unless the criterion itself is`,
|
|
253
|
+
` about them (for example, a token must never appear in a URL).`,
|
|
254
|
+
` - Never write a placeholder: no test.fixme / test.skip / test.todo`,
|
|
255
|
+
` declarations. A precondition the environment lacks is a run-time skip`,
|
|
256
|
+
` inside a real test, stating what was observed.`,
|
|
257
|
+
``,
|
|
241
258
|
`SELF-CHECK BEFORE YOU OUTPUT. Go through every test you wrote and name, for`,
|
|
242
|
-
`each line, its evidence under E1–E5
|
|
243
|
-
`
|
|
244
|
-
`
|
|
259
|
+
`each line, its evidence under E1–E5, and the distinct outcome each test`,
|
|
260
|
+
`adds. A line with no evidence is fixed, or its test is removed; a test that`,
|
|
261
|
+
`adds no new outcome is removed. When in doubt, report LESS: a missing test`,
|
|
262
|
+
`costs one line of coverage; a false failure costs the team a release.`,
|
|
245
263
|
``,
|
|
246
264
|
...(grounding
|
|
247
265
|
? [
|
|
@@ -276,9 +294,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
276
294
|
`and let the test FAIL — a real bug surfaced is the goal, never`,
|
|
277
295
|
`something to paper over. NEVER reshape an assertion to match observed`,
|
|
278
296
|
`code behaviour just to make it green; that rubber-stamps the bug. If`,
|
|
279
|
-
`the document does not state the expected outcome for a case,
|
|
280
|
-
`
|
|
281
|
-
`expectation from the code.`,
|
|
297
|
+
`the document does not state the expected outcome for a case, write no`,
|
|
298
|
+
`test for it — do not invent the expectation from the code.`,
|
|
282
299
|
`REQUEST URLS: build every request path as the FULL mounted route the`,
|
|
283
300
|
`router actually registers — INCLUDING the version/area prefix (e.g.`,
|
|
284
301
|
`/v{VERSION}/<area>/<action>, reading the real version from its env var).`,
|
|
@@ -292,12 +309,12 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
292
309
|
`fallback overrides the token for an AUTHENTICATED call: the testkit`,
|
|
293
310
|
`seeds the owner fixtures under the authenticated identity, so expect`,
|
|
294
311
|
`the token owner's rows. A second, non-owner identity (for IDOR/scope`,
|
|
295
|
-
`tests) needs a distinct token; if unavailable,
|
|
312
|
+
`tests) needs a distinct token; if unavailable, the test skips at run time`,
|
|
313
|
+
`naming the missing identity.`,
|
|
296
314
|
`Do NOT write or edit any file; output only the test code to stdout.`,
|
|
297
|
-
`EVERY
|
|
298
|
-
`
|
|
299
|
-
|
|
300
|
-
`testing anything.`,
|
|
315
|
+
`EVERY test MUST contain at least one real assertion. If you genuinely`,
|
|
316
|
+
`cannot assert an outcome, write no test for that case — never a test body`,
|
|
317
|
+
`with no assertion, which passes without testing anything.`,
|
|
301
318
|
``,
|
|
302
319
|
isApi
|
|
303
320
|
? [
|
|
@@ -346,9 +363,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
346
363
|
`ground on role/label/text/name instead of inventing a test-id.`,
|
|
347
364
|
`NEVER invent a data-testid/data-cy: an unverified test-id is`,
|
|
348
365
|
`REJECTED by the selector-grounding gate and wastes the whole file.`,
|
|
349
|
-
`Only when an element cannot be identified by ANY real handle
|
|
350
|
-
`
|
|
351
|
-
`not identify — do not default to pending, never guess.`,
|
|
366
|
+
`Only when an element cannot be identified by ANY real handle, leave`,
|
|
367
|
+
`that single case out and name the control in a comment — never guess.`,
|
|
352
368
|
`NAVIGATION: page.goto(...) takes ONLY a concrete URL or path —`,
|
|
353
369
|
`never a description/comment, and never a route-file placeholder`,
|
|
354
370
|
`like "/s/[shortCode]" or "/user/:id". Fill the path with a real`,
|
|
@@ -358,8 +374,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
358
374
|
`CRITICAL Playwright API: import { test, expect } from`,
|
|
359
375
|
`'@playwright/test'. There is NO test.todo / it.* / describe.skip`,
|
|
360
376
|
`in Playwright — using them throws at collection and breaks the`,
|
|
361
|
-
`WHOLE file.
|
|
362
|
-
`test.describe(...).`,
|
|
377
|
+
`WHOLE file. Group with test.describe(...).`,
|
|
363
378
|
`CRITICAL API-request API: the \`request\` FIXTURE (test args`,
|
|
364
379
|
`\`async ({ request }) => …\`) IS already an APIRequestContext —`,
|
|
365
380
|
`call it DIRECTLY: \`await request.post(url, { data })\`,`,
|
|
@@ -416,7 +431,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
416
431
|
`- Do NOT wrap the code in markdown fences.`,
|
|
417
432
|
`- Tag the top with a comment: // @generated by testforge`,
|
|
418
433
|
`- Never write "TODO" or "FIXME" in the file. Something you cannot`,
|
|
419
|
-
` establish is
|
|
434
|
+
` establish is left out, never a note to finish later.`,
|
|
420
435
|
``,
|
|
421
436
|
`--- PROJECT MAP (data layer) ---`,
|
|
422
437
|
map,
|
|
@@ -116,3 +116,7 @@ export function reasonlessPending(content) {
|
|
|
116
116
|
.filter((d) => !d.hasDetails)
|
|
117
117
|
.map((d) => content.slice(d.open + 1, d.titleEnd + 1).trim().slice(1, 61));
|
|
118
118
|
}
|
|
119
|
+
/** How many declaration-form pending tests (placeholders) the content holds. */
|
|
120
|
+
export function pendingDeclarations(content) {
|
|
121
|
+
return pendingDecls(content).length + (content.match(/\btest\.todo\s*\(/g) || []).length;
|
|
122
|
+
}
|
|
@@ -406,6 +406,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
406
406
|
knownSelectors,
|
|
407
407
|
externalModules: config.externalModules,
|
|
408
408
|
});
|
|
409
|
+
// A freshly written file also may not hold placeholders; a repair is held to
|
|
410
|
+
// the repair guard instead (it may not add a skip), so old files stay repairable.
|
|
411
|
+
const genGate = (content) => testGate(content) ??
|
|
412
|
+
checkTestGates(content, ext, { framework, forbidPlaceholders: true });
|
|
409
413
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
410
414
|
// to retry (a resume pass fills those in without touching real files).
|
|
411
415
|
if (!force &&
|
|
@@ -513,7 +517,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
513
517
|
let status;
|
|
514
518
|
if (useAI) {
|
|
515
519
|
const facts = await grounding();
|
|
516
|
-
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext,
|
|
520
|
+
const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, genGate);
|
|
517
521
|
if (ai !== null) {
|
|
518
522
|
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
519
523
|
status = 'ai';
|
|
@@ -15,11 +15,48 @@
|
|
|
15
15
|
* per-file generator turns each entry into an actual focused test.
|
|
16
16
|
*/
|
|
17
17
|
import { execFileSync } from 'node:child_process';
|
|
18
|
-
import { readFileSync, existsSync } from 'node:fs';
|
|
18
|
+
import { readFileSync, existsSync, readdirSync } from 'node:fs';
|
|
19
|
+
import { join } from 'node:path';
|
|
19
20
|
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
21
|
+
import { PLANS_DIR } from '../paths.js';
|
|
20
22
|
const MAP_PATH = 'testforge.md';
|
|
23
|
+
/**
|
|
24
|
+
* The other journeys in the same area — their names, and the files already
|
|
25
|
+
* planned for them. A planner that sees only its own journey re-tests every
|
|
26
|
+
* shared form, rule and component it touches; on one real suite that was
|
|
27
|
+
* well over half of all tests. With the siblings in view, each behaviour is
|
|
28
|
+
* planned once, in the journey that owns it.
|
|
29
|
+
*/
|
|
30
|
+
export function siblingJourneys(journey, docsDir = 'docs') {
|
|
31
|
+
const lines = [];
|
|
32
|
+
const planned = new Set();
|
|
33
|
+
const dir = join(PLANS_DIR, journey.area);
|
|
34
|
+
for (const f of existsSync(dir) ? readdirSync(dir) : []) {
|
|
35
|
+
const id = f.replace(/\.json$/, '');
|
|
36
|
+
if (!f.endsWith('.json') || id === journey.id)
|
|
37
|
+
continue;
|
|
38
|
+
try {
|
|
39
|
+
const plan = JSON.parse(readFileSync(join(dir, f), 'utf-8'));
|
|
40
|
+
lines.push(`- ${id} (planned):`, ...(plan.files ?? []).map((x) => ` - ${x.title ?? ''}: ${x.covers ?? ''}`));
|
|
41
|
+
planned.add(id);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// unreadable plan — the doc name below still names the journey
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const area = join(docsDir, journey.area);
|
|
48
|
+
for (const f of existsSync(area) ? readdirSync(area) : []) {
|
|
49
|
+
if (!f.endsWith('.md'))
|
|
50
|
+
continue;
|
|
51
|
+
const name = (readFileSync(join(area, f), 'utf-8').match(/^# Journey:\s*(.+)$/m) ?? [])[1];
|
|
52
|
+
if (name && name.trim() !== journey.name && !lines.some((l) => l.includes(name.trim()))) {
|
|
53
|
+
lines.push(`- ${name.trim()} (not planned yet)`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return lines.join('\n');
|
|
57
|
+
}
|
|
21
58
|
/** Build the planning prompt for one journey. */
|
|
22
|
-
function buildPlanPrompt(journey, map) {
|
|
59
|
+
export function buildPlanPrompt(journey, map, siblings = '') {
|
|
23
60
|
return [
|
|
24
61
|
`You are a senior test architect. Plan how to split ONE user journey`,
|
|
25
62
|
`into a set of FOCUSED test files — one concern per file — the way a`,
|
|
@@ -43,7 +80,19 @@ function buildPlanPrompt(journey, map) {
|
|
|
43
80
|
`- "## Controls" and "## Coverage" are indexes, "## Test setup" is what`,
|
|
44
81
|
` every file needs to run, and "## Questions for product (not`,
|
|
45
82
|
` asserted)" is not tested: plan no file for any of them.`,
|
|
83
|
+
`- ONE OWNER PER BEHAVIOUR. Every behaviour is tested in exactly one`,
|
|
84
|
+
` file of exactly one journey. A form, component or rule this journey`,
|
|
85
|
+
` shares with another journey (a login form, a code-entry step, a`,
|
|
86
|
+
` password rule, a sign-out) belongs to the journey whose subject it is;`,
|
|
87
|
+
` this journey tests only what is its OWN: how the customer reaches it`,
|
|
88
|
+
` from here, and the outcome specific to this journey. Say so in`,
|
|
89
|
+
` "covers" ("opening the login form from X and landing on X after`,
|
|
90
|
+
` login — the form's own validation is owned by <journey>").`,
|
|
91
|
+
`- Within this journey, no two files test the same behaviour.`,
|
|
46
92
|
``,
|
|
93
|
+
...(siblings
|
|
94
|
+
? [`OTHER JOURNEYS IN THIS AREA (what they own or will own):`, siblings, ``]
|
|
95
|
+
: []),
|
|
47
96
|
`Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
|
|
48
97
|
` [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,
|
|
49
98
|
` "covers": "<one or two sentences: exactly what this file tests>" }]`,
|
|
@@ -99,7 +148,7 @@ function safeSlug(raw) {
|
|
|
99
148
|
*/
|
|
100
149
|
export function planSuite(journey) {
|
|
101
150
|
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
102
|
-
const prompt = buildPlanPrompt(journey, map);
|
|
151
|
+
const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey));
|
|
103
152
|
const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
|
|
104
153
|
let raw = '';
|
|
105
154
|
try {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|