@activate-core/test-forge 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +29 -11
- package/dist/core/docs/doc-guidance.js +10 -0
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +12 -1
- package/dist/core/generate/ai-generator.js +181 -267
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +181 -226
- package/dist/core/generate/pending.js +118 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +70 -73
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
package/README.md
CHANGED
|
@@ -335,11 +335,41 @@ The test runner produces its own pass/fail report.
|
|
|
335
335
|
Every AI phase may read them, so the map captures the real data layer and
|
|
336
336
|
tests can check what the API stores. A headless run cannot read outside
|
|
337
337
|
the project otherwise.
|
|
338
|
+
- **decisions** — a file or link listing what is NOT a requirement yet:
|
|
339
|
+
open product decisions and known issues left out on purpose. Docs do not
|
|
340
|
+
turn them into criteria, and a failing test that falls under one is
|
|
341
|
+
reported as a pending decision, never as a defect. Keeps the reference
|
|
342
|
+
you share with a client clean while the tests still know what is undecided.
|
|
343
|
+
|
|
344
|
+
## How `generate` checks its own work
|
|
345
|
+
|
|
346
|
+
Every test is run against the real app before it is handed over, so a
|
|
347
|
+
mistake in a generated test does not reach your team as a product defect:
|
|
348
|
+
|
|
349
|
+
1. **Each file on its own.** A test that fails because of its own mistake
|
|
350
|
+
(a wrong locator, a field the server never sends, a bad precondition) is
|
|
351
|
+
repaired from the runner's output. A guard refuses any repair that
|
|
352
|
+
deletes a test, adds a skip or removes an assertion.
|
|
353
|
+
2. **The whole journey together**, the way CI runs it. Files that pass alone
|
|
354
|
+
but break each other (one signs in again as a shared account and ends
|
|
355
|
+
another's session) are repaired with that context.
|
|
356
|
+
3. **An independent review of every claimed defect.** A separate reviewer
|
|
357
|
+
starts from "not proven" and must find the application code at fault.
|
|
358
|
+
`generate` then lists, one line per root cause:
|
|
359
|
+
- **defects**: a [spec] requirement the app breaks;
|
|
360
|
+
- **findings**: real behaviour against a [baseline]/[observed] criterion;
|
|
361
|
+
- **pending decisions**: covered by `decisions`.
|
|
362
|
+
A claim it rejects is repaired once more. Anything still unexplained is
|
|
363
|
+
reported as unverified, with the advice not to report it.
|
|
364
|
+
|
|
365
|
+
Each run is time-limited (`verifyTimeoutMs`, per file). On expiry the
|
|
366
|
+
runner stops with everything it started.
|
|
338
367
|
|
|
339
368
|
## Notes
|
|
340
369
|
|
|
341
|
-
- Generated tests and docs are a strong draft, not final. Review them
|
|
342
|
-
|
|
370
|
+
- Generated tests and docs are a strong draft, not final. Review them.
|
|
371
|
+
Where something could not be established, a test is pending, with the
|
|
372
|
+
reason in its annotation (and in the dashboard).
|
|
343
373
|
- Secrets are never written to docs, configs, or tests. Connections use
|
|
344
374
|
environment variable names only; real values come from your
|
|
345
375
|
environment at run time.
|
package/dist/cli/index.js
CHANGED
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
* (an auth-gated page can move unseen).
|
|
20
20
|
*/
|
|
21
21
|
import { VERSION } from '../core/index.js';
|
|
22
|
-
import { runGenerate
|
|
22
|
+
import { runGenerate } from '../core/generate/run-generate.js';
|
|
23
23
|
import { runTests } from '../core/run/run-tests.js';
|
|
24
24
|
import { runDiscover } from '../core/docs/discover.js';
|
|
25
25
|
import { runMap } from '../core/project/map.js';
|
|
@@ -29,7 +29,7 @@ import { runUpdate } from '../core/project/update.js';
|
|
|
29
29
|
import { runEnvTemplate } from '../core/project/env-template.js';
|
|
30
30
|
import { loadConfig } from '../core/project/config.js';
|
|
31
31
|
import { publishReport, loadReportConfig, loadDotenv, scanInventory, publishInventory } from '../core/run/report.js';
|
|
32
|
-
import { dirname } from 'node:path';
|
|
32
|
+
import { basename, dirname } from 'node:path';
|
|
33
33
|
const command = process.argv[2];
|
|
34
34
|
const useBasic = process.argv.includes('--basic');
|
|
35
35
|
const force = process.argv.includes('--force');
|
|
@@ -79,7 +79,7 @@ function printUsage() {
|
|
|
79
79
|
console.log(' testforge generate --force Regenerate everything');
|
|
80
80
|
console.log(' testforge generate --no-wait Stop when the AI hits its usage limit (default: wait for the reset and resume)');
|
|
81
81
|
console.log(' testforge generate --no-verify Skip running each journey once after generating (default: run and repair)');
|
|
82
|
-
console.log(' testforge generate --journey <name> Generate just ONE journey'
|
|
82
|
+
console.log(' testforge generate --journey <name> Generate just ONE journey');
|
|
83
83
|
console.log(' testforge generate --area <area> Generate a whole area (e.g. api)');
|
|
84
84
|
console.log(' testforge discover Draft journey docs (asks: map or reference)');
|
|
85
85
|
console.log(' testforge discover --journey <name> Draft just ONE journey doc');
|
|
@@ -144,13 +144,6 @@ async function main() {
|
|
|
144
144
|
});
|
|
145
145
|
process.exitCode = code;
|
|
146
146
|
}
|
|
147
|
-
else if (command === 'generate' && process.argv.includes('--harness')) {
|
|
148
|
-
// Regenerate only the shared harness (testkit), leaving every test file
|
|
149
|
-
// as it is — e.g. after a TestForge upgrade changes what a harness does.
|
|
150
|
-
console.log('Regenerating the shared test harness...\n');
|
|
151
|
-
const written = await runHarness(flagValue('--area'));
|
|
152
|
-
console.log(written.length ? `Written:\n ${written.join('\n ')}` : 'No harness written.');
|
|
153
|
-
}
|
|
154
147
|
else if (command === 'generate') {
|
|
155
148
|
const useAI = !useBasic;
|
|
156
149
|
console.log(useAI
|
|
@@ -197,6 +190,9 @@ async function main() {
|
|
|
197
190
|
for (const w of r.warnings ?? []) {
|
|
198
191
|
console.log(` ! ${w}`);
|
|
199
192
|
}
|
|
193
|
+
if (r.verified === true)
|
|
194
|
+
console.log(' verified: every file ran; every failure left was reviewed and is listed below');
|
|
195
|
+
printReviewed(r.reviewed ?? []);
|
|
200
196
|
}
|
|
201
197
|
const generated = results.filter((r) => r.status !== 'skipped').length;
|
|
202
198
|
// Rows with no test dir are notices (a non-journey file, the AI going
|
|
@@ -246,7 +242,7 @@ async function main() {
|
|
|
246
242
|
return;
|
|
247
243
|
}
|
|
248
244
|
if (!resultsPath || !project || !dashboardUrl || !apiKey) {
|
|
249
|
-
console.error('Usage: testforge report --results <jest-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
|
|
245
|
+
console.error('Usage: testforge report --results <jest-or-playwright-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
|
|
250
246
|
console.error(' project / dashboard may also come from testforge.config.json.');
|
|
251
247
|
console.error(' TESTFORGE_KEY must be set in the environment (the key the dashboard gave you).');
|
|
252
248
|
process.exit(1);
|
|
@@ -264,6 +260,28 @@ async function main() {
|
|
|
264
260
|
printUsage();
|
|
265
261
|
}
|
|
266
262
|
}
|
|
263
|
+
/** Reviewed failures, one line per root cause, strongest first. */
|
|
264
|
+
function printReviewed(reviewed) {
|
|
265
|
+
const sections = [
|
|
266
|
+
['defect', 'DEFECTS — a [spec] requirement the app breaks'],
|
|
267
|
+
['finding', 'FINDINGS — real app behaviour against a [baseline]/[observed] criterion (lower priority)'],
|
|
268
|
+
['pending-decision', 'PENDING DECISIONS — not requirements yet; take them to product, do not report as defects'],
|
|
269
|
+
];
|
|
270
|
+
for (const [verdict, heading] of sections) {
|
|
271
|
+
const rows = reviewed.filter((r) => r.verdict === verdict);
|
|
272
|
+
if (!rows.length)
|
|
273
|
+
continue;
|
|
274
|
+
console.log(` ${heading}:`);
|
|
275
|
+
for (const r of rows) {
|
|
276
|
+
const where = r.tests.slice(0, 3).map((t) => `${basename(t.file)}:${t.line}`).join(', ');
|
|
277
|
+
const more = r.tests.length > 3 ? ` +${r.tests.length - 3} more` : '';
|
|
278
|
+
console.log(` - ${r.claim}`);
|
|
279
|
+
console.log(` ${r.tests.length} test(s): ${where}${more}${r.evidence ? ` · app: ${r.evidence}` : ''}`);
|
|
280
|
+
if (r.reason)
|
|
281
|
+
console.log(` ${r.reason}`);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
267
285
|
main().catch((err) => {
|
|
268
286
|
console.error(err instanceof Error ? err.message : String(err));
|
|
269
287
|
process.exit(1);
|
|
@@ -80,6 +80,16 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
80
80
|
` same way (the screen accepts exactly what the server accepts). A`,
|
|
81
81
|
` failure is an inconsistency, not a spec defect. Never use [observed]`,
|
|
82
82
|
` to override or soften a [spec] criterion.`,
|
|
83
|
+
` CHECK AN [observed] RULE BEFORE YOU STATE IT. Run each pattern in your`,
|
|
84
|
+
` head against the concrete values you list — what it really accepts and`,
|
|
85
|
+
` refuses, not what its name suggests. For the server's verdict, follow`,
|
|
86
|
+
` the request through everything that runs BEFORE validation: global`,
|
|
87
|
+
` middleware that trims, normalises or converts empty strings, and the`,
|
|
88
|
+
` input mutators the handler applies. A row whose verdict is wrong`,
|
|
89
|
+
` becomes a test that fails on a correct product.`,
|
|
90
|
+
` Only what a test can drive is a criterion. Behaviour no screen or`,
|
|
91
|
+
` endpoint can reach ("not tested through the screen") is not tagged —`,
|
|
92
|
+
` leave it out, or mention it in Test setup if the generator needs to know.`,
|
|
83
93
|
``,
|
|
84
94
|
`METHOD. Work through these steps for every journey, in order:`,
|
|
85
95
|
`1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
|
|
@@ -109,6 +109,15 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
|
|
|
109
109
|
DEEP_DOC_GUIDANCE,
|
|
110
110
|
``,
|
|
111
111
|
`Reference: ${reference}`,
|
|
112
|
+
...(loadConfig().decisions
|
|
113
|
+
? [
|
|
114
|
+
`NOT YET REQUIREMENTS: the project keeps its open product decisions and`,
|
|
115
|
+
`known issues left out on purpose at: ${loadConfig().decisions}`,
|
|
116
|
+
`Read it. Nothing listed there becomes a criterion — no [spec], no`,
|
|
117
|
+
`[observed], no "Likely defect"; name it under "Questions for product`,
|
|
118
|
+
`(not asserted)" if the journey touches it.`,
|
|
119
|
+
]
|
|
120
|
+
: []),
|
|
112
121
|
`Read it FULLY with your available tools (read the file, fetch the`,
|
|
113
122
|
`URL/page — follow sub-pages/links the reference points to). Do not`,
|
|
114
123
|
`summarise shallowly; pull out every flow, rule, state and edge case`,
|
|
@@ -20,6 +20,7 @@ import { execFileSync } from 'node:child_process';
|
|
|
20
20
|
import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
|
21
21
|
import { join } from 'node:path';
|
|
22
22
|
import { tmpdir } from 'node:os';
|
|
23
|
+
import { reasonlessPending } from '../generate/pending.js';
|
|
23
24
|
/**
|
|
24
25
|
* Third-party network SDKs common enough to be worth gating everywhere:
|
|
25
26
|
* a backend test that imports one without mocking it is talking to a real
|
|
@@ -163,6 +164,16 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
163
164
|
'too; scope the stub to one of those, or name the ones you ruled out');
|
|
164
165
|
}
|
|
165
166
|
}
|
|
167
|
+
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
168
|
+
// as a bare "fixme", indistinguishable from the tool giving up. Runs on
|
|
169
|
+
// content normalisePending() has already given every reason it could find.
|
|
170
|
+
if (ctx.framework === 'playwright') {
|
|
171
|
+
const bare = reasonlessPending(content);
|
|
172
|
+
if (bare.length) {
|
|
173
|
+
return (`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
|
|
174
|
+
`test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
166
177
|
// F10 — selector grounding: a RUNNABLE (non-skipped) UI test must not lean
|
|
167
178
|
// on a guessed selector. Unchecked, the generator invents plausible
|
|
168
179
|
// data-testids that time out at runtime — the journey is never really
|
|
@@ -221,7 +232,7 @@ function stripPendingBlocks(content) {
|
|
|
221
232
|
* skipping over quoted strings, template literals, and comments. Returns -1
|
|
222
233
|
* if no match is found.
|
|
223
234
|
*/
|
|
224
|
-
function matchParen(s, open) {
|
|
235
|
+
export function matchParen(s, open) {
|
|
225
236
|
let depth = 0;
|
|
226
237
|
for (let i = open; i < s.length; i++) {
|
|
227
238
|
const c = s[i];
|