@activate-core/test-forge 1.0.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +27 -2
- package/dist/core/docs/doc-guidance.js +25 -5
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +17 -1
- package/dist/core/generate/ai-generator.js +208 -283
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +166 -236
- package/dist/core/generate/pending.js +122 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +63 -68
- package/dist/core/generate/suite-planner.js +52 -3
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
package/README.md
CHANGED
|
@@ -335,11 +335,41 @@ The test runner produces its own pass/fail report.
|
|
|
335
335
|
Every AI phase may read them, so the map captures the real data layer and
|
|
336
336
|
tests can check what the API stores. A headless run cannot read outside
|
|
337
337
|
the project otherwise.
|
|
338
|
+
- **decisions** — a file or link listing what is NOT a requirement yet:
|
|
339
|
+
open product decisions and known issues left out on purpose. Docs do not
|
|
340
|
+
turn them into criteria, and a failing test that falls under one is
|
|
341
|
+
reported as a pending decision, never as a defect. Keeps the reference
|
|
342
|
+
you share with a client clean while the tests still know what is undecided.
|
|
343
|
+
|
|
344
|
+
## How `generate` checks its own work
|
|
345
|
+
|
|
346
|
+
Every test is run against the real app before it is handed over, so a
|
|
347
|
+
mistake in a generated test does not reach your team as a product defect:
|
|
348
|
+
|
|
349
|
+
1. **Each file on its own.** A test that fails because of its own mistake
|
|
350
|
+
(a wrong locator, a field the server never sends, a bad precondition) is
|
|
351
|
+
repaired from the runner's output. A guard refuses any repair that
|
|
352
|
+
deletes a test, adds a skip or removes an assertion.
|
|
353
|
+
2. **The whole journey together**, the way CI runs it. Files that pass alone
|
|
354
|
+
but break each other (one signs in again as a shared account and ends
|
|
355
|
+
another's session) are repaired with that context.
|
|
356
|
+
3. **An independent review of every claimed defect.** A separate reviewer
|
|
357
|
+
starts from "not proven" and must find the application code at fault.
|
|
358
|
+
`generate` then lists, one line per root cause:
|
|
359
|
+
- **defects**: a [spec] requirement the app breaks;
|
|
360
|
+
- **findings**: real behaviour against a [baseline]/[observed] criterion;
|
|
361
|
+
- **pending decisions**: covered by `decisions`.
|
|
362
|
+
A claim it rejects is repaired once more. Anything still unexplained is
|
|
363
|
+
reported as unverified, with the advice not to report it.
|
|
364
|
+
|
|
365
|
+
Each run is time-limited (`verifyTimeoutMs`, per file). On expiry the
|
|
366
|
+
runner stops with everything it started.
|
|
338
367
|
|
|
339
368
|
## Notes
|
|
340
369
|
|
|
341
|
-
- Generated tests and docs are a strong draft, not final. Review them
|
|
342
|
-
|
|
370
|
+
- Generated tests and docs are a strong draft, not final. Review them.
|
|
371
|
+
Where something could not be established, a test is pending, with the
|
|
372
|
+
reason in its annotation (and in the dashboard).
|
|
343
373
|
- Secrets are never written to docs, configs, or tests. Connections use
|
|
344
374
|
environment variable names only; real values come from your
|
|
345
375
|
environment at run time.
|
package/dist/cli/index.js
CHANGED
|
@@ -29,7 +29,7 @@ import { runUpdate } from '../core/project/update.js';
|
|
|
29
29
|
import { runEnvTemplate } from '../core/project/env-template.js';
|
|
30
30
|
import { loadConfig } from '../core/project/config.js';
|
|
31
31
|
import { publishReport, loadReportConfig, loadDotenv, scanInventory, publishInventory } from '../core/run/report.js';
|
|
32
|
-
import { dirname } from 'node:path';
|
|
32
|
+
import { basename, dirname } from 'node:path';
|
|
33
33
|
const command = process.argv[2];
|
|
34
34
|
const useBasic = process.argv.includes('--basic');
|
|
35
35
|
const force = process.argv.includes('--force');
|
|
@@ -190,6 +190,9 @@ async function main() {
|
|
|
190
190
|
for (const w of r.warnings ?? []) {
|
|
191
191
|
console.log(` ! ${w}`);
|
|
192
192
|
}
|
|
193
|
+
if (r.verified === true)
|
|
194
|
+
console.log(' verified: every file ran; every failure left was reviewed and is listed below');
|
|
195
|
+
printReviewed(r.reviewed ?? []);
|
|
193
196
|
}
|
|
194
197
|
const generated = results.filter((r) => r.status !== 'skipped').length;
|
|
195
198
|
// Rows with no test dir are notices (a non-journey file, the AI going
|
|
@@ -239,7 +242,7 @@ async function main() {
|
|
|
239
242
|
return;
|
|
240
243
|
}
|
|
241
244
|
if (!resultsPath || !project || !dashboardUrl || !apiKey) {
|
|
242
|
-
console.error('Usage: testforge report --results <jest-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
|
|
245
|
+
console.error('Usage: testforge report --results <jest-or-playwright-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
|
|
243
246
|
console.error(' project / dashboard may also come from testforge.config.json.');
|
|
244
247
|
console.error(' TESTFORGE_KEY must be set in the environment (the key the dashboard gave you).');
|
|
245
248
|
process.exit(1);
|
|
@@ -257,6 +260,28 @@ async function main() {
|
|
|
257
260
|
printUsage();
|
|
258
261
|
}
|
|
259
262
|
}
|
|
263
|
+
/** Reviewed failures, one line per root cause, strongest first. */
|
|
264
|
+
function printReviewed(reviewed) {
|
|
265
|
+
const sections = [
|
|
266
|
+
['defect', 'DEFECTS — a [spec] requirement the app breaks'],
|
|
267
|
+
['finding', 'FINDINGS — real app behaviour against a [baseline]/[observed] criterion (lower priority)'],
|
|
268
|
+
['pending-decision', 'PENDING DECISIONS — not requirements yet; take them to product, do not report as defects'],
|
|
269
|
+
];
|
|
270
|
+
for (const [verdict, heading] of sections) {
|
|
271
|
+
const rows = reviewed.filter((r) => r.verdict === verdict);
|
|
272
|
+
if (!rows.length)
|
|
273
|
+
continue;
|
|
274
|
+
console.log(` ${heading}:`);
|
|
275
|
+
for (const r of rows) {
|
|
276
|
+
const where = r.tests.slice(0, 3).map((t) => `${basename(t.file)}:${t.line}`).join(', ');
|
|
277
|
+
const more = r.tests.length > 3 ? ` +${r.tests.length - 3} more` : '';
|
|
278
|
+
console.log(` - ${r.claim}`);
|
|
279
|
+
console.log(` ${r.tests.length} test(s): ${where}${more}${r.evidence ? ` · app: ${r.evidence}` : ''}`);
|
|
280
|
+
if (r.reason)
|
|
281
|
+
console.log(` ${r.reason}`);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
260
285
|
main().catch((err) => {
|
|
261
286
|
console.error(err instanceof Error ? err.message : String(err));
|
|
262
287
|
process.exit(1);
|
|
@@ -80,6 +80,20 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
80
80
|
` same way (the screen accepts exactly what the server accepts). A`,
|
|
81
81
|
` failure is an inconsistency, not a spec defect. Never use [observed]`,
|
|
82
82
|
` to override or soften a [spec] criterion.`,
|
|
83
|
+
` CHECK AN [observed] RULE BEFORE YOU STATE IT. Run each pattern in your`,
|
|
84
|
+
` head against the concrete values you list — what it really accepts and`,
|
|
85
|
+
` refuses, not what its name suggests. For the server's verdict, follow`,
|
|
86
|
+
` the request through everything that runs BEFORE validation: global`,
|
|
87
|
+
` middleware that trims, normalises or converts empty strings, and the`,
|
|
88
|
+
` input mutators the handler applies. A row whose verdict is wrong`,
|
|
89
|
+
` becomes a test that fails on a correct product.`,
|
|
90
|
+
` An [observed] criterion is about what the customer sees or what the`,
|
|
91
|
+
` server accepts or refuses — never an implementation detail (a storage`,
|
|
92
|
+
` key name, a CSS class, an internal flag, a request field name) unless the`,
|
|
93
|
+
` source itself is about it.`,
|
|
94
|
+
` Only what a test can drive is a criterion. Behaviour no screen or`,
|
|
95
|
+
` endpoint can reach ("not tested through the screen") is not tagged —`,
|
|
96
|
+
` leave it out, or mention it in Test setup if the generator needs to know.`,
|
|
83
97
|
``,
|
|
84
98
|
`METHOD. Work through these steps for every journey, in order:`,
|
|
85
99
|
`1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
|
|
@@ -90,8 +104,11 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
90
104
|
` [baseline] item that fits to EACH row, not just to the main form.`,
|
|
91
105
|
`2. COMBINATIONS. When the source states a rule across alternatives and`,
|
|
92
106
|
` conditions ("for login and sign-up, also on a slow connection and on`,
|
|
93
|
-
` a retry"),
|
|
94
|
-
`
|
|
107
|
+
` a retry"), check in the code which of them take a different path. Write`,
|
|
108
|
+
` one criterion per combination that takes its own path; alternatives`,
|
|
109
|
+
` handled by the same code share ONE criterion that names them all. A`,
|
|
110
|
+
` criterion per opener, per page or per input that all reach the same`,
|
|
111
|
+
` code becomes one test per item — volume, not coverage.`,
|
|
95
112
|
`3. CODE BRANCHES. Read the code the journey runs — screens, components,`,
|
|
96
113
|
` client API calls, and the server handlers they reach (related repos`,
|
|
97
114
|
` included) — and walk EVERY validation rule, error response and`,
|
|
@@ -104,9 +121,12 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
104
121
|
`4. QUOTE [spec]. End every [spec] criterion with the source sentence it`,
|
|
105
122
|
` rests on, quoted: (source: "..."). A criterion you cannot quote is`,
|
|
106
123
|
` NOT [spec] — make it [baseline] or [observed], or a Question.`,
|
|
107
|
-
`5. SHARED RULES. A rule shared by several
|
|
108
|
-
`
|
|
109
|
-
`
|
|
124
|
+
`5. SHARED RULES AND SHARED CONTROLS. A rule or control shared by several`,
|
|
125
|
+
` journeys (a password rule, a code-entry step, the login form, sign-out)`,
|
|
126
|
+
` is specified in full ONCE, in the journey whose subject it is. Other`,
|
|
127
|
+
` journeys that pass through it state only their own part: how the`,
|
|
128
|
+
` customer reaches it from there and the outcome specific to that journey,`,
|
|
129
|
+
` and name the journey that holds the full rule.`,
|
|
110
130
|
`6. COVERAGE. After the numbered sections, add "## Coverage": one row per`,
|
|
111
131
|
` control from step 1, naming the section that covers each [baseline]`,
|
|
112
132
|
` item that fits it. A blank cell is a gap — fill it before finishing.`,
|
|
@@ -109,6 +109,15 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
|
|
|
109
109
|
DEEP_DOC_GUIDANCE,
|
|
110
110
|
``,
|
|
111
111
|
`Reference: ${reference}`,
|
|
112
|
+
...(loadConfig().decisions
|
|
113
|
+
? [
|
|
114
|
+
`NOT YET REQUIREMENTS: the project keeps its open product decisions and`,
|
|
115
|
+
`known issues left out on purpose at: ${loadConfig().decisions}`,
|
|
116
|
+
`Read it. Nothing listed there becomes a criterion — no [spec], no`,
|
|
117
|
+
`[observed], no "Likely defect"; name it under "Questions for product`,
|
|
118
|
+
`(not asserted)" if the journey touches it.`,
|
|
119
|
+
]
|
|
120
|
+
: []),
|
|
112
121
|
`Read it FULLY with your available tools (read the file, fetch the`,
|
|
113
122
|
`URL/page — follow sub-pages/links the reference points to). Do not`,
|
|
114
123
|
`summarise shallowly; pull out every flow, rule, state and edge case`,
|
|
@@ -20,6 +20,7 @@ import { execFileSync } from 'node:child_process';
|
|
|
20
20
|
import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
|
21
21
|
import { join } from 'node:path';
|
|
22
22
|
import { tmpdir } from 'node:os';
|
|
23
|
+
import { reasonlessPending, pendingDeclarations } from '../generate/pending.js';
|
|
23
24
|
/**
|
|
24
25
|
* Third-party network SDKs common enough to be worth gating everywhere:
|
|
25
26
|
* a backend test that imports one without mocking it is talking to a real
|
|
@@ -163,6 +164,21 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
163
164
|
'too; scope the stub to one of those, or name the ones you ruled out');
|
|
164
165
|
}
|
|
165
166
|
}
|
|
167
|
+
// F16 — no placeholders in a freshly written file: a placeholder never runs,
|
|
168
|
+
// and a suite full of them reads as coverage it does not have.
|
|
169
|
+
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
170
|
+
return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
|
|
171
|
+
}
|
|
172
|
+
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
173
|
+
// as a bare "fixme", indistinguishable from the tool giving up. Runs on
|
|
174
|
+
// content normalisePending() has already given every reason it could find.
|
|
175
|
+
if (ctx.framework === 'playwright') {
|
|
176
|
+
const bare = reasonlessPending(content);
|
|
177
|
+
if (bare.length) {
|
|
178
|
+
return (`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
|
|
179
|
+
`test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
166
182
|
// F10 — selector grounding: a RUNNABLE (non-skipped) UI test must not lean
|
|
167
183
|
// on a guessed selector. Unchecked, the generator invents plausible
|
|
168
184
|
// data-testids that time out at runtime — the journey is never really
|
|
@@ -221,7 +237,7 @@ function stripPendingBlocks(content) {
|
|
|
221
237
|
* skipping over quoted strings, template literals, and comments. Returns -1
|
|
222
238
|
* if no match is found.
|
|
223
239
|
*/
|
|
224
|
-
function matchParen(s, open) {
|
|
240
|
+
export function matchParen(s, open) {
|
|
225
241
|
let depth = 0;
|
|
226
242
|
for (let i = open; i < s.length; i++) {
|
|
227
243
|
const c = s[i];
|