@activate-core/test-forge 1.0.4 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -2
- package/dist/cli/index.js +27 -2
- package/dist/core/docs/doc-guidance.js +10 -0
- package/dist/core/docs/reference.js +9 -0
- package/dist/core/gates/validate.js +12 -1
- package/dist/core/generate/ai-generator.js +181 -271
- package/dist/core/generate/defect-review.js +177 -0
- package/dist/core/generate/harness.js +166 -236
- package/dist/core/generate/pending.js +118 -0
- package/dist/core/generate/playwright-writer.js +4 -4
- package/dist/core/generate/run-generate.js +59 -68
- package/dist/core/generate/verify-journey.js +162 -0
- package/dist/core/generate/verify.js +227 -67
- package/dist/core/project/env-template.js +3 -1
- package/dist/core/run/report.js +55 -20
- package/dist/core/run/run-tests.js +10 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +54 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +6 -2
package/README.md
CHANGED
|
@@ -335,11 +335,41 @@ The test runner produces its own pass/fail report.
|
|
|
335
335
|
Every AI phase may read them, so the map captures the real data layer and
|
|
336
336
|
tests can check what the API stores. A headless run cannot read outside
|
|
337
337
|
the project otherwise.
|
|
338
|
+
- **decisions** — a file or link listing what is NOT a requirement yet:
|
|
339
|
+
open product decisions and known issues left out on purpose. Docs do not
|
|
340
|
+
turn them into criteria, and a failing test that falls under one is
|
|
341
|
+
reported as a pending decision, never as a defect. Keeps the reference
|
|
342
|
+
you share with a client clean while the tests still know what is undecided.
|
|
343
|
+
|
|
344
|
+
## How `generate` checks its own work
|
|
345
|
+
|
|
346
|
+
Every test is run against the real app before it is handed over, so a
|
|
347
|
+
mistake in a generated test does not reach your team as a product defect:
|
|
348
|
+
|
|
349
|
+
1. **Each file on its own.** A test that fails because of its own mistake
|
|
350
|
+
(a wrong locator, a field the server never sends, a bad precondition) is
|
|
351
|
+
repaired from the runner's output. A guard refuses any repair that
|
|
352
|
+
deletes a test, adds a skip or removes an assertion.
|
|
353
|
+
2. **The whole journey together**, the way CI runs it. Files that pass alone
|
|
354
|
+
but break each other (one signs in again as a shared account and ends
|
|
355
|
+
another's session) are repaired with that context.
|
|
356
|
+
3. **An independent review of every claimed defect.** A separate reviewer
|
|
357
|
+
starts from "not proven" and must find the application code at fault.
|
|
358
|
+
`generate` then lists, one line per root cause:
|
|
359
|
+
- **defects**: a [spec] requirement the app breaks;
|
|
360
|
+
- **findings**: real behaviour against a [baseline]/[observed] criterion;
|
|
361
|
+
- **pending decisions**: covered by `decisions`.
|
|
362
|
+
A claim it rejects is repaired once more. Anything still unexplained is
|
|
363
|
+
reported as unverified, with the advice not to report it.
|
|
364
|
+
|
|
365
|
+
Each run is time-limited (`verifyTimeoutMs`, per file). On expiry the
|
|
366
|
+
runner stops with everything it started.
|
|
338
367
|
|
|
339
368
|
## Notes
|
|
340
369
|
|
|
341
|
-
- Generated tests and docs are a strong draft, not final. Review them
|
|
342
|
-
|
|
370
|
+
- Generated tests and docs are a strong draft, not final. Review them.
|
|
371
|
+
Where something could not be established, a test is pending, with the
|
|
372
|
+
reason in its annotation (and in the dashboard).
|
|
343
373
|
- Secrets are never written to docs, configs, or tests. Connections use
|
|
344
374
|
environment variable names only; real values come from your
|
|
345
375
|
environment at run time.
|
package/dist/cli/index.js
CHANGED
|
@@ -29,7 +29,7 @@ import { runUpdate } from '../core/project/update.js';
|
|
|
29
29
|
import { runEnvTemplate } from '../core/project/env-template.js';
|
|
30
30
|
import { loadConfig } from '../core/project/config.js';
|
|
31
31
|
import { publishReport, loadReportConfig, loadDotenv, scanInventory, publishInventory } from '../core/run/report.js';
|
|
32
|
-
import { dirname } from 'node:path';
|
|
32
|
+
import { basename, dirname } from 'node:path';
|
|
33
33
|
const command = process.argv[2];
|
|
34
34
|
const useBasic = process.argv.includes('--basic');
|
|
35
35
|
const force = process.argv.includes('--force');
|
|
@@ -190,6 +190,9 @@ async function main() {
|
|
|
190
190
|
for (const w of r.warnings ?? []) {
|
|
191
191
|
console.log(` ! ${w}`);
|
|
192
192
|
}
|
|
193
|
+
if (r.verified === true)
|
|
194
|
+
console.log(' verified: every file ran; every failure left was reviewed and is listed below');
|
|
195
|
+
printReviewed(r.reviewed ?? []);
|
|
193
196
|
}
|
|
194
197
|
const generated = results.filter((r) => r.status !== 'skipped').length;
|
|
195
198
|
// Rows with no test dir are notices (a non-journey file, the AI going
|
|
@@ -239,7 +242,7 @@ async function main() {
|
|
|
239
242
|
return;
|
|
240
243
|
}
|
|
241
244
|
if (!resultsPath || !project || !dashboardUrl || !apiKey) {
|
|
242
|
-
console.error('Usage: testforge report --results <jest-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
|
|
245
|
+
console.error('Usage: testforge report --results <jest-or-playwright-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
|
|
243
246
|
console.error(' project / dashboard may also come from testforge.config.json.');
|
|
244
247
|
console.error(' TESTFORGE_KEY must be set in the environment (the key the dashboard gave you).');
|
|
245
248
|
process.exit(1);
|
|
@@ -257,6 +260,28 @@ async function main() {
|
|
|
257
260
|
printUsage();
|
|
258
261
|
}
|
|
259
262
|
}
|
|
263
|
+
/** Reviewed failures, one line per root cause, strongest first. */
|
|
264
|
+
function printReviewed(reviewed) {
|
|
265
|
+
const sections = [
|
|
266
|
+
['defect', 'DEFECTS — a [spec] requirement the app breaks'],
|
|
267
|
+
['finding', 'FINDINGS — real app behaviour against a [baseline]/[observed] criterion (lower priority)'],
|
|
268
|
+
['pending-decision', 'PENDING DECISIONS — not requirements yet; take them to product, do not report as defects'],
|
|
269
|
+
];
|
|
270
|
+
for (const [verdict, heading] of sections) {
|
|
271
|
+
const rows = reviewed.filter((r) => r.verdict === verdict);
|
|
272
|
+
if (!rows.length)
|
|
273
|
+
continue;
|
|
274
|
+
console.log(` ${heading}:`);
|
|
275
|
+
for (const r of rows) {
|
|
276
|
+
const where = r.tests.slice(0, 3).map((t) => `${basename(t.file)}:${t.line}`).join(', ');
|
|
277
|
+
const more = r.tests.length > 3 ? ` +${r.tests.length - 3} more` : '';
|
|
278
|
+
console.log(` - ${r.claim}`);
|
|
279
|
+
console.log(` ${r.tests.length} test(s): ${where}${more}${r.evidence ? ` · app: ${r.evidence}` : ''}`);
|
|
280
|
+
if (r.reason)
|
|
281
|
+
console.log(` ${r.reason}`);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
260
285
|
main().catch((err) => {
|
|
261
286
|
console.error(err instanceof Error ? err.message : String(err));
|
|
262
287
|
process.exit(1);
|
|
@@ -80,6 +80,16 @@ export const DEEP_DOC_GUIDANCE = [
|
|
|
80
80
|
` same way (the screen accepts exactly what the server accepts). A`,
|
|
81
81
|
` failure is an inconsistency, not a spec defect. Never use [observed]`,
|
|
82
82
|
` to override or soften a [spec] criterion.`,
|
|
83
|
+
` CHECK AN [observed] RULE BEFORE YOU STATE IT. Run each pattern in your`,
|
|
84
|
+
` head against the concrete values you list — what it really accepts and`,
|
|
85
|
+
` refuses, not what its name suggests. For the server's verdict, follow`,
|
|
86
|
+
` the request through everything that runs BEFORE validation: global`,
|
|
87
|
+
` middleware that trims, normalises or converts empty strings, and the`,
|
|
88
|
+
` input mutators the handler applies. A row whose verdict is wrong`,
|
|
89
|
+
` becomes a test that fails on a correct product.`,
|
|
90
|
+
` Only what a test can drive is a criterion. Behaviour no screen or`,
|
|
91
|
+
` endpoint can reach ("not tested through the screen") is not tagged —`,
|
|
92
|
+
` leave it out, or mention it in Test setup if the generator needs to know.`,
|
|
83
93
|
``,
|
|
84
94
|
`METHOD. Work through these steps for every journey, in order:`,
|
|
85
95
|
`1. CONTROLS. Start the doc with a "## Controls" table listing EVERY`,
|
|
@@ -109,6 +109,15 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
|
|
|
109
109
|
DEEP_DOC_GUIDANCE,
|
|
110
110
|
``,
|
|
111
111
|
`Reference: ${reference}`,
|
|
112
|
+
...(loadConfig().decisions
|
|
113
|
+
? [
|
|
114
|
+
`NOT YET REQUIREMENTS: the project keeps its open product decisions and`,
|
|
115
|
+
`known issues left out on purpose at: ${loadConfig().decisions}`,
|
|
116
|
+
`Read it. Nothing listed there becomes a criterion — no [spec], no`,
|
|
117
|
+
`[observed], no "Likely defect"; name it under "Questions for product`,
|
|
118
|
+
`(not asserted)" if the journey touches it.`,
|
|
119
|
+
]
|
|
120
|
+
: []),
|
|
112
121
|
`Read it FULLY with your available tools (read the file, fetch the`,
|
|
113
122
|
`URL/page — follow sub-pages/links the reference points to). Do not`,
|
|
114
123
|
`summarise shallowly; pull out every flow, rule, state and edge case`,
|
|
@@ -20,6 +20,7 @@ import { execFileSync } from 'node:child_process';
|
|
|
20
20
|
import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
|
21
21
|
import { join } from 'node:path';
|
|
22
22
|
import { tmpdir } from 'node:os';
|
|
23
|
+
import { reasonlessPending } from '../generate/pending.js';
|
|
23
24
|
/**
|
|
24
25
|
* Third-party network SDKs common enough to be worth gating everywhere:
|
|
25
26
|
* a backend test that imports one without mocking it is talking to a real
|
|
@@ -163,6 +164,16 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
163
164
|
'too; scope the stub to one of those, or name the ones you ruled out');
|
|
164
165
|
}
|
|
165
166
|
}
|
|
167
|
+
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
168
|
+
// as a bare "fixme", indistinguishable from the tool giving up. Runs on
|
|
169
|
+
// content normalisePending() has already given every reason it could find.
|
|
170
|
+
if (ctx.framework === 'playwright') {
|
|
171
|
+
const bare = reasonlessPending(content);
|
|
172
|
+
if (bare.length) {
|
|
173
|
+
return (`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
|
|
174
|
+
`test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
166
177
|
// F10 — selector grounding: a RUNNABLE (non-skipped) UI test must not lean
|
|
167
178
|
// on a guessed selector. Unchecked, the generator invents plausible
|
|
168
179
|
// data-testids that time out at runtime — the journey is never really
|
|
@@ -221,7 +232,7 @@ function stripPendingBlocks(content) {
|
|
|
221
232
|
* skipping over quoted strings, template literals, and comments. Returns -1
|
|
222
233
|
* if no match is found.
|
|
223
234
|
*/
|
|
224
|
-
function matchParen(s, open) {
|
|
235
|
+
export function matchParen(s, open) {
|
|
225
236
|
let depth = 0;
|
|
226
237
|
for (let i = open; i < s.length; i++) {
|
|
227
238
|
const c = s[i];
|