@activate-core/test-forge 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +61 -23
- package/dist/core/docs/excerpts.js +245 -0
- package/dist/core/docs/grounding.js +49 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +202 -104
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +139 -0
- package/dist/core/generate/harness.js +3 -31
- package/dist/core/generate/journey-memory.js +325 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +169 -78
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +115 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +119 -0
- package/dist/core/run/run-tests.js +23 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
package/reporters/ui-helpers.cjs
CHANGED
|
@@ -53,4 +53,81 @@ function watchDialogs(page) {
|
|
|
53
53
|
return messages;
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
/** The ARIA role each kind of element should have. */
|
|
57
|
+
const ROLES = {
|
|
58
|
+
button: 'button', link: 'link', input: 'textbox', textbox: 'textbox', field: 'textbox', textarea: 'textbox',
|
|
59
|
+
search: 'searchbox', dropdown: 'combobox', select: 'combobox', combobox: 'combobox', listbox: 'listbox', option: 'option',
|
|
60
|
+
checkbox: 'checkbox', radio: 'radio', toggle: 'switch', switch: 'switch', slider: 'slider', spinbutton: 'spinbutton',
|
|
61
|
+
tab: 'tab', tabpanel: 'tabpanel', menu: 'menu', menuitem: 'menuitem', dialog: 'dialog', popup: 'dialog', modal: 'dialog',
|
|
62
|
+
heading: 'heading', alert: 'alert', toast: 'status', status: 'status', image: 'img', img: 'img',
|
|
63
|
+
list: 'list', listitem: 'listitem', table: 'table', row: 'row', cell: 'cell', navigation: 'navigation', form: 'form',
|
|
64
|
+
};
|
|
65
|
+
/** Kinds a user types into or chooses with: they are also found by their label and placeholder. */
|
|
66
|
+
const FORM_KINDS = new Set(['input', 'textbox', 'field', 'textarea', 'search', 'dropdown', 'select', 'combobox', 'listbox',
|
|
67
|
+
'checkbox', 'radio', 'toggle', 'switch', 'slider', 'spinbutton']);
|
|
68
|
+
|
|
69
|
+
const cssString = (s) => String(s).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
70
|
+
|
|
71
|
+
/** Note a finding on the running test, so the report shows it without failing the test. */
|
|
72
|
+
function noteFinding(description) {
|
|
73
|
+
try {
|
|
74
|
+
// eslint-disable-next-line global-require
|
|
75
|
+
const { test } = require('@playwright/test');
|
|
76
|
+
test.info().annotations.push({ type: 'finding', description });
|
|
77
|
+
} catch {
|
|
78
|
+
// outside a Playwright test: nothing to annotate
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Find an element the way a user does: by what it is and what it says. Built
|
|
84
|
+
* before the feature, a test cannot know how the element will be made, so it
|
|
85
|
+
* is looked for in order — the proper role and name; for form controls their
|
|
86
|
+
* label and placeholder; an aria-label or title; for an image its alt text;
|
|
87
|
+
* finally the visible text. A match by anything but the proper role still
|
|
88
|
+
* works, and adds a finding to the test (e.g. a "button" built as a div,
|
|
89
|
+
* which keyboard and screen-reader users cannot use). Nothing found within
|
|
90
|
+
* `timeout` fails with what was tried.
|
|
91
|
+
*
|
|
92
|
+
* kind: button, link, input, dropdown, checkbox, radio, toggle, tab, menuitem,
|
|
93
|
+
* dialog, heading, alert, toast, image, list, table … (or any ARIA role)
|
|
94
|
+
* name: the text the doc gives it; a RegExp is allowed.
|
|
95
|
+
*/
|
|
96
|
+
async function find(page, kind, name, { exact = false, timeout = 10000, within } = {}) {
|
|
97
|
+
const root = within || page;
|
|
98
|
+
const role = ROLES[kind] || kind;
|
|
99
|
+
const isText = typeof name === 'string';
|
|
100
|
+
const tries = [
|
|
101
|
+
['role', () => root.getByRole(role, { name, exact })],
|
|
102
|
+
...(FORM_KINDS.has(kind) ? [
|
|
103
|
+
['label', () => root.getByLabel(name, { exact })],
|
|
104
|
+
['placeholder', () => root.getByPlaceholder(name, { exact })],
|
|
105
|
+
] : []),
|
|
106
|
+
...(isText ? [['aria-label or title', () => root.locator(`[aria-label="${cssString(name)}"], [title="${cssString(name)}"]`)]] : []),
|
|
107
|
+
...(kind === 'image' || kind === 'img' ? [['alt text', () => root.getByAltText(name, { exact })]] : []),
|
|
108
|
+
['visible text', () => root.getByText(name, { exact })],
|
|
109
|
+
];
|
|
110
|
+
const deadline = Date.now() + timeout;
|
|
111
|
+
for (;;) {
|
|
112
|
+
for (const [how, make] of tries) {
|
|
113
|
+
const all = make();
|
|
114
|
+
const n = await all.count().catch(() => 0);
|
|
115
|
+
if (!n) continue;
|
|
116
|
+
for (let i = 0; i < n; i++) {
|
|
117
|
+
const el = all.nth(i);
|
|
118
|
+
if (!(await el.isVisible().catch(() => false))) continue;
|
|
119
|
+
if (how !== 'role') {
|
|
120
|
+
noteFinding(`"${name}" was expected as a ${kind} (role ${role}) but was found only by its ${how}: ` +
|
|
121
|
+
'it is not exposed as one, so keyboard and screen-reader users may not be able to use it.');
|
|
122
|
+
}
|
|
123
|
+
return el;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (Date.now() > deadline) {
|
|
127
|
+
throw new Error(`No ${kind} "${name}" on the page: tried ${tries.map(([h]) => h).join(', ')}.`);
|
|
128
|
+
}
|
|
129
|
+
await settle(250);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
module.exports = { settledClick, fillFrameInputs, watchDialogs, find };
|
|
@@ -1,215 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* testforge — check a journey's generated tests against the real app before
|
|
3
|
-
* they are handed over.
|
|
4
|
-
*
|
|
5
|
-
* A test that passes every static gate can still target a selector, route or
|
|
6
|
-
* field the app does not have; only running it can tell. Three passes:
|
|
7
|
-
*
|
|
8
|
-
* 1. Each file on its own. A failure, or a skip the test decided by itself,
|
|
9
|
-
* is repaired from the runner's own output. A test the repair judges to
|
|
10
|
-
* be failing on the APPLICATION is left as it is and marked.
|
|
11
|
-
* 2. The whole journey in one run, as a team or CI runs it. Files that pass
|
|
12
|
-
* alone can still break each other (one signs in again as a shared
|
|
13
|
-
* account and ends another's session); those are repaired with that
|
|
14
|
-
* context.
|
|
15
|
-
* 3. An independent review of every marked failure. Only what it confirms
|
|
16
|
-
* is reported; the rest is repaired once more or reported as unverified.
|
|
17
|
-
*
|
|
18
|
-
* A guard refuses any repair that deletes a test, adds a skip or loses an
|
|
19
|
-
* assertion, so no repair can turn a caught defect into a silent one.
|
|
20
|
-
*/
|
|
21
|
-
import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync } from 'node:fs';
|
|
22
|
-
import { createHash } from 'node:crypto';
|
|
23
|
-
import { basename, join, resolve } from 'node:path';
|
|
24
|
-
import { STATE_DIR } from '../paths.js';
|
|
25
|
-
import { DEFECT_MARK } from './ai-generator.js';
|
|
26
|
-
import { groupClaims, normaliseClaims, reviewDefects } from './defect-review.js';
|
|
27
|
-
import { saveVerifyResults } from './review-journey.js';
|
|
28
|
-
import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
|
|
29
|
-
const describe = (label, issues) => issues.length
|
|
30
|
-
? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
|
|
31
|
-
: [];
|
|
32
|
-
const key = (i) => `${i.file}:${i.line}:${i.title}`;
|
|
33
|
-
const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
|
|
34
|
-
const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
|
|
35
|
-
const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
|
|
36
|
-
const usable = (o) => o.ran && !o.infra;
|
|
37
|
-
const PROGRESS_DIR = join(STATE_DIR, 'verify-progress');
|
|
38
|
-
const fileHash = (f) => (existsSync(f) ? createHash('sha1').update(readFileSync(f)).digest('hex') : '');
|
|
39
|
-
function loadProgress(journeyId) {
|
|
40
|
-
try {
|
|
41
|
-
return JSON.parse(readFileSync(join(PROGRESS_DIR, `${journeyId}.json`), 'utf-8'));
|
|
42
|
-
}
|
|
43
|
-
catch {
|
|
44
|
-
return {};
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
function saveProgress(journeyId, p) {
|
|
48
|
-
mkdirSync(PROGRESS_DIR, { recursive: true });
|
|
49
|
-
writeFileSync(join(PROGRESS_DIR, `${journeyId}.json`), JSON.stringify(p), 'utf-8');
|
|
50
|
-
}
|
|
51
|
-
export async function verifyJourney(ctx) {
|
|
52
|
-
const { config, journey, targets } = ctx;
|
|
53
|
-
const warnings = [];
|
|
54
|
-
const notRun = [];
|
|
55
|
-
// One guarded repair of one file; true when the file changed.
|
|
56
|
-
const repairOnce = async (target, open, suspect, note = '') => {
|
|
57
|
-
const current = readFileSync(target, 'utf-8');
|
|
58
|
-
const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
|
|
59
|
-
const repaired = await ctx.repair(target, current, failure);
|
|
60
|
-
if (repaired === null)
|
|
61
|
-
return false;
|
|
62
|
-
// Only a quoted [spec] rule may be claimed as a defect; anything else becomes a finding mark.
|
|
63
|
-
const { content: next, downgraded } = normaliseClaims(repaired, journey.rawMarkdown);
|
|
64
|
-
if (downgraded)
|
|
65
|
-
console.log(` ${basename(target)}: ${downgraded} defect claim(s) against a non-[spec] rule kept as finding(s)`);
|
|
66
|
-
const refused = repairGuard(current, next);
|
|
67
|
-
if (refused) {
|
|
68
|
-
warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
|
|
69
|
-
return false;
|
|
70
|
-
}
|
|
71
|
-
writeFileSync(target, next, 'utf-8');
|
|
72
|
-
return true;
|
|
73
|
-
};
|
|
74
|
-
// 1. Each file on its own: what is wrong inside the file.
|
|
75
|
-
// A file that finished this step before (same content since) is not run
|
|
76
|
-
// again: an interrupted verify — a usage limit, a closed laptop — resumes
|
|
77
|
-
// at the first file it had not finished.
|
|
78
|
-
let spent = 0;
|
|
79
|
-
const soloSkips = new Set();
|
|
80
|
-
const lastSolo = new Map();
|
|
81
|
-
const progress = loadProgress(journey.id);
|
|
82
|
-
for (const target of targets) {
|
|
83
|
-
const done = progress[resolve(target)];
|
|
84
|
-
if (done && done.hash === fileHash(target)) {
|
|
85
|
-
console.log(` ${basename(target)}: verified earlier, unchanged — kept`);
|
|
86
|
-
if (!usable(done.outcome))
|
|
87
|
-
notRun.push(`${basename(target)}: ${done.outcome.infra ?? done.outcome.reason}`);
|
|
88
|
-
lastSolo.set(target, done.outcome);
|
|
89
|
-
for (const k of done.outcome.skips)
|
|
90
|
-
soloSkips.add(key(k));
|
|
91
|
-
continue;
|
|
92
|
-
}
|
|
93
|
-
const run = async () => {
|
|
94
|
-
const started = Date.now();
|
|
95
|
-
const o = await verifyFile(config, journey.area, target);
|
|
96
|
-
spent += Date.now() - started;
|
|
97
|
-
return o;
|
|
98
|
-
};
|
|
99
|
-
let outcome = await run();
|
|
100
|
-
for (let attempt = 0;; attempt += 1) {
|
|
101
|
-
if (!usable(outcome)) {
|
|
102
|
-
// The file itself does not load (a bad import, a syntax the runner
|
|
103
|
-
// rejects): that is the file's fault, so it is repaired like a
|
|
104
|
-
// failing test. Only an environment problem (infra) is left alone.
|
|
105
|
-
const loadError = !outcome.ran && !outcome.infra;
|
|
106
|
-
if (loadError && attempt < ctx.attempts) {
|
|
107
|
-
console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
108
|
-
const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
|
|
109
|
-
if (await repairOnce(target, [issue], [])) {
|
|
110
|
-
outcome = await run();
|
|
111
|
-
continue;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
|
|
115
|
-
break;
|
|
116
|
-
}
|
|
117
|
-
const open = unclaimed(outcome.failures);
|
|
118
|
-
// A skip the test decided by itself is a suspect once: its own setup may
|
|
119
|
-
// have looked in the wrong place. A skip that only names a missing env
|
|
120
|
-
// var or opt-in flag is the environment's to fix.
|
|
121
|
-
const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
|
|
122
|
-
if ((!open.length && !suspect.length) || attempt === ctx.attempts)
|
|
123
|
-
break;
|
|
124
|
-
console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
125
|
-
if (!(await repairOnce(target, open, suspect)))
|
|
126
|
-
break;
|
|
127
|
-
outcome = await run();
|
|
128
|
-
}
|
|
129
|
-
lastSolo.set(target, outcome);
|
|
130
|
-
for (const k of outcome.skips)
|
|
131
|
-
soloSkips.add(key(k));
|
|
132
|
-
progress[resolve(target)] = { hash: fileHash(target), outcome };
|
|
133
|
-
saveProgress(journey.id, progress);
|
|
134
|
-
}
|
|
135
|
-
// 2. The whole journey in ONE run. Whatever fails only here is repaired with
|
|
136
|
-
// that context, once.
|
|
137
|
-
let together;
|
|
138
|
-
if (!notRun.length && targets.length > 1) {
|
|
139
|
-
const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
|
|
140
|
-
const order = targets.map((t) => basename(t)).sort().join(', ');
|
|
141
|
-
const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
|
|
142
|
-
`each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
|
|
143
|
-
`(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
|
|
144
|
-
`backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
|
|
145
|
-
`data it sets up itself.`;
|
|
146
|
-
// The runner reports absolute paths; targets may be relative.
|
|
147
|
-
const byAbs = new Map(targets.map((t) => [resolve(t), t]));
|
|
148
|
-
console.log(` running all ${targets.length} files together…`);
|
|
149
|
-
together = await verifyRun(config, journey.area, targets, budget);
|
|
150
|
-
if (usable(together)) {
|
|
151
|
-
const open = unclaimed(together.failures);
|
|
152
|
-
const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
|
|
153
|
-
const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
|
|
154
|
-
if (files.length) {
|
|
155
|
-
console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
|
|
156
|
-
for (const f of files) {
|
|
157
|
-
await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
|
|
158
|
-
}
|
|
159
|
-
together = await verifyRun(config, journey.area, targets, budget);
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
if (!usable(together))
|
|
163
|
-
notRun.push(`all files together: ${together.infra ?? together.reason}`);
|
|
164
|
-
}
|
|
165
|
-
// 3. Judge what is left on the LAST run that covered each file. Nothing is
|
|
166
|
-
// reported as a defect on the repair's word alone.
|
|
167
|
-
const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
|
|
168
|
-
const failures = finals.flatMap((o) => o.failures);
|
|
169
|
-
// Kept so `testforge review` can judge these failures again without re-running verify.
|
|
170
|
-
saveVerifyResults(journey.id, failures, finals.flatMap((o) => o.passes));
|
|
171
|
-
const unresolved = unclaimed(failures).map(label);
|
|
172
|
-
let reviewed = [];
|
|
173
|
-
const groups = groupClaims(failures, claimOf);
|
|
174
|
-
if (groups.length) {
|
|
175
|
-
console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
|
|
176
|
-
const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes), undefined, join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`));
|
|
177
|
-
reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
|
|
178
|
-
// A rejected claim is the test's fault until shown otherwise: one guarded
|
|
179
|
-
// repair with the reviewer's reason; whatever still fails is unverified and
|
|
180
|
-
// is not re-claimed as a defect.
|
|
181
|
-
for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
|
|
182
|
-
const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
|
|
183
|
-
`Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
|
|
184
|
-
for (const f of new Set(r.tests.map((t) => t.file))) {
|
|
185
|
-
const target = targets.find((t) => resolve(t) === f) ?? f;
|
|
186
|
-
const mine = r.tests.filter((t) => t.file === f);
|
|
187
|
-
if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
|
|
188
|
-
const again = await verifyFile(config, journey.area, target);
|
|
189
|
-
// Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
|
|
190
|
-
for (const t of again.failures) {
|
|
191
|
-
if (mine.some((m) => m.title === t.title) || !claimOf(t))
|
|
192
|
-
unresolved.push(`${label(t)} (after review)`);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
else {
|
|
196
|
-
for (const t of mine)
|
|
197
|
-
unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
|
|
202
|
-
writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
|
|
203
|
-
rmSync(join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`), { force: true });
|
|
204
|
-
}
|
|
205
|
-
// The checkpoints only serve an interrupted verify; a finished one starts fresh next time.
|
|
206
|
-
rmSync(join(PROGRESS_DIR, `${journey.id}.json`), { force: true });
|
|
207
|
-
if (notRun.length) {
|
|
208
|
-
warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
|
|
209
|
-
}
|
|
210
|
-
if (unresolved.length) {
|
|
211
|
-
warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
|
|
212
|
-
`do not report them; ${unresolved.slice(0, 5).join('; ')}`);
|
|
213
|
-
}
|
|
214
|
-
return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
|
|
215
|
-
}
|