@activate-core/test-forge 1.3.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +51 -28
  2. package/dist/cli/index.js +39 -33
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +49 -2
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +49 -22
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +60 -3
  10. package/dist/core/docs/review.js +38 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +3 -3
  13. package/dist/core/gates/coverage.js +1 -16
  14. package/dist/core/gates/validate.js +171 -7
  15. package/dist/core/generate/ai-generator.js +202 -104
  16. package/dist/core/generate/ai-usage.js +116 -5
  17. package/dist/core/generate/bind.js +91 -0
  18. package/dist/core/generate/defect-review.js +59 -9
  19. package/dist/core/generate/doc-diff.js +139 -0
  20. package/dist/core/generate/harness.js +3 -31
  21. package/dist/core/generate/journey-memory.js +325 -0
  22. package/dist/core/generate/plan-cache.js +114 -7
  23. package/dist/core/generate/review-journey.js +3 -3
  24. package/dist/core/generate/run-generate.js +169 -78
  25. package/dist/core/generate/suite-planner.js +128 -23
  26. package/dist/core/generate/test-blocks.js +84 -0
  27. package/dist/core/generate/update-journey.js +115 -0
  28. package/dist/core/paths.js +90 -13
  29. package/dist/core/project/config.js +25 -2
  30. package/dist/core/project/env-template.js +2 -1
  31. package/dist/core/run/ci.js +1 -1
  32. package/dist/core/run/clean.js +1 -1
  33. package/dist/core/run/readiness.js +119 -0
  34. package/dist/core/run/run-tests.js +23 -1
  35. package/package.json +1 -1
  36. package/reporters/_shared.cjs +2 -2
  37. package/reporters/run-data.cjs +1 -1
  38. package/reporters/ui-helpers.cjs +78 -1
  39. package/dist/core/generate/verify-journey.js +0 -215
@@ -53,4 +53,81 @@ function watchDialogs(page) {
53
53
  return messages;
54
54
  }
55
55
 
56
- module.exports = { settledClick, fillFrameInputs, watchDialogs };
56
+ /** The ARIA role each kind of element should have. */
57
+ const ROLES = {
58
+ button: 'button', link: 'link', input: 'textbox', textbox: 'textbox', field: 'textbox', textarea: 'textbox',
59
+ search: 'searchbox', dropdown: 'combobox', select: 'combobox', combobox: 'combobox', listbox: 'listbox', option: 'option',
60
+ checkbox: 'checkbox', radio: 'radio', toggle: 'switch', switch: 'switch', slider: 'slider', spinbutton: 'spinbutton',
61
+ tab: 'tab', tabpanel: 'tabpanel', menu: 'menu', menuitem: 'menuitem', dialog: 'dialog', popup: 'dialog', modal: 'dialog',
62
+ heading: 'heading', alert: 'alert', toast: 'status', status: 'status', image: 'img', img: 'img',
63
+ list: 'list', listitem: 'listitem', table: 'table', row: 'row', cell: 'cell', navigation: 'navigation', form: 'form',
64
+ };
65
+ /** Kinds a user types into or chooses with: they are also found by their label and placeholder. */
66
+ const FORM_KINDS = new Set(['input', 'textbox', 'field', 'textarea', 'search', 'dropdown', 'select', 'combobox', 'listbox',
67
+ 'checkbox', 'radio', 'toggle', 'switch', 'slider', 'spinbutton']);
68
+
69
+ const cssString = (s) => String(s).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
70
+
71
+ /** Note a finding on the running test, so the report shows it without failing the test. */
72
+ function noteFinding(description) {
73
+ try {
74
+ // eslint-disable-next-line global-require
75
+ const { test } = require('@playwright/test');
76
+ test.info().annotations.push({ type: 'finding', description });
77
+ } catch {
78
+ // outside a Playwright test: nothing to annotate
79
+ }
80
+ }
81
+
82
+ /**
83
+ * Find an element the way a user does: by what it is and what it says. Built
84
+ * before the feature, a test cannot know how the element will be made, so it
85
+ * is looked for in order — the proper role and name; for form controls their
86
+ * label and placeholder; an aria-label or title; for an image its alt text;
87
+ * finally the visible text. A match by anything but the proper role still
88
+ * works, and adds a finding to the test (e.g. a "button" built as a div,
89
+ * which keyboard and screen-reader users cannot use). Nothing found within
90
+ * `timeout` fails with what was tried.
91
+ *
92
+ * kind: button, link, input, dropdown, checkbox, radio, toggle, tab, menuitem,
93
+ * dialog, heading, alert, toast, image, list, table … (or any ARIA role)
94
+ * name: the text the doc gives it; a RegExp is allowed.
95
+ */
96
+ async function find(page, kind, name, { exact = false, timeout = 10000, within } = {}) {
97
+ const root = within || page;
98
+ const role = ROLES[kind] || kind;
99
+ const isText = typeof name === 'string';
100
+ const tries = [
101
+ ['role', () => root.getByRole(role, { name, exact })],
102
+ ...(FORM_KINDS.has(kind) ? [
103
+ ['label', () => root.getByLabel(name, { exact })],
104
+ ['placeholder', () => root.getByPlaceholder(name, { exact })],
105
+ ] : []),
106
+ ...(isText ? [['aria-label or title', () => root.locator(`[aria-label="${cssString(name)}"], [title="${cssString(name)}"]`)]] : []),
107
+ ...(kind === 'image' || kind === 'img' ? [['alt text', () => root.getByAltText(name, { exact })]] : []),
108
+ ['visible text', () => root.getByText(name, { exact })],
109
+ ];
110
+ const deadline = Date.now() + timeout;
111
+ for (;;) {
112
+ for (const [how, make] of tries) {
113
+ const all = make();
114
+ const n = await all.count().catch(() => 0);
115
+ if (!n) continue;
116
+ for (let i = 0; i < n; i++) {
117
+ const el = all.nth(i);
118
+ if (!(await el.isVisible().catch(() => false))) continue;
119
+ if (how !== 'role') {
120
+ noteFinding(`"${name}" was expected as a ${kind} (role ${role}) but was found only by its ${how}: ` +
121
+ 'it is not exposed as one, so keyboard and screen-reader users may not be able to use it.');
122
+ }
123
+ return el;
124
+ }
125
+ }
126
+ if (Date.now() > deadline) {
127
+ throw new Error(`No ${kind} "${name}" on the page: tried ${tries.map(([h]) => h).join(', ')}.`);
128
+ }
129
+ await settle(250);
130
+ }
131
+ }
132
+
133
+ module.exports = { settledClick, fillFrameInputs, watchDialogs, find };
@@ -1,215 +0,0 @@
1
- /**
2
- * testforge — check a journey's generated tests against the real app before
3
- * they are handed over.
4
- *
5
- * A test that passes every static gate can still target a selector, route or
6
- * field the app does not have; only running it can tell. Three passes:
7
- *
8
- * 1. Each file on its own. A failure, or a skip the test decided by itself,
9
- * is repaired from the runner's own output. A test the repair judges to
10
- * be failing on the APPLICATION is left as it is and marked.
11
- * 2. The whole journey in one run, as a team or CI runs it. Files that pass
12
- * alone can still break each other (one signs in again as a shared
13
- * account and ends another's session); those are repaired with that
14
- * context.
15
- * 3. An independent review of every marked failure. Only what it confirms
16
- * is reported; the rest is repaired once more or reported as unverified.
17
- *
18
- * A guard refuses any repair that deletes a test, adds a skip or loses an
19
- * assertion, so no repair can turn a caught defect into a silent one.
20
- */
21
- import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync } from 'node:fs';
22
- import { createHash } from 'node:crypto';
23
- import { basename, join, resolve } from 'node:path';
24
- import { STATE_DIR } from '../paths.js';
25
- import { DEFECT_MARK } from './ai-generator.js';
26
- import { groupClaims, normaliseClaims, reviewDefects } from './defect-review.js';
27
- import { saveVerifyResults } from './review-journey.js';
28
- import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
29
- const describe = (label, issues) => issues.length
30
- ? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
31
- : [];
32
- const key = (i) => `${i.file}:${i.line}:${i.title}`;
33
- const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
34
- const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
35
- const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
36
- const usable = (o) => o.ran && !o.infra;
37
- const PROGRESS_DIR = join(STATE_DIR, 'verify-progress');
38
- const fileHash = (f) => (existsSync(f) ? createHash('sha1').update(readFileSync(f)).digest('hex') : '');
39
- function loadProgress(journeyId) {
40
- try {
41
- return JSON.parse(readFileSync(join(PROGRESS_DIR, `${journeyId}.json`), 'utf-8'));
42
- }
43
- catch {
44
- return {};
45
- }
46
- }
47
- function saveProgress(journeyId, p) {
48
- mkdirSync(PROGRESS_DIR, { recursive: true });
49
- writeFileSync(join(PROGRESS_DIR, `${journeyId}.json`), JSON.stringify(p), 'utf-8');
50
- }
51
- export async function verifyJourney(ctx) {
52
- const { config, journey, targets } = ctx;
53
- const warnings = [];
54
- const notRun = [];
55
- // One guarded repair of one file; true when the file changed.
56
- const repairOnce = async (target, open, suspect, note = '') => {
57
- const current = readFileSync(target, 'utf-8');
58
- const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
59
- const repaired = await ctx.repair(target, current, failure);
60
- if (repaired === null)
61
- return false;
62
- // Only a quoted [spec] rule may be claimed as a defect; anything else becomes a finding mark.
63
- const { content: next, downgraded } = normaliseClaims(repaired, journey.rawMarkdown);
64
- if (downgraded)
65
- console.log(` ${basename(target)}: ${downgraded} defect claim(s) against a non-[spec] rule kept as finding(s)`);
66
- const refused = repairGuard(current, next);
67
- if (refused) {
68
- warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
69
- return false;
70
- }
71
- writeFileSync(target, next, 'utf-8');
72
- return true;
73
- };
74
- // 1. Each file on its own: what is wrong inside the file.
75
- // A file that finished this step before (same content since) is not run
76
- // again: an interrupted verify — a usage limit, a closed laptop — resumes
77
- // at the first file it had not finished.
78
- let spent = 0;
79
- const soloSkips = new Set();
80
- const lastSolo = new Map();
81
- const progress = loadProgress(journey.id);
82
- for (const target of targets) {
83
- const done = progress[resolve(target)];
84
- if (done && done.hash === fileHash(target)) {
85
- console.log(` ${basename(target)}: verified earlier, unchanged — kept`);
86
- if (!usable(done.outcome))
87
- notRun.push(`${basename(target)}: ${done.outcome.infra ?? done.outcome.reason}`);
88
- lastSolo.set(target, done.outcome);
89
- for (const k of done.outcome.skips)
90
- soloSkips.add(key(k));
91
- continue;
92
- }
93
- const run = async () => {
94
- const started = Date.now();
95
- const o = await verifyFile(config, journey.area, target);
96
- spent += Date.now() - started;
97
- return o;
98
- };
99
- let outcome = await run();
100
- for (let attempt = 0;; attempt += 1) {
101
- if (!usable(outcome)) {
102
- // The file itself does not load (a bad import, a syntax the runner
103
- // rejects): that is the file's fault, so it is repaired like a
104
- // failing test. Only an environment problem (infra) is left alone.
105
- const loadError = !outcome.ran && !outcome.infra;
106
- if (loadError && attempt < ctx.attempts) {
107
- console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
108
- const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
109
- if (await repairOnce(target, [issue], [])) {
110
- outcome = await run();
111
- continue;
112
- }
113
- }
114
- notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
115
- break;
116
- }
117
- const open = unclaimed(outcome.failures);
118
- // A skip the test decided by itself is a suspect once: its own setup may
119
- // have looked in the wrong place. A skip that only names a missing env
120
- // var or opt-in flag is the environment's to fix.
121
- const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
122
- if ((!open.length && !suspect.length) || attempt === ctx.attempts)
123
- break;
124
- console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
125
- if (!(await repairOnce(target, open, suspect)))
126
- break;
127
- outcome = await run();
128
- }
129
- lastSolo.set(target, outcome);
130
- for (const k of outcome.skips)
131
- soloSkips.add(key(k));
132
- progress[resolve(target)] = { hash: fileHash(target), outcome };
133
- saveProgress(journey.id, progress);
134
- }
135
- // 2. The whole journey in ONE run. Whatever fails only here is repaired with
136
- // that context, once.
137
- let together;
138
- if (!notRun.length && targets.length > 1) {
139
- const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
140
- const order = targets.map((t) => basename(t)).sort().join(', ');
141
- const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
142
- `each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
143
- `(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
144
- `backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
145
- `data it sets up itself.`;
146
- // The runner reports absolute paths; targets may be relative.
147
- const byAbs = new Map(targets.map((t) => [resolve(t), t]));
148
- console.log(` running all ${targets.length} files together…`);
149
- together = await verifyRun(config, journey.area, targets, budget);
150
- if (usable(together)) {
151
- const open = unclaimed(together.failures);
152
- const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
153
- const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
154
- if (files.length) {
155
- console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
156
- for (const f of files) {
157
- await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
158
- }
159
- together = await verifyRun(config, journey.area, targets, budget);
160
- }
161
- }
162
- if (!usable(together))
163
- notRun.push(`all files together: ${together.infra ?? together.reason}`);
164
- }
165
- // 3. Judge what is left on the LAST run that covered each file. Nothing is
166
- // reported as a defect on the repair's word alone.
167
- const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
168
- const failures = finals.flatMap((o) => o.failures);
169
- // Kept so `testforge review` can judge these failures again without re-running verify.
170
- saveVerifyResults(journey.id, failures, finals.flatMap((o) => o.passes));
171
- const unresolved = unclaimed(failures).map(label);
172
- let reviewed = [];
173
- const groups = groupClaims(failures, claimOf);
174
- if (groups.length) {
175
- console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
176
- const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes), undefined, join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`));
177
- reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
178
- // A rejected claim is the test's fault until shown otherwise: one guarded
179
- // repair with the reviewer's reason; whatever still fails is unverified and
180
- // is not re-claimed as a defect.
181
- for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
182
- const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
183
- `Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
184
- for (const f of new Set(r.tests.map((t) => t.file))) {
185
- const target = targets.find((t) => resolve(t) === f) ?? f;
186
- const mine = r.tests.filter((t) => t.file === f);
187
- if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
188
- const again = await verifyFile(config, journey.area, target);
189
- // Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
190
- for (const t of again.failures) {
191
- if (mine.some((m) => m.title === t.title) || !claimOf(t))
192
- unresolved.push(`${label(t)} (after review)`);
193
- }
194
- }
195
- else {
196
- for (const t of mine)
197
- unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
198
- }
199
- }
200
- }
201
- mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
202
- writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
203
- rmSync(join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`), { force: true });
204
- }
205
- // The checkpoints only serve an interrupted verify; a finished one starts fresh next time.
206
- rmSync(join(PROGRESS_DIR, `${journey.id}.json`), { force: true });
207
- if (notRun.length) {
208
- warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
209
- }
210
- if (unresolved.length) {
211
- warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
212
- `do not report them; ${unresolved.slice(0, 5).join('; ')}`);
213
- }
214
- return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
215
- }