@activate-core/test-forge 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +98 -34
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +51 -4
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +51 -24
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +68 -5
  10. package/dist/core/docs/review.js +81 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +56 -0
  13. package/dist/core/gates/coverage.js +12 -19
  14. package/dist/core/gates/validate.js +277 -4
  15. package/dist/core/generate/ai-generator.js +243 -96
  16. package/dist/core/generate/ai-session.js +8 -0
  17. package/dist/core/generate/ai-usage.js +150 -0
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/defect-review.js +213 -20
  20. package/dist/core/generate/doc-diff.js +139 -0
  21. package/dist/core/generate/harness.js +52 -32
  22. package/dist/core/generate/journey-memory.js +325 -0
  23. package/dist/core/generate/plan-cache.js +114 -7
  24. package/dist/core/generate/required-data.js +75 -0
  25. package/dist/core/generate/review-journey.js +104 -0
  26. package/dist/core/generate/run-generate.js +193 -80
  27. package/dist/core/generate/seed-generator.js +2 -2
  28. package/dist/core/generate/suite-planner.js +138 -25
  29. package/dist/core/generate/test-blocks.js +84 -0
  30. package/dist/core/generate/update-journey.js +115 -0
  31. package/dist/core/generate/verify.js +9 -4
  32. package/dist/core/paths.js +90 -13
  33. package/dist/core/project/config.js +25 -2
  34. package/dist/core/project/env-diff.js +122 -0
  35. package/dist/core/project/env-template.js +3 -2
  36. package/dist/core/project/init.js +2 -2
  37. package/dist/core/project/map.js +2 -2
  38. package/dist/core/run/ci.js +94 -0
  39. package/dist/core/run/clean.js +111 -0
  40. package/dist/core/run/readiness.js +119 -0
  41. package/dist/core/run/report.js +2 -0
  42. package/dist/core/run/run-tests.js +23 -1
  43. package/package.json +1 -1
  44. package/reporters/_shared.cjs +24 -3
  45. package/reporters/jest-reporter.cjs +3 -1
  46. package/reporters/playwright-reporter.cjs +3 -1
  47. package/reporters/run-data.cjs +66 -0
  48. package/reporters/ui-helpers.cjs +133 -0
  49. package/dist/core/generate/verify-journey.js +0 -174
@@ -1,7 +1,7 @@
1
1
  // TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to playwright.config.js reporter array:
3
3
  // ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
4
- const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgePlaywrightReporter {
7
7
  constructor(options) {
@@ -10,6 +10,7 @@ module.exports = class TestForgePlaywrightReporter {
10
10
  this.start = Date.now();
11
11
  this.root = findRoot(process.cwd());
12
12
  this.testRoot = testRoot(this.root, this.category);
13
+ this.verdictOf = reviewVerdicts(this.root);
13
14
  }
14
15
  onBegin(config) {
15
16
  // Playwright's own testDir is the authority when it has one.
@@ -47,6 +48,7 @@ module.exports = class TestForgePlaywrightReporter {
47
48
  checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
48
49
  || testChecks(test.location && test.location.file, test.location && test.location.line),
49
50
  ...(status === 'failed' ? this.failed(test, result) : {}),
51
+ ...(status === 'failed' ? this.verdictOf(test.location && test.location.file, test.title) : {}),
50
52
  });
51
53
  }
52
54
  failed(test, result) {
@@ -0,0 +1,66 @@
1
+ // TestForge run data — one id per test run, shared by every worker, and a record
2
+ // of what the run created so it can be traced on the dashboard and cleaned later.
3
+ //
4
+ // The generated runner config calls runId() at load, so the main process sets
5
+ // TESTFORGE_RUN_ID before any worker starts and every worker inherits it.
6
+ const fs = require('fs');
7
+ const path = require('path');
8
+ const crypto = require('crypto');
9
+ const { findRoot } = require('./_shared.cjs');
10
+
11
+ const dir = (root) => path.join(root || findRoot(process.cwd()), 'testforge', 'runs', 'run-data');
12
+
13
+ /** The current run's id: <yyyymmdd>-<6 hex>, created once and kept in the env. */
14
+ function runId() {
15
+ if (!process.env.TESTFORGE_RUN_ID) {
16
+ process.env.TESTFORGE_RUN_ID = `${new Date().toISOString().slice(0, 10).replace(/-/g, '')}-${crypto.randomBytes(3).toString('hex')}`;
17
+ }
18
+ return process.env.TESTFORGE_RUN_ID;
19
+ }
20
+
21
+ /** A value no other run produces: label + the run id's last 6 characters (letters and digits only). */
22
+ function unique(label = '') {
23
+ return `${label}${runId().slice(-6)}`;
24
+ }
25
+
26
+ /**
27
+ * Record one thing the run created. entry: { kind, id, role?, cleanup?: { method, url } }
28
+ * — cleanup is the request that removes or cancels it, when the product has one.
29
+ * Appending one JSON line is safe from several workers at once.
30
+ */
31
+ function record(entry) {
32
+ const d = dir();
33
+ fs.mkdirSync(d, { recursive: true });
34
+ // The testkit that recorded it, so `testforge clean` signs in through the same one.
35
+ const kit = callerFile();
36
+ fs.appendFileSync(path.join(d, `${runId()}.jsonl`), JSON.stringify({ at: new Date().toISOString(), runId: runId(), ...(kit ? { kit } : {}), ...entry }) + '\n');
37
+ }
38
+
39
+ function callerFile() {
40
+ const lines = String(new Error().stack || '').split('\n').slice(1);
41
+ for (const l of lines) {
42
+ const m = /\(?((?:file:\/\/)?\/[^():]+):\d+:\d+\)?\s*$/.exec(l);
43
+ if (m && !m[1].endsWith('run-data.cjs')) return m[1].replace(/^file:\/\//, '');
44
+ }
45
+ return undefined;
46
+ }
47
+
48
+ /** Every record of one run. */
49
+ function readRun(id, root) {
50
+ try {
51
+ return fs.readFileSync(path.join(dir(root), `${id}.jsonl`), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
52
+ } catch {
53
+ return [];
54
+ }
55
+ }
56
+
57
+ /** Runs that recorded data, oldest first: { runId, at, count }. */
58
+ function listRuns(root) {
59
+ let files = [];
60
+ try { files = fs.readdirSync(dir(root)).filter((f) => f.endsWith('.jsonl')); } catch { return []; }
61
+ return files
62
+ .map((f) => { const id = f.slice(0, -6); const rows = readRun(id, root); return { runId: id, at: rows[0] ? rows[0].at : '', count: rows.length }; })
63
+ .sort((a, b) => (a.at < b.at ? -1 : 1));
64
+ }
65
+
66
+ module.exports = { runId, unique, record, readRun, listRuns };
@@ -0,0 +1,133 @@
1
+ // TestForge UI helpers for Playwright specs — shipped once, re-exported by every
2
+ // generated testkit, so no spec redefines (and diverges on) the same helper.
3
+ const settle = (ms) => new Promise((r) => setTimeout(r, ms));
4
+
5
+ /**
6
+ * Click a control once its layout has stopped moving. A pop-up whose content
7
+ * grows (a payment form opening extra fields, a list loading) scrolls the
8
+ * control while Playwright's own click is in flight, and the click lands on
9
+ * whatever is underneath. Scroll it into view, wait until its box is the same
10
+ * twice in a row, then click.
11
+ */
12
+ async function settledClick(locator, { settleMs = 300, tries = 10 } = {}) {
13
+ await locator.scrollIntoViewIfNeeded();
14
+ let prev = JSON.stringify(await locator.boundingBox());
15
+ for (let i = 0; i < tries; i++) {
16
+ await settle(settleMs);
17
+ const cur = JSON.stringify(await locator.boundingBox());
18
+ if (cur === prev) break;
19
+ prev = cur;
20
+ }
21
+ await locator.click();
22
+ }
23
+
24
+ /**
25
+ * Fill named inputs inside an iframe found by its name prefix (an embedded
26
+ * payment or address element). values: { inputName: value }. An input the
27
+ * frame does not show is skipped, so one call fits every variant of the form.
28
+ * Returns the names it filled.
29
+ */
30
+ async function fillFrameInputs(page, framePrefix, values) {
31
+ const frame = page.frameLocator(`iframe[name^="${framePrefix}"]`).first();
32
+ const filled = [];
33
+ for (const [name, value] of Object.entries(values)) {
34
+ const input = frame.locator(`input[name="${name}"]`);
35
+ if (await input.isVisible().catch(() => false)) {
36
+ await input.fill(String(value));
37
+ filled.push(name);
38
+ }
39
+ }
40
+ return filled;
41
+ }
42
+
43
+ /**
44
+ * Accept every browser dialog (alert / confirm) and keep its message, in order.
45
+ * Call before the action that raises it. Returns the live array of messages.
46
+ */
47
+ function watchDialogs(page) {
48
+ const messages = [];
49
+ page.on('dialog', (d) => {
50
+ messages.push(d.message());
51
+ d.accept().catch(() => {});
52
+ });
53
+ return messages;
54
+ }
55
+
56
+ /** The ARIA role each kind of element should have. */
57
+ const ROLES = {
58
+ button: 'button', link: 'link', input: 'textbox', textbox: 'textbox', field: 'textbox', textarea: 'textbox',
59
+ search: 'searchbox', dropdown: 'combobox', select: 'combobox', combobox: 'combobox', listbox: 'listbox', option: 'option',
60
+ checkbox: 'checkbox', radio: 'radio', toggle: 'switch', switch: 'switch', slider: 'slider', spinbutton: 'spinbutton',
61
+ tab: 'tab', tabpanel: 'tabpanel', menu: 'menu', menuitem: 'menuitem', dialog: 'dialog', popup: 'dialog', modal: 'dialog',
62
+ heading: 'heading', alert: 'alert', toast: 'status', status: 'status', image: 'img', img: 'img',
63
+ list: 'list', listitem: 'listitem', table: 'table', row: 'row', cell: 'cell', navigation: 'navigation', form: 'form',
64
+ };
65
+ /** Kinds a user types into or chooses with: they are also found by their label and placeholder. */
66
+ const FORM_KINDS = new Set(['input', 'textbox', 'field', 'textarea', 'search', 'dropdown', 'select', 'combobox', 'listbox',
67
+ 'checkbox', 'radio', 'toggle', 'switch', 'slider', 'spinbutton']);
68
+
69
+ const cssString = (s) => String(s).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
70
+
71
+ /** Note a finding on the running test, so the report shows it without failing the test. */
72
+ function noteFinding(description) {
73
+ try {
74
+ // eslint-disable-next-line global-require
75
+ const { test } = require('@playwright/test');
76
+ test.info().annotations.push({ type: 'finding', description });
77
+ } catch {
78
+ // outside a Playwright test: nothing to annotate
79
+ }
80
+ }
81
+
82
+ /**
83
+ * Find an element the way a user does: by what it is and what it says. Built
84
+ * before the feature, a test cannot know how the element will be made, so it
85
+ * is looked for in order — the proper role and name; for form controls their
86
+ * label and placeholder; an aria-label or title; for an image its alt text;
87
+ * finally the visible text. A match by anything but the proper role still
88
+ * works, and adds a finding to the test (e.g. a "button" built as a div,
89
+ * which keyboard and screen-reader users cannot use). Nothing found within
90
+ * `timeout` fails with what was tried.
91
+ *
92
+ * kind: button, link, input, dropdown, checkbox, radio, toggle, tab, menuitem,
93
+ * dialog, heading, alert, toast, image, list, table … (or any ARIA role)
94
+ * name: the text the doc gives it; a RegExp is allowed.
95
+ */
96
+ async function find(page, kind, name, { exact = false, timeout = 10000, within } = {}) {
97
+ const root = within || page;
98
+ const role = ROLES[kind] || kind;
99
+ const isText = typeof name === 'string';
100
+ const tries = [
101
+ ['role', () => root.getByRole(role, { name, exact })],
102
+ ...(FORM_KINDS.has(kind) ? [
103
+ ['label', () => root.getByLabel(name, { exact })],
104
+ ['placeholder', () => root.getByPlaceholder(name, { exact })],
105
+ ] : []),
106
+ ...(isText ? [['aria-label or title', () => root.locator(`[aria-label="${cssString(name)}"], [title="${cssString(name)}"]`)]] : []),
107
+ ...(kind === 'image' || kind === 'img' ? [['alt text', () => root.getByAltText(name, { exact })]] : []),
108
+ ['visible text', () => root.getByText(name, { exact })],
109
+ ];
110
+ const deadline = Date.now() + timeout;
111
+ for (;;) {
112
+ for (const [how, make] of tries) {
113
+ const all = make();
114
+ const n = await all.count().catch(() => 0);
115
+ if (!n) continue;
116
+ for (let i = 0; i < n; i++) {
117
+ const el = all.nth(i);
118
+ if (!(await el.isVisible().catch(() => false))) continue;
119
+ if (how !== 'role') {
120
+ noteFinding(`"${name}" was expected as a ${kind} (role ${role}) but was found only by its ${how}: ` +
121
+ 'it is not exposed as one, so keyboard and screen-reader users may not be able to use it.');
122
+ }
123
+ return el;
124
+ }
125
+ }
126
+ if (Date.now() > deadline) {
127
+ throw new Error(`No ${kind} "${name}" on the page: tried ${tries.map(([h]) => h).join(', ')}.`);
128
+ }
129
+ await settle(250);
130
+ }
131
+ }
132
+
133
+ module.exports = { settledClick, fillFrameInputs, watchDialogs, find };
@@ -1,174 +0,0 @@
1
- /**
2
- * testforge — check a journey's generated tests against the real app before
3
- * they are handed over.
4
- *
5
- * A test that passes every static gate can still target a selector, route or
6
- * field the app does not have; only running it can tell. Three passes:
7
- *
8
- * 1. Each file on its own. A failure, or a skip the test decided by itself,
9
- * is repaired from the runner's own output. A test the repair judges to
10
- * be failing on the APPLICATION is left as it is and marked.
11
- * 2. The whole journey in one run, as a team or CI runs it. Files that pass
12
- * alone can still break each other (one signs in again as a shared
13
- * account and ends another's session); those are repaired with that
14
- * context.
15
- * 3. An independent review of every marked failure. Only what it confirms
16
- * is reported; the rest is repaired once more or reported as unverified.
17
- *
18
- * A guard refuses any repair that deletes a test, adds a skip or loses an
19
- * assertion, so no repair can turn a caught defect into a silent one.
20
- */
21
- import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
22
- import { basename, join, resolve } from 'node:path';
23
- import { STATE_DIR } from '../paths.js';
24
- import { DEFECT_MARK } from './ai-generator.js';
25
- import { groupClaims, reviewDefects } from './defect-review.js';
26
- import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
27
- const describe = (label, issues) => issues.length
28
- ? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
29
- : [];
30
- const key = (i) => `${i.file}:${i.line}:${i.title}`;
31
- const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
32
- const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
33
- const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
34
- const usable = (o) => o.ran && !o.infra;
35
- export async function verifyJourney(ctx) {
36
- const { config, journey, targets } = ctx;
37
- const warnings = [];
38
- const notRun = [];
39
- // One guarded repair of one file; true when the file changed.
40
- const repairOnce = async (target, open, suspect, note = '') => {
41
- const current = readFileSync(target, 'utf-8');
42
- const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
43
- const next = await ctx.repair(target, current, failure);
44
- if (next === null)
45
- return false;
46
- const refused = repairGuard(current, next);
47
- if (refused) {
48
- warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
49
- return false;
50
- }
51
- writeFileSync(target, next, 'utf-8');
52
- return true;
53
- };
54
- // 1. Each file on its own: what is wrong inside the file.
55
- let spent = 0;
56
- const soloSkips = new Set();
57
- const lastSolo = new Map();
58
- for (const target of targets) {
59
- const run = async () => {
60
- const started = Date.now();
61
- const o = await verifyFile(config, journey.area, target);
62
- spent += Date.now() - started;
63
- return o;
64
- };
65
- let outcome = await run();
66
- for (let attempt = 0;; attempt += 1) {
67
- if (!usable(outcome)) {
68
- // The file itself does not load (a bad import, a syntax the runner
69
- // rejects): that is the file's fault, so it is repaired like a
70
- // failing test. Only an environment problem (infra) is left alone.
71
- const loadError = !outcome.ran && !outcome.infra;
72
- if (loadError && attempt < ctx.attempts) {
73
- console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
74
- const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
75
- if (await repairOnce(target, [issue], [])) {
76
- outcome = await run();
77
- continue;
78
- }
79
- }
80
- notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
81
- break;
82
- }
83
- const open = unclaimed(outcome.failures);
84
- // A skip the test decided by itself is a suspect once: its own setup may
85
- // have looked in the wrong place. A skip that only names a missing env
86
- // var or opt-in flag is the environment's to fix.
87
- const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
88
- if ((!open.length && !suspect.length) || attempt === ctx.attempts)
89
- break;
90
- console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
91
- if (!(await repairOnce(target, open, suspect)))
92
- break;
93
- outcome = await run();
94
- }
95
- lastSolo.set(target, outcome);
96
- for (const k of outcome.skips)
97
- soloSkips.add(key(k));
98
- }
99
- // 2. The whole journey in ONE run. Whatever fails only here is repaired with
100
- // that context, once.
101
- let together;
102
- if (!notRun.length && targets.length > 1) {
103
- const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
104
- const order = targets.map((t) => basename(t)).sort().join(', ');
105
- const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
106
- `each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
107
- `(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
108
- `backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
109
- `data it sets up itself.`;
110
- // The runner reports absolute paths; targets may be relative.
111
- const byAbs = new Map(targets.map((t) => [resolve(t), t]));
112
- console.log(` running all ${targets.length} files together…`);
113
- together = await verifyRun(config, journey.area, targets, budget);
114
- if (usable(together)) {
115
- const open = unclaimed(together.failures);
116
- const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
117
- const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
118
- if (files.length) {
119
- console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
120
- for (const f of files) {
121
- await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
122
- }
123
- together = await verifyRun(config, journey.area, targets, budget);
124
- }
125
- }
126
- if (!usable(together))
127
- notRun.push(`all files together: ${together.infra ?? together.reason}`);
128
- }
129
- // 3. Judge what is left on the LAST run that covered each file. Nothing is
130
- // reported as a defect on the repair's word alone.
131
- const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
132
- const failures = finals.flatMap((o) => o.failures);
133
- const unresolved = unclaimed(failures).map(label);
134
- let reviewed = [];
135
- const groups = groupClaims(failures, claimOf);
136
- if (groups.length) {
137
- console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
138
- const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes));
139
- reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
140
- // A rejected claim is the test's fault until shown otherwise: one guarded
141
- // repair with the reviewer's reason; whatever still fails is unverified and
142
- // is not re-claimed as a defect.
143
- for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
144
- const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
145
- `Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
146
- for (const f of new Set(r.tests.map((t) => t.file))) {
147
- const target = targets.find((t) => resolve(t) === f) ?? f;
148
- const mine = r.tests.filter((t) => t.file === f);
149
- if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
150
- const again = await verifyFile(config, journey.area, target);
151
- // Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
152
- for (const t of again.failures) {
153
- if (mine.some((m) => m.title === t.title) || !claimOf(t))
154
- unresolved.push(`${label(t)} (after review)`);
155
- }
156
- }
157
- else {
158
- for (const t of mine)
159
- unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
160
- }
161
- }
162
- }
163
- mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
164
- writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
165
- }
166
- if (notRun.length) {
167
- warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
168
- }
169
- if (unresolved.length) {
170
- warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
171
- `do not report them; ${unresolved.slice(0, 5).join('; ')}`);
172
- }
173
- return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
174
- }