@activate-core/test-forge 1.3.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +70 -29
- package/dist/core/docs/excerpts.js +294 -0
- package/dist/core/docs/grounding.js +55 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/test-quality.js +295 -0
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +267 -117
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/complete-file.js +77 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +140 -0
- package/dist/core/generate/harness.js +18 -34
- package/dist/core/generate/journey-memory.js +394 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +240 -82
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +118 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +146 -0
- package/dist/core/run/run-tests.js +26 -2
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — has the feature a test file checks been built yet?
|
|
3
|
+
*
|
|
4
|
+
* Tests are written from the doc, often before the code exists. Before a run,
|
|
5
|
+
* each file's names (the buttons, links, fields, headings, images it looks
|
|
6
|
+
* for) and the pages it opens are looked for in the app's source. A file whose
|
|
7
|
+
* names are all missing tests a feature not built yet, and is held back as
|
|
8
|
+
* pending rather than run to fail; a name missing with a close one in the code
|
|
9
|
+
* is a naming difference to settle. Once the code has them, the file runs. No AI.
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import { usedLabels } from '../gates/validate.js';
|
|
14
|
+
/** A name as the code would write it: no required-field star, no closing punctuation. */
|
|
15
|
+
const bare = (n) => n.replace(/\s*\*\s*$/, '').replace(/[\s.:!?…]+$/, '').replace(/\s+/g, ' ').trim();
|
|
16
|
+
/**
|
|
17
|
+
* What a test file needs the code to have: the names it locates elements by to
|
|
18
|
+
* act on them (role names, labels, placeholders, kit.find) and the pages it
|
|
19
|
+
* opens. Texts it only expects to see are left out: one the code lacks is a
|
|
20
|
+
* finding for the test to report, not a sign the feature is missing.
|
|
21
|
+
*/
|
|
22
|
+
export function namesInTest(content) {
|
|
23
|
+
const names = new Set(usedLabels(content).map(bare).filter((n) => n.length >= 3));
|
|
24
|
+
const pages = new Set();
|
|
25
|
+
for (const m of content.matchAll(/\bgoto\(\s*[`'"](?:\$\{[^}]+\})?(\/[\w\-./]*)/g)) {
|
|
26
|
+
const p = m[1].replace(/\/+$/, '') || '/';
|
|
27
|
+
if (p !== '/')
|
|
28
|
+
pages.add(p);
|
|
29
|
+
}
|
|
30
|
+
return { names: [...names], pages: [...pages] };
|
|
31
|
+
}
|
|
32
|
+
/** The words that carry meaning: three letters or more. */
|
|
33
|
+
const words = (t) => new Set(t.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 3));
|
|
34
|
+
function closeness(a, b) {
|
|
35
|
+
const x = words(a);
|
|
36
|
+
const y = words(b);
|
|
37
|
+
let both = 0;
|
|
38
|
+
for (const w of x)
|
|
39
|
+
if (y.has(w))
|
|
40
|
+
both += 1;
|
|
41
|
+
return x.size + y.size ? both / (x.size + y.size - both) : 0;
|
|
42
|
+
}
|
|
43
|
+
/** Texts the code can show: JSX text and short string literals, for "did you mean". */
|
|
44
|
+
export function shownTexts(source) {
|
|
45
|
+
const out = new Set();
|
|
46
|
+
for (const m of source.matchAll(/>\s*([^<>{}\n]{2,60}?)\s*</g))
|
|
47
|
+
out.add(m[1].trim());
|
|
48
|
+
for (const m of source.matchAll(/(['"`])([^'"`\n]{2,60})\1/g))
|
|
49
|
+
if (/[a-z]/i.test(m[2]) && /\s|^[A-Z]/.test(m[2]))
|
|
50
|
+
out.add(m[2].trim());
|
|
51
|
+
return [...out];
|
|
52
|
+
}
|
|
53
|
+
/** The nearest text the code has to a name, when one is close enough to be the same thing renamed. */
|
|
54
|
+
export function nearestText(name, texts) {
|
|
55
|
+
let best = 0.3;
|
|
56
|
+
let nearest;
|
|
57
|
+
for (const t of texts) {
|
|
58
|
+
const c = closeness(name, t);
|
|
59
|
+
if (c > best) {
|
|
60
|
+
best = c;
|
|
61
|
+
nearest = t;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return nearest;
|
|
65
|
+
}
|
|
66
|
+
/** Does the app have this page: a route string in the source, or a Next.js / file-based route folder? */
|
|
67
|
+
function hasPage(path, haystack, appRoots) {
|
|
68
|
+
if (haystack.includes(`'${path}'`) || haystack.includes(`"${path}"`) || haystack.includes(`\`${path}`) || haystack.includes(`${path}/`))
|
|
69
|
+
return true;
|
|
70
|
+
const segs = path.replace(/^\//, '');
|
|
71
|
+
return appRoots.some((r) => ['app', 'pages', 'src/app', 'src/pages'].some((d) => existsSync(join(r, d, segs))));
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Classify each test file: built (everything found), partial (some found),
|
|
75
|
+
* not-built (nothing found), renamed (a missing name has a close one in the code).
|
|
76
|
+
*/
|
|
77
|
+
export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
|
|
78
|
+
let texts;
|
|
79
|
+
return files.map((file) => {
|
|
80
|
+
const content = readFileSync(file, 'utf-8');
|
|
81
|
+
// The doc says the feature is not live: held back whatever the code has.
|
|
82
|
+
const pending = PENDING_STAMP.exec(content)?.[1];
|
|
83
|
+
if (pending)
|
|
84
|
+
return { file, state: 'not-built', missing: [pending.trim()], renamed: [] };
|
|
85
|
+
const { names, pages } = namesInTest(content);
|
|
86
|
+
const missing = [];
|
|
87
|
+
const renamed = [];
|
|
88
|
+
for (const n of names) {
|
|
89
|
+
if (haystack.includes(n.toLowerCase().replace(/\s+/g, ' ')))
|
|
90
|
+
continue;
|
|
91
|
+
texts ??= sourceTexts();
|
|
92
|
+
const near = nearestText(n, texts);
|
|
93
|
+
if (near)
|
|
94
|
+
renamed.push({ name: n, nearest: near });
|
|
95
|
+
else
|
|
96
|
+
missing.push(n);
|
|
97
|
+
}
|
|
98
|
+
for (const p of pages)
|
|
99
|
+
if (!hasPage(p, haystack, appRoots))
|
|
100
|
+
missing.push(p);
|
|
101
|
+
const total = names.length + pages.length;
|
|
102
|
+
const found = total - missing.length - renamed.length;
|
|
103
|
+
// Not built: the code has none of it. A different name or a partial match still runs, and fails if it must.
|
|
104
|
+
const state = !total || found === total ? 'built' : found === 0 && !renamed.length ? 'not-built' : renamed.length ? 'renamed' : 'partial';
|
|
105
|
+
return { file, state, missing, renamed };
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
/** The line that holds back a file whose feature is not live yet. */
|
|
109
|
+
export const PENDING_STAMP = /^\/\/\s*testforge-pending:\s*(.+)$/m;
|
|
110
|
+
/**
|
|
111
|
+
* The status a journey doc gives its feature: "Status: not built", or
|
|
112
|
+
* "Status: behind flag <name>" for code that ships switched off. Undefined
|
|
113
|
+
* when the feature is live.
|
|
114
|
+
*/
|
|
115
|
+
export function pendingReason(markdown) {
|
|
116
|
+
const m = /^status:\s*(not built(?: yet)?|behind (?:a )?flag\b[^\n]*)$/im.exec(markdown);
|
|
117
|
+
if (!m)
|
|
118
|
+
return undefined;
|
|
119
|
+
return /^not built/i.test(m[1]) ? 'the feature is not built yet' : `the feature is ${m[1].trim().replace(/^behind (?:a )?flag\s*/i, 'behind the flag ').trim()} — run it once that is switched on`;
|
|
120
|
+
}
|
|
121
|
+
/** The file with the pending stamp set to `reason`, or removed when there is none. */
|
|
122
|
+
export function withPendingStamp(content, reason) {
|
|
123
|
+
const without = content.replace(/^\/\/\s*testforge-pending:.*\n/m, '');
|
|
124
|
+
if (!reason)
|
|
125
|
+
return without;
|
|
126
|
+
const marker = /^\/\/ @generated by testforge.*\n/m.exec(without);
|
|
127
|
+
const at = marker ? marker.index + marker[0].length : 0;
|
|
128
|
+
return `${without.slice(0, at)}// testforge-pending: ${reason}\n${without.slice(at)}`;
|
|
129
|
+
}
|
|
130
|
+
/** The test files under a directory. */
|
|
131
|
+
export function testFilesIn(dir) {
|
|
132
|
+
const out = [];
|
|
133
|
+
const walk = (d) => {
|
|
134
|
+
for (const e of existsSync(d) ? readdirSync(d, { withFileTypes: true }) : []) {
|
|
135
|
+
const p = join(d, e.name);
|
|
136
|
+
if (e.isDirectory()) {
|
|
137
|
+
if (e.name !== 'helpers' && e.name !== 'node_modules')
|
|
138
|
+
walk(p);
|
|
139
|
+
}
|
|
140
|
+
else if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(e.name))
|
|
141
|
+
out.push(p);
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
walk(dir);
|
|
145
|
+
return out;
|
|
146
|
+
}
|
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
* It runs from the area's nearest package root (where package.json /
|
|
13
13
|
* playwright.config live), with config.envFile merged into the env.
|
|
14
14
|
*/
|
|
15
|
+
import { checkReadiness, testFilesIn } from './readiness.js';
|
|
16
|
+
import { sourceText } from '../docs/excerpts.js';
|
|
15
17
|
import { spawnSync } from 'node:child_process';
|
|
16
18
|
import { existsSync } from 'node:fs';
|
|
17
19
|
import { join, dirname, resolve, relative } from 'node:path';
|
|
@@ -128,7 +130,27 @@ export function runTests(opts = {}) {
|
|
|
128
130
|
const framework = resolveFramework(config, area, area);
|
|
129
131
|
const root = packageRoot(testDir);
|
|
130
132
|
const rel = relative(root, testDir) || '.';
|
|
131
|
-
|
|
133
|
+
// Tests written before their feature: hold back the files whose names and
|
|
134
|
+
// pages the code does not have yet, instead of running them to fail.
|
|
135
|
+
let targets;
|
|
136
|
+
if (framework === 'playwright' && !opts.all) {
|
|
137
|
+
const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
|
|
138
|
+
const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
|
|
139
|
+
const roots = ['src', ...(config.relatedDirs ?? [])];
|
|
140
|
+
const states = checkReadiness(files, sourceText(roots), () => [], [root]);
|
|
141
|
+
const held = states.filter((s) => s.state === 'not-built');
|
|
142
|
+
if (held.length) {
|
|
143
|
+
console.log(`\n${held.length} file(s) pending — their feature is not live yet (\`testforge bind\` shows why; --all runs them anyway):`);
|
|
144
|
+
for (const h of held.slice(0, 15))
|
|
145
|
+
console.log(` not built ${relative(root, h.file)}: ${h.missing.slice(0, 3).map((m) => `"${m}"`).join(', ')}`);
|
|
146
|
+
targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
|
|
147
|
+
if (!targets.length) {
|
|
148
|
+
console.log(`[skip] ${area}: nothing to run yet`);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
let cmd = runnerCommand(config, area, framework, root, rel, opts.journey, targets);
|
|
132
154
|
if (opts.passThrough?.length)
|
|
133
155
|
cmd += ` ${opts.passThrough.join(' ')}`;
|
|
134
156
|
ranAnything = true;
|
|
@@ -142,6 +164,8 @@ export function runTests(opts = {}) {
|
|
|
142
164
|
console.log('\nNothing to run — generate tests first (`testforge generate`).');
|
|
143
165
|
return 1;
|
|
144
166
|
}
|
|
145
|
-
|
|
167
|
+
// The runner's own summary above says how many ran and how many skipped; a run where
|
|
168
|
+
// everything skipped has no failure, but it is not a pass.
|
|
169
|
+
console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
|
|
146
170
|
return anyFail ? 1 : 0;
|
|
147
171
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/_shared.cjs
CHANGED
|
@@ -374,12 +374,12 @@ function testRoot(root, category) {
|
|
|
374
374
|
return d ? path.resolve(root, d) : undefined;
|
|
375
375
|
}
|
|
376
376
|
|
|
377
|
-
// The defect review's verdict for each failed test (
|
|
377
|
+
// The defect review's verdict for each failed test (testforge/runs/defect-review/*.json),
|
|
378
378
|
// so the dashboard can say whether a failure is a product defect, a test fault or the
|
|
379
379
|
// environment. Keyed by absolute file + test title; an absent or broken file means none.
|
|
380
380
|
function reviewVerdicts(root) {
|
|
381
381
|
const map = new Map();
|
|
382
|
-
const dir = path.join(root || process.cwd(), '
|
|
382
|
+
const dir = path.join(root || process.cwd(), 'testforge', 'runs', 'defect-review');
|
|
383
383
|
let files = [];
|
|
384
384
|
try { files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); } catch { /* no reviews yet */ }
|
|
385
385
|
for (const f of files) {
|
package/reporters/run-data.cjs
CHANGED
|
@@ -8,7 +8,7 @@ const path = require('path');
|
|
|
8
8
|
const crypto = require('crypto');
|
|
9
9
|
const { findRoot } = require('./_shared.cjs');
|
|
10
10
|
|
|
11
|
-
const dir = (root) => path.join(root || findRoot(process.cwd()), '
|
|
11
|
+
const dir = (root) => path.join(root || findRoot(process.cwd()), 'testforge', 'runs', 'run-data');
|
|
12
12
|
|
|
13
13
|
/** The current run's id: <yyyymmdd>-<6 hex>, created once and kept in the env. */
|
|
14
14
|
function runId() {
|
package/reporters/ui-helpers.cjs
CHANGED
|
@@ -53,4 +53,81 @@ function watchDialogs(page) {
|
|
|
53
53
|
return messages;
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
|
|
56
|
+
/** The ARIA role each kind of element should have. */
|
|
57
|
+
const ROLES = {
|
|
58
|
+
button: 'button', link: 'link', input: 'textbox', textbox: 'textbox', field: 'textbox', textarea: 'textbox',
|
|
59
|
+
search: 'searchbox', dropdown: 'combobox', select: 'combobox', combobox: 'combobox', listbox: 'listbox', option: 'option',
|
|
60
|
+
checkbox: 'checkbox', radio: 'radio', toggle: 'switch', switch: 'switch', slider: 'slider', spinbutton: 'spinbutton',
|
|
61
|
+
tab: 'tab', tabpanel: 'tabpanel', menu: 'menu', menuitem: 'menuitem', dialog: 'dialog', popup: 'dialog', modal: 'dialog',
|
|
62
|
+
heading: 'heading', alert: 'alert', toast: 'status', status: 'status', image: 'img', img: 'img',
|
|
63
|
+
list: 'list', listitem: 'listitem', table: 'table', row: 'row', cell: 'cell', navigation: 'navigation', form: 'form',
|
|
64
|
+
};
|
|
65
|
+
/** Kinds a user types into or chooses with: they are also found by their label and placeholder. */
|
|
66
|
+
const FORM_KINDS = new Set(['input', 'textbox', 'field', 'textarea', 'search', 'dropdown', 'select', 'combobox', 'listbox',
|
|
67
|
+
'checkbox', 'radio', 'toggle', 'switch', 'slider', 'spinbutton']);
|
|
68
|
+
|
|
69
|
+
const cssString = (s) => String(s).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
70
|
+
|
|
71
|
+
/** Note a finding on the running test, so the report shows it without failing the test. */
|
|
72
|
+
function noteFinding(description) {
|
|
73
|
+
try {
|
|
74
|
+
// eslint-disable-next-line global-require
|
|
75
|
+
const { test } = require('@playwright/test');
|
|
76
|
+
test.info().annotations.push({ type: 'finding', description });
|
|
77
|
+
} catch {
|
|
78
|
+
// outside a Playwright test: nothing to annotate
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Find an element the way a user does: by what it is and what it says. Built
|
|
84
|
+
* before the feature, a test cannot know how the element will be made, so it
|
|
85
|
+
* is looked for in order — the proper role and name; for form controls their
|
|
86
|
+
* label and placeholder; an aria-label or title; for an image its alt text;
|
|
87
|
+
* finally the visible text. A match by anything but the proper role still
|
|
88
|
+
* works, and adds a finding to the test (e.g. a "button" built as a div,
|
|
89
|
+
* which keyboard and screen-reader users cannot use). Nothing found within
|
|
90
|
+
* `timeout` fails with what was tried.
|
|
91
|
+
*
|
|
92
|
+
* kind: button, link, input, dropdown, checkbox, radio, toggle, tab, menuitem,
|
|
93
|
+
* dialog, heading, alert, toast, image, list, table … (or any ARIA role)
|
|
94
|
+
* name: the text the doc gives it; a RegExp is allowed.
|
|
95
|
+
*/
|
|
96
|
+
async function find(page, kind, name, { exact = false, timeout = 10000, within } = {}) {
|
|
97
|
+
const root = within || page;
|
|
98
|
+
const role = ROLES[kind] || kind;
|
|
99
|
+
const isText = typeof name === 'string';
|
|
100
|
+
const tries = [
|
|
101
|
+
['role', () => root.getByRole(role, { name, exact })],
|
|
102
|
+
...(FORM_KINDS.has(kind) ? [
|
|
103
|
+
['label', () => root.getByLabel(name, { exact })],
|
|
104
|
+
['placeholder', () => root.getByPlaceholder(name, { exact })],
|
|
105
|
+
] : []),
|
|
106
|
+
...(isText ? [['aria-label or title', () => root.locator(`[aria-label="${cssString(name)}"], [title="${cssString(name)}"]`)]] : []),
|
|
107
|
+
...(kind === 'image' || kind === 'img' ? [['alt text', () => root.getByAltText(name, { exact })]] : []),
|
|
108
|
+
['visible text', () => root.getByText(name, { exact })],
|
|
109
|
+
];
|
|
110
|
+
const deadline = Date.now() + timeout;
|
|
111
|
+
for (;;) {
|
|
112
|
+
for (const [how, make] of tries) {
|
|
113
|
+
const all = make();
|
|
114
|
+
const n = await all.count().catch(() => 0);
|
|
115
|
+
if (!n) continue;
|
|
116
|
+
for (let i = 0; i < n; i++) {
|
|
117
|
+
const el = all.nth(i);
|
|
118
|
+
if (!(await el.isVisible().catch(() => false))) continue;
|
|
119
|
+
if (how !== 'role') {
|
|
120
|
+
noteFinding(`"${name}" was expected as a ${kind} (role ${role}) but was found only by its ${how}: ` +
|
|
121
|
+
'it is not exposed as one, so keyboard and screen-reader users may not be able to use it.');
|
|
122
|
+
}
|
|
123
|
+
return el;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (Date.now() > deadline) {
|
|
127
|
+
throw new Error(`No ${kind} "${name}" on the page: tried ${tries.map(([h]) => h).join(', ')}.`);
|
|
128
|
+
}
|
|
129
|
+
await settle(250);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
module.exports = { settledClick, fillFrameInputs, watchDialogs, find };
|
|
@@ -1,215 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* testforge — check a journey's generated tests against the real app before
|
|
3
|
-
* they are handed over.
|
|
4
|
-
*
|
|
5
|
-
* A test that passes every static gate can still target a selector, route or
|
|
6
|
-
* field the app does not have; only running it can tell. Three passes:
|
|
7
|
-
*
|
|
8
|
-
* 1. Each file on its own. A failure, or a skip the test decided by itself,
|
|
9
|
-
* is repaired from the runner's own output. A test the repair judges to
|
|
10
|
-
* be failing on the APPLICATION is left as it is and marked.
|
|
11
|
-
* 2. The whole journey in one run, as a team or CI runs it. Files that pass
|
|
12
|
-
* alone can still break each other (one signs in again as a shared
|
|
13
|
-
* account and ends another's session); those are repaired with that
|
|
14
|
-
* context.
|
|
15
|
-
* 3. An independent review of every marked failure. Only what it confirms
|
|
16
|
-
* is reported; the rest is repaired once more or reported as unverified.
|
|
17
|
-
*
|
|
18
|
-
* A guard refuses any repair that deletes a test, adds a skip or loses an
|
|
19
|
-
* assertion, so no repair can turn a caught defect into a silent one.
|
|
20
|
-
*/
|
|
21
|
-
import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync } from 'node:fs';
|
|
22
|
-
import { createHash } from 'node:crypto';
|
|
23
|
-
import { basename, join, resolve } from 'node:path';
|
|
24
|
-
import { STATE_DIR } from '../paths.js';
|
|
25
|
-
import { DEFECT_MARK } from './ai-generator.js';
|
|
26
|
-
import { groupClaims, normaliseClaims, reviewDefects } from './defect-review.js';
|
|
27
|
-
import { saveVerifyResults } from './review-journey.js';
|
|
28
|
-
import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
|
|
29
|
-
const describe = (label, issues) => issues.length
|
|
30
|
-
? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
|
|
31
|
-
: [];
|
|
32
|
-
const key = (i) => `${i.file}:${i.line}:${i.title}`;
|
|
33
|
-
const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
|
|
34
|
-
const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
|
|
35
|
-
const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
|
|
36
|
-
const usable = (o) => o.ran && !o.infra;
|
|
37
|
-
const PROGRESS_DIR = join(STATE_DIR, 'verify-progress');
|
|
38
|
-
const fileHash = (f) => (existsSync(f) ? createHash('sha1').update(readFileSync(f)).digest('hex') : '');
|
|
39
|
-
function loadProgress(journeyId) {
|
|
40
|
-
try {
|
|
41
|
-
return JSON.parse(readFileSync(join(PROGRESS_DIR, `${journeyId}.json`), 'utf-8'));
|
|
42
|
-
}
|
|
43
|
-
catch {
|
|
44
|
-
return {};
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
function saveProgress(journeyId, p) {
|
|
48
|
-
mkdirSync(PROGRESS_DIR, { recursive: true });
|
|
49
|
-
writeFileSync(join(PROGRESS_DIR, `${journeyId}.json`), JSON.stringify(p), 'utf-8');
|
|
50
|
-
}
|
|
51
|
-
export async function verifyJourney(ctx) {
|
|
52
|
-
const { config, journey, targets } = ctx;
|
|
53
|
-
const warnings = [];
|
|
54
|
-
const notRun = [];
|
|
55
|
-
// One guarded repair of one file; true when the file changed.
|
|
56
|
-
const repairOnce = async (target, open, suspect, note = '') => {
|
|
57
|
-
const current = readFileSync(target, 'utf-8');
|
|
58
|
-
const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
|
|
59
|
-
const repaired = await ctx.repair(target, current, failure);
|
|
60
|
-
if (repaired === null)
|
|
61
|
-
return false;
|
|
62
|
-
// Only a quoted [spec] rule may be claimed as a defect; anything else becomes a finding mark.
|
|
63
|
-
const { content: next, downgraded } = normaliseClaims(repaired, journey.rawMarkdown);
|
|
64
|
-
if (downgraded)
|
|
65
|
-
console.log(` ${basename(target)}: ${downgraded} defect claim(s) against a non-[spec] rule kept as finding(s)`);
|
|
66
|
-
const refused = repairGuard(current, next);
|
|
67
|
-
if (refused) {
|
|
68
|
-
warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
|
|
69
|
-
return false;
|
|
70
|
-
}
|
|
71
|
-
writeFileSync(target, next, 'utf-8');
|
|
72
|
-
return true;
|
|
73
|
-
};
|
|
74
|
-
// 1. Each file on its own: what is wrong inside the file.
|
|
75
|
-
// A file that finished this step before (same content since) is not run
|
|
76
|
-
// again: an interrupted verify — a usage limit, a closed laptop — resumes
|
|
77
|
-
// at the first file it had not finished.
|
|
78
|
-
let spent = 0;
|
|
79
|
-
const soloSkips = new Set();
|
|
80
|
-
const lastSolo = new Map();
|
|
81
|
-
const progress = loadProgress(journey.id);
|
|
82
|
-
for (const target of targets) {
|
|
83
|
-
const done = progress[resolve(target)];
|
|
84
|
-
if (done && done.hash === fileHash(target)) {
|
|
85
|
-
console.log(` ${basename(target)}: verified earlier, unchanged — kept`);
|
|
86
|
-
if (!usable(done.outcome))
|
|
87
|
-
notRun.push(`${basename(target)}: ${done.outcome.infra ?? done.outcome.reason}`);
|
|
88
|
-
lastSolo.set(target, done.outcome);
|
|
89
|
-
for (const k of done.outcome.skips)
|
|
90
|
-
soloSkips.add(key(k));
|
|
91
|
-
continue;
|
|
92
|
-
}
|
|
93
|
-
const run = async () => {
|
|
94
|
-
const started = Date.now();
|
|
95
|
-
const o = await verifyFile(config, journey.area, target);
|
|
96
|
-
spent += Date.now() - started;
|
|
97
|
-
return o;
|
|
98
|
-
};
|
|
99
|
-
let outcome = await run();
|
|
100
|
-
for (let attempt = 0;; attempt += 1) {
|
|
101
|
-
if (!usable(outcome)) {
|
|
102
|
-
// The file itself does not load (a bad import, a syntax the runner
|
|
103
|
-
// rejects): that is the file's fault, so it is repaired like a
|
|
104
|
-
// failing test. Only an environment problem (infra) is left alone.
|
|
105
|
-
const loadError = !outcome.ran && !outcome.infra;
|
|
106
|
-
if (loadError && attempt < ctx.attempts) {
|
|
107
|
-
console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
108
|
-
const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
|
|
109
|
-
if (await repairOnce(target, [issue], [])) {
|
|
110
|
-
outcome = await run();
|
|
111
|
-
continue;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
|
|
115
|
-
break;
|
|
116
|
-
}
|
|
117
|
-
const open = unclaimed(outcome.failures);
|
|
118
|
-
// A skip the test decided by itself is a suspect once: its own setup may
|
|
119
|
-
// have looked in the wrong place. A skip that only names a missing env
|
|
120
|
-
// var or opt-in flag is the environment's to fix.
|
|
121
|
-
const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
|
|
122
|
-
if ((!open.length && !suspect.length) || attempt === ctx.attempts)
|
|
123
|
-
break;
|
|
124
|
-
console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
|
|
125
|
-
if (!(await repairOnce(target, open, suspect)))
|
|
126
|
-
break;
|
|
127
|
-
outcome = await run();
|
|
128
|
-
}
|
|
129
|
-
lastSolo.set(target, outcome);
|
|
130
|
-
for (const k of outcome.skips)
|
|
131
|
-
soloSkips.add(key(k));
|
|
132
|
-
progress[resolve(target)] = { hash: fileHash(target), outcome };
|
|
133
|
-
saveProgress(journey.id, progress);
|
|
134
|
-
}
|
|
135
|
-
// 2. The whole journey in ONE run. Whatever fails only here is repaired with
|
|
136
|
-
// that context, once.
|
|
137
|
-
let together;
|
|
138
|
-
if (!notRun.length && targets.length > 1) {
|
|
139
|
-
const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
|
|
140
|
-
const order = targets.map((t) => basename(t)).sort().join(', ');
|
|
141
|
-
const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
|
|
142
|
-
`each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
|
|
143
|
-
`(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
|
|
144
|
-
`backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
|
|
145
|
-
`data it sets up itself.`;
|
|
146
|
-
// The runner reports absolute paths; targets may be relative.
|
|
147
|
-
const byAbs = new Map(targets.map((t) => [resolve(t), t]));
|
|
148
|
-
console.log(` running all ${targets.length} files together…`);
|
|
149
|
-
together = await verifyRun(config, journey.area, targets, budget);
|
|
150
|
-
if (usable(together)) {
|
|
151
|
-
const open = unclaimed(together.failures);
|
|
152
|
-
const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
|
|
153
|
-
const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
|
|
154
|
-
if (files.length) {
|
|
155
|
-
console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
|
|
156
|
-
for (const f of files) {
|
|
157
|
-
await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
|
|
158
|
-
}
|
|
159
|
-
together = await verifyRun(config, journey.area, targets, budget);
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
if (!usable(together))
|
|
163
|
-
notRun.push(`all files together: ${together.infra ?? together.reason}`);
|
|
164
|
-
}
|
|
165
|
-
// 3. Judge what is left on the LAST run that covered each file. Nothing is
|
|
166
|
-
// reported as a defect on the repair's word alone.
|
|
167
|
-
const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
|
|
168
|
-
const failures = finals.flatMap((o) => o.failures);
|
|
169
|
-
// Kept so `testforge review` can judge these failures again without re-running verify.
|
|
170
|
-
saveVerifyResults(journey.id, failures, finals.flatMap((o) => o.passes));
|
|
171
|
-
const unresolved = unclaimed(failures).map(label);
|
|
172
|
-
let reviewed = [];
|
|
173
|
-
const groups = groupClaims(failures, claimOf);
|
|
174
|
-
if (groups.length) {
|
|
175
|
-
console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
|
|
176
|
-
const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes), undefined, join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`));
|
|
177
|
-
reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
|
|
178
|
-
// A rejected claim is the test's fault until shown otherwise: one guarded
|
|
179
|
-
// repair with the reviewer's reason; whatever still fails is unverified and
|
|
180
|
-
// is not re-claimed as a defect.
|
|
181
|
-
for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
|
|
182
|
-
const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
|
|
183
|
-
`Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
|
|
184
|
-
for (const f of new Set(r.tests.map((t) => t.file))) {
|
|
185
|
-
const target = targets.find((t) => resolve(t) === f) ?? f;
|
|
186
|
-
const mine = r.tests.filter((t) => t.file === f);
|
|
187
|
-
if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
|
|
188
|
-
const again = await verifyFile(config, journey.area, target);
|
|
189
|
-
// Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
|
|
190
|
-
for (const t of again.failures) {
|
|
191
|
-
if (mine.some((m) => m.title === t.title) || !claimOf(t))
|
|
192
|
-
unresolved.push(`${label(t)} (after review)`);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
else {
|
|
196
|
-
for (const t of mine)
|
|
197
|
-
unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
|
|
202
|
-
writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
|
|
203
|
-
rmSync(join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`), { force: true });
|
|
204
|
-
}
|
|
205
|
-
// The checkpoints only serve an interrupted verify; a finished one starts fresh next time.
|
|
206
|
-
rmSync(join(PROGRESS_DIR, `${journey.id}.json`), { force: true });
|
|
207
|
-
if (notRun.length) {
|
|
208
|
-
warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
|
|
209
|
-
}
|
|
210
|
-
if (unresolved.length) {
|
|
211
|
-
warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
|
|
212
|
-
`do not report them; ${unresolved.slice(0, 5).join('; ')}`);
|
|
213
|
-
}
|
|
214
|
-
return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
|
|
215
|
-
}
|