@activate-core/test-forge 1.3.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +39 -33
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +49 -2
  5. package/dist/core/docs/doc-guidance.js +70 -29
  6. package/dist/core/docs/excerpts.js +294 -0
  7. package/dist/core/docs/grounding.js +55 -22
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +60 -3
  10. package/dist/core/docs/review.js +38 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +3 -3
  13. package/dist/core/gates/coverage.js +1 -16
  14. package/dist/core/gates/test-quality.js +295 -0
  15. package/dist/core/gates/validate.js +171 -7
  16. package/dist/core/generate/ai-generator.js +267 -117
  17. package/dist/core/generate/ai-usage.js +116 -5
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/complete-file.js +77 -0
  20. package/dist/core/generate/defect-review.js +59 -9
  21. package/dist/core/generate/doc-diff.js +140 -0
  22. package/dist/core/generate/harness.js +18 -34
  23. package/dist/core/generate/journey-memory.js +394 -0
  24. package/dist/core/generate/plan-cache.js +114 -7
  25. package/dist/core/generate/review-journey.js +3 -3
  26. package/dist/core/generate/run-generate.js +240 -82
  27. package/dist/core/generate/suite-planner.js +128 -23
  28. package/dist/core/generate/test-blocks.js +84 -0
  29. package/dist/core/generate/update-journey.js +118 -0
  30. package/dist/core/paths.js +90 -13
  31. package/dist/core/project/config.js +25 -2
  32. package/dist/core/project/env-template.js +2 -1
  33. package/dist/core/run/ci.js +1 -1
  34. package/dist/core/run/clean.js +1 -1
  35. package/dist/core/run/readiness.js +146 -0
  36. package/dist/core/run/run-tests.js +26 -2
  37. package/package.json +1 -1
  38. package/reporters/_shared.cjs +2 -2
  39. package/reporters/run-data.cjs +1 -1
  40. package/reporters/ui-helpers.cjs +78 -1
  41. package/dist/core/generate/verify-journey.js +0 -215
@@ -0,0 +1,146 @@
1
+ /**
2
+ * testforge — has the feature a test file checks been built yet?
3
+ *
4
+ * Tests are written from the doc, often before the code exists. Before a run,
5
+ * each file's names (the buttons, links, fields, headings, images it looks
6
+ * for) and the pages it opens are looked for in the app's source. A file whose
7
+ * names are all missing tests a feature not built yet, and is held back as
8
+ * pending rather than run to fail; a name missing with a close one in the code
9
+ * is a naming difference to settle. Once the code has them, the file runs. No AI.
10
+ */
11
+ import { existsSync, readdirSync, readFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ import { usedLabels } from '../gates/validate.js';
14
+ /** A name as the code would write it: no required-field star, no closing punctuation. */
15
+ const bare = (n) => n.replace(/\s*\*\s*$/, '').replace(/[\s.:!?…]+$/, '').replace(/\s+/g, ' ').trim();
16
+ /**
17
+ * What a test file needs the code to have: the names it locates elements by to
18
+ * act on them (role names, labels, placeholders, kit.find) and the pages it
19
+ * opens. Texts it only expects to see are left out: one the code lacks is a
20
+ * finding for the test to report, not a sign the feature is missing.
21
+ */
22
+ export function namesInTest(content) {
23
+ const names = new Set(usedLabels(content).map(bare).filter((n) => n.length >= 3));
24
+ const pages = new Set();
25
+ for (const m of content.matchAll(/\bgoto\(\s*[`'"](?:\$\{[^}]+\})?(\/[\w\-./]*)/g)) {
26
+ const p = m[1].replace(/\/+$/, '') || '/';
27
+ if (p !== '/')
28
+ pages.add(p);
29
+ }
30
+ return { names: [...names], pages: [...pages] };
31
+ }
32
+ /** The words that carry meaning: three letters or more. */
33
+ const words = (t) => new Set(t.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 3));
34
+ function closeness(a, b) {
35
+ const x = words(a);
36
+ const y = words(b);
37
+ let both = 0;
38
+ for (const w of x)
39
+ if (y.has(w))
40
+ both += 1;
41
+ return x.size + y.size ? both / (x.size + y.size - both) : 0;
42
+ }
43
+ /** Texts the code can show: JSX text and short string literals, for "did you mean". */
44
+ export function shownTexts(source) {
45
+ const out = new Set();
46
+ for (const m of source.matchAll(/>\s*([^<>{}\n]{2,60}?)\s*</g))
47
+ out.add(m[1].trim());
48
+ for (const m of source.matchAll(/(['"`])([^'"`\n]{2,60})\1/g))
49
+ if (/[a-z]/i.test(m[2]) && /\s|^[A-Z]/.test(m[2]))
50
+ out.add(m[2].trim());
51
+ return [...out];
52
+ }
53
+ /** The nearest text the code has to a name, when one is close enough to be the same thing renamed. */
54
+ export function nearestText(name, texts) {
55
+ let best = 0.3;
56
+ let nearest;
57
+ for (const t of texts) {
58
+ const c = closeness(name, t);
59
+ if (c > best) {
60
+ best = c;
61
+ nearest = t;
62
+ }
63
+ }
64
+ return nearest;
65
+ }
66
+ /** Does the app have this page: a route string in the source, or a Next.js / file-based route folder? */
67
+ function hasPage(path, haystack, appRoots) {
68
+ if (haystack.includes(`'${path}'`) || haystack.includes(`"${path}"`) || haystack.includes(`\`${path}`) || haystack.includes(`${path}/`))
69
+ return true;
70
+ const segs = path.replace(/^\//, '');
71
+ return appRoots.some((r) => ['app', 'pages', 'src/app', 'src/pages'].some((d) => existsSync(join(r, d, segs))));
72
+ }
73
+ /**
74
+ * Classify each test file: built (everything found), partial (some found),
75
+ * not-built (nothing found), renamed (a missing name has a close one in the code).
76
+ */
77
+ export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
78
+ let texts;
79
+ return files.map((file) => {
80
+ const content = readFileSync(file, 'utf-8');
81
+ // The doc says the feature is not live: held back whatever the code has.
82
+ const pending = PENDING_STAMP.exec(content)?.[1];
83
+ if (pending)
84
+ return { file, state: 'not-built', missing: [pending.trim()], renamed: [] };
85
+ const { names, pages } = namesInTest(content);
86
+ const missing = [];
87
+ const renamed = [];
88
+ for (const n of names) {
89
+ if (haystack.includes(n.toLowerCase().replace(/\s+/g, ' ')))
90
+ continue;
91
+ texts ??= sourceTexts();
92
+ const near = nearestText(n, texts);
93
+ if (near)
94
+ renamed.push({ name: n, nearest: near });
95
+ else
96
+ missing.push(n);
97
+ }
98
+ for (const p of pages)
99
+ if (!hasPage(p, haystack, appRoots))
100
+ missing.push(p);
101
+ const total = names.length + pages.length;
102
+ const found = total - missing.length - renamed.length;
103
+ // Not built: the code has none of it. A different name or a partial match still runs, and fails if it must.
104
+ const state = !total || found === total ? 'built' : found === 0 && !renamed.length ? 'not-built' : renamed.length ? 'renamed' : 'partial';
105
+ return { file, state, missing, renamed };
106
+ });
107
+ }
108
+ /** The line that holds back a file whose feature is not live yet. */
109
+ export const PENDING_STAMP = /^\/\/\s*testforge-pending:\s*(.+)$/m;
110
+ /**
111
+ * The status a journey doc gives its feature: "Status: not built", or
112
+ * "Status: behind flag <name>" for code that ships switched off. Undefined
113
+ * when the feature is live.
114
+ */
115
+ export function pendingReason(markdown) {
116
+ const m = /^status:\s*(not built(?: yet)?|behind (?:a )?flag\b[^\n]*)$/im.exec(markdown);
117
+ if (!m)
118
+ return undefined;
119
+ return /^not built/i.test(m[1]) ? 'the feature is not built yet' : `the feature is ${m[1].trim().replace(/^behind (?:a )?flag\s*/i, 'behind the flag ').trim()} — run it once that is switched on`;
120
+ }
121
+ /** The file with the pending stamp set to `reason`, or removed when there is none. */
122
+ export function withPendingStamp(content, reason) {
123
+ const without = content.replace(/^\/\/\s*testforge-pending:.*\n/m, '');
124
+ if (!reason)
125
+ return without;
126
+ const marker = /^\/\/ @generated by testforge.*\n/m.exec(without);
127
+ const at = marker ? marker.index + marker[0].length : 0;
128
+ return `${without.slice(0, at)}// testforge-pending: ${reason}\n${without.slice(at)}`;
129
+ }
130
+ /** The test files under a directory. */
131
+ export function testFilesIn(dir) {
132
+ const out = [];
133
+ const walk = (d) => {
134
+ for (const e of existsSync(d) ? readdirSync(d, { withFileTypes: true }) : []) {
135
+ const p = join(d, e.name);
136
+ if (e.isDirectory()) {
137
+ if (e.name !== 'helpers' && e.name !== 'node_modules')
138
+ walk(p);
139
+ }
140
+ else if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(e.name))
141
+ out.push(p);
142
+ }
143
+ };
144
+ walk(dir);
145
+ return out;
146
+ }
@@ -12,6 +12,8 @@
12
12
  * It runs from the area's nearest package root (where package.json /
13
13
  * playwright.config live), with config.envFile merged into the env.
14
14
  */
15
+ import { checkReadiness, testFilesIn } from './readiness.js';
16
+ import { sourceText } from '../docs/excerpts.js';
15
17
  import { spawnSync } from 'node:child_process';
16
18
  import { existsSync } from 'node:fs';
17
19
  import { join, dirname, resolve, relative } from 'node:path';
@@ -128,7 +130,27 @@ export function runTests(opts = {}) {
128
130
  const framework = resolveFramework(config, area, area);
129
131
  const root = packageRoot(testDir);
130
132
  const rel = relative(root, testDir) || '.';
131
- let cmd = runnerCommand(config, area, framework, root, rel, opts.journey);
133
+ // Tests written before their feature: hold back the files whose names and
134
+ // pages the code does not have yet, instead of running them to fail.
135
+ let targets;
136
+ if (framework === 'playwright' && !opts.all) {
137
+ const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
138
+ const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
139
+ const roots = ['src', ...(config.relatedDirs ?? [])];
140
+ const states = checkReadiness(files, sourceText(roots), () => [], [root]);
141
+ const held = states.filter((s) => s.state === 'not-built');
142
+ if (held.length) {
143
+ console.log(`\n${held.length} file(s) pending — their feature is not live yet (\`testforge bind\` shows why; --all runs them anyway):`);
144
+ for (const h of held.slice(0, 15))
145
+ console.log(` not built ${relative(root, h.file)}: ${h.missing.slice(0, 3).map((m) => `"${m}"`).join(', ')}`);
146
+ targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
147
+ if (!targets.length) {
148
+ console.log(`[skip] ${area}: nothing to run yet`);
149
+ continue;
150
+ }
151
+ }
152
+ }
153
+ let cmd = runnerCommand(config, area, framework, root, rel, opts.journey, targets);
132
154
  if (opts.passThrough?.length)
133
155
  cmd += ` ${opts.passThrough.join(' ')}`;
134
156
  ranAnything = true;
@@ -142,6 +164,8 @@ export function runTests(opts = {}) {
142
164
  console.log('\nNothing to run — generate tests first (`testforge generate`).');
143
165
  return 1;
144
166
  }
145
- console.log(anyFail ? '\nSome tests failed.' : '\nAll test runs passed.');
167
+ // The runner's own summary above says how many ran and how many skipped; a run where
168
+ // everything skipped has no failure, but it is not a pass.
169
+ console.log(anyFail ? '\nSome tests failed.' : '\nNo test failed — check the runner\'s summary above for skipped tests.');
146
170
  return anyFail ? 1 : 0;
147
171
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "1.3.0",
3
+ "version": "2.1.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -374,12 +374,12 @@ function testRoot(root, category) {
374
374
  return d ? path.resolve(root, d) : undefined;
375
375
  }
376
376
 
377
- // The defect review's verdict for each failed test (test-artifacts/defect-review/*.json),
377
+ // The defect review's verdict for each failed test (testforge/runs/defect-review/*.json),
378
378
  // so the dashboard can say whether a failure is a product defect, a test fault or the
379
379
  // environment. Keyed by absolute file + test title; an absent or broken file means none.
380
380
  function reviewVerdicts(root) {
381
381
  const map = new Map();
382
- const dir = path.join(root || process.cwd(), 'test-artifacts', 'defect-review');
382
+ const dir = path.join(root || process.cwd(), 'testforge', 'runs', 'defect-review');
383
383
  let files = [];
384
384
  try { files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); } catch { /* no reviews yet */ }
385
385
  for (const f of files) {
@@ -8,7 +8,7 @@ const path = require('path');
8
8
  const crypto = require('crypto');
9
9
  const { findRoot } = require('./_shared.cjs');
10
10
 
11
- const dir = (root) => path.join(root || findRoot(process.cwd()), 'test-artifacts', 'run-data');
11
+ const dir = (root) => path.join(root || findRoot(process.cwd()), 'testforge', 'runs', 'run-data');
12
12
 
13
13
  /** The current run's id: <yyyymmdd>-<6 hex>, created once and kept in the env. */
14
14
  function runId() {
@@ -53,4 +53,81 @@ function watchDialogs(page) {
53
53
  return messages;
54
54
  }
55
55
 
56
- module.exports = { settledClick, fillFrameInputs, watchDialogs };
56
+ /** The ARIA role each kind of element should have. */
57
+ const ROLES = {
58
+ button: 'button', link: 'link', input: 'textbox', textbox: 'textbox', field: 'textbox', textarea: 'textbox',
59
+ search: 'searchbox', dropdown: 'combobox', select: 'combobox', combobox: 'combobox', listbox: 'listbox', option: 'option',
60
+ checkbox: 'checkbox', radio: 'radio', toggle: 'switch', switch: 'switch', slider: 'slider', spinbutton: 'spinbutton',
61
+ tab: 'tab', tabpanel: 'tabpanel', menu: 'menu', menuitem: 'menuitem', dialog: 'dialog', popup: 'dialog', modal: 'dialog',
62
+ heading: 'heading', alert: 'alert', toast: 'status', status: 'status', image: 'img', img: 'img',
63
+ list: 'list', listitem: 'listitem', table: 'table', row: 'row', cell: 'cell', navigation: 'navigation', form: 'form',
64
+ };
65
+ /** Kinds a user types into or chooses with: they are also found by their label and placeholder. */
66
+ const FORM_KINDS = new Set(['input', 'textbox', 'field', 'textarea', 'search', 'dropdown', 'select', 'combobox', 'listbox',
67
+ 'checkbox', 'radio', 'toggle', 'switch', 'slider', 'spinbutton']);
68
+
69
+ const cssString = (s) => String(s).replace(/\\/g, '\\\\').replace(/"/g, '\\"');
70
+
71
+ /** Note a finding on the running test, so the report shows it without failing the test. */
72
+ function noteFinding(description) {
73
+ try {
74
+ // eslint-disable-next-line global-require
75
+ const { test } = require('@playwright/test');
76
+ test.info().annotations.push({ type: 'finding', description });
77
+ } catch {
78
+ // outside a Playwright test: nothing to annotate
79
+ }
80
+ }
81
+
82
+ /**
83
+ * Find an element the way a user does: by what it is and what it says. Built
84
+ * before the feature, a test cannot know how the element will be made, so it
85
+ * is looked for in order — the proper role and name; for form controls their
86
+ * label and placeholder; an aria-label or title; for an image its alt text;
87
+ * finally the visible text. A match by anything but the proper role still
88
+ * works, and adds a finding to the test (e.g. a "button" built as a div,
89
+ * which keyboard and screen-reader users cannot use). Nothing found within
90
+ * `timeout` fails with what was tried.
91
+ *
92
+ * kind: button, link, input, dropdown, checkbox, radio, toggle, tab, menuitem,
93
+ * dialog, heading, alert, toast, image, list, table … (or any ARIA role)
94
+ * name: the text the doc gives it; a RegExp is allowed.
95
+ */
96
+ async function find(page, kind, name, { exact = false, timeout = 10000, within } = {}) {
97
+ const root = within || page;
98
+ const role = ROLES[kind] || kind;
99
+ const isText = typeof name === 'string';
100
+ const tries = [
101
+ ['role', () => root.getByRole(role, { name, exact })],
102
+ ...(FORM_KINDS.has(kind) ? [
103
+ ['label', () => root.getByLabel(name, { exact })],
104
+ ['placeholder', () => root.getByPlaceholder(name, { exact })],
105
+ ] : []),
106
+ ...(isText ? [['aria-label or title', () => root.locator(`[aria-label="${cssString(name)}"], [title="${cssString(name)}"]`)]] : []),
107
+ ...(kind === 'image' || kind === 'img' ? [['alt text', () => root.getByAltText(name, { exact })]] : []),
108
+ ['visible text', () => root.getByText(name, { exact })],
109
+ ];
110
+ const deadline = Date.now() + timeout;
111
+ for (;;) {
112
+ for (const [how, make] of tries) {
113
+ const all = make();
114
+ const n = await all.count().catch(() => 0);
115
+ if (!n) continue;
116
+ for (let i = 0; i < n; i++) {
117
+ const el = all.nth(i);
118
+ if (!(await el.isVisible().catch(() => false))) continue;
119
+ if (how !== 'role') {
120
+ noteFinding(`"${name}" was expected as a ${kind} (role ${role}) but was found only by its ${how}: ` +
121
+ 'it is not exposed as one, so keyboard and screen-reader users may not be able to use it.');
122
+ }
123
+ return el;
124
+ }
125
+ }
126
+ if (Date.now() > deadline) {
127
+ throw new Error(`No ${kind} "${name}" on the page: tried ${tries.map(([h]) => h).join(', ')}.`);
128
+ }
129
+ await settle(250);
130
+ }
131
+ }
132
+
133
+ module.exports = { settledClick, fillFrameInputs, watchDialogs, find };
@@ -1,215 +0,0 @@
1
- /**
2
- * testforge — check a journey's generated tests against the real app before
3
- * they are handed over.
4
- *
5
- * A test that passes every static gate can still target a selector, route or
6
- * field the app does not have; only running it can tell. Three passes:
7
- *
8
- * 1. Each file on its own. A failure, or a skip the test decided by itself,
9
- * is repaired from the runner's own output. A test the repair judges to
10
- * be failing on the APPLICATION is left as it is and marked.
11
- * 2. The whole journey in one run, as a team or CI runs it. Files that pass
12
- * alone can still break each other (one signs in again as a shared
13
- * account and ends another's session); those are repaired with that
14
- * context.
15
- * 3. An independent review of every marked failure. Only what it confirms
16
- * is reported; the rest is repaired once more or reported as unverified.
17
- *
18
- * A guard refuses any repair that deletes a test, adds a skip or loses an
19
- * assertion, so no repair can turn a caught defect into a silent one.
20
- */
21
- import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync } from 'node:fs';
22
- import { createHash } from 'node:crypto';
23
- import { basename, join, resolve } from 'node:path';
24
- import { STATE_DIR } from '../paths.js';
25
- import { DEFECT_MARK } from './ai-generator.js';
26
- import { groupClaims, normaliseClaims, reviewDefects } from './defect-review.js';
27
- import { saveVerifyResults } from './review-journey.js';
28
- import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
29
- const describe = (label, issues) => issues.length
30
- ? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
31
- : [];
32
- const key = (i) => `${i.file}:${i.line}:${i.title}`;
33
- const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
34
- const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
35
- const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
36
- const usable = (o) => o.ran && !o.infra;
37
- const PROGRESS_DIR = join(STATE_DIR, 'verify-progress');
38
- const fileHash = (f) => (existsSync(f) ? createHash('sha1').update(readFileSync(f)).digest('hex') : '');
39
- function loadProgress(journeyId) {
40
- try {
41
- return JSON.parse(readFileSync(join(PROGRESS_DIR, `${journeyId}.json`), 'utf-8'));
42
- }
43
- catch {
44
- return {};
45
- }
46
- }
47
- function saveProgress(journeyId, p) {
48
- mkdirSync(PROGRESS_DIR, { recursive: true });
49
- writeFileSync(join(PROGRESS_DIR, `${journeyId}.json`), JSON.stringify(p), 'utf-8');
50
- }
51
- export async function verifyJourney(ctx) {
52
- const { config, journey, targets } = ctx;
53
- const warnings = [];
54
- const notRun = [];
55
- // One guarded repair of one file; true when the file changed.
56
- const repairOnce = async (target, open, suspect, note = '') => {
57
- const current = readFileSync(target, 'utf-8');
58
- const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
59
- const repaired = await ctx.repair(target, current, failure);
60
- if (repaired === null)
61
- return false;
62
- // Only a quoted [spec] rule may be claimed as a defect; anything else becomes a finding mark.
63
- const { content: next, downgraded } = normaliseClaims(repaired, journey.rawMarkdown);
64
- if (downgraded)
65
- console.log(` ${basename(target)}: ${downgraded} defect claim(s) against a non-[spec] rule kept as finding(s)`);
66
- const refused = repairGuard(current, next);
67
- if (refused) {
68
- warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
69
- return false;
70
- }
71
- writeFileSync(target, next, 'utf-8');
72
- return true;
73
- };
74
- // 1. Each file on its own: what is wrong inside the file.
75
- // A file that finished this step before (same content since) is not run
76
- // again: an interrupted verify — a usage limit, a closed laptop — resumes
77
- // at the first file it had not finished.
78
- let spent = 0;
79
- const soloSkips = new Set();
80
- const lastSolo = new Map();
81
- const progress = loadProgress(journey.id);
82
- for (const target of targets) {
83
- const done = progress[resolve(target)];
84
- if (done && done.hash === fileHash(target)) {
85
- console.log(` ${basename(target)}: verified earlier, unchanged — kept`);
86
- if (!usable(done.outcome))
87
- notRun.push(`${basename(target)}: ${done.outcome.infra ?? done.outcome.reason}`);
88
- lastSolo.set(target, done.outcome);
89
- for (const k of done.outcome.skips)
90
- soloSkips.add(key(k));
91
- continue;
92
- }
93
- const run = async () => {
94
- const started = Date.now();
95
- const o = await verifyFile(config, journey.area, target);
96
- spent += Date.now() - started;
97
- return o;
98
- };
99
- let outcome = await run();
100
- for (let attempt = 0;; attempt += 1) {
101
- if (!usable(outcome)) {
102
- // The file itself does not load (a bad import, a syntax the runner
103
- // rejects): that is the file's fault, so it is repaired like a
104
- // failing test. Only an environment problem (infra) is left alone.
105
- const loadError = !outcome.ran && !outcome.infra;
106
- if (loadError && attempt < ctx.attempts) {
107
- console.log(` ${basename(target)}: does not load — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
108
- const issue = { file: resolve(target), line: 0, title: 'the file does not load', detail: outcome.reason ?? '' };
109
- if (await repairOnce(target, [issue], [])) {
110
- outcome = await run();
111
- continue;
112
- }
113
- }
114
- notRun.push(`${basename(target)}: ${outcome.infra ?? outcome.reason}`);
115
- break;
116
- }
117
- const open = unclaimed(outcome.failures);
118
- // A skip the test decided by itself is a suspect once: its own setup may
119
- // have looked in the wrong place. A skip that only names a missing env
120
- // var or opt-in flag is the environment's to fix.
121
- const suspect = attempt === 0 ? outcome.skips.filter((k) => !environmentSkip(k.detail)) : [];
122
- if ((!open.length && !suspect.length) || attempt === ctx.attempts)
123
- break;
124
- console.log(` ${basename(target)}: ${open.length} failing, ${suspect.length} self-skipped — repairing (attempt ${attempt + 1}/${ctx.attempts})`);
125
- if (!(await repairOnce(target, open, suspect)))
126
- break;
127
- outcome = await run();
128
- }
129
- lastSolo.set(target, outcome);
130
- for (const k of outcome.skips)
131
- soloSkips.add(key(k));
132
- progress[resolve(target)] = { hash: fileHash(target), outcome };
133
- saveProgress(journey.id, progress);
134
- }
135
- // 2. The whole journey in ONE run. Whatever fails only here is repaired with
136
- // that context, once.
137
- let together;
138
- if (!notRun.length && targets.length > 1) {
139
- const budget = Math.max(config.verifyTimeoutMs ?? 600_000, Math.round(spent * 1.5) + 300_000);
140
- const order = targets.map((t) => basename(t)).sort().join(', ');
141
- const note = `These failed only when all of this journey's files ran TOGETHER in one run (in this order: ${order}); ` +
142
- `each passes on its own. Find what another file changes for this one — a shared account signed in again ` +
143
- `(ending this file's session), data another file creates or deletes, state left in the browser or the ` +
144
- `backend — and make this file independent of it: its own identity, tokens re-confirmed through the harness, ` +
145
- `data it sets up itself.`;
146
- // The runner reports absolute paths; targets may be relative.
147
- const byAbs = new Map(targets.map((t) => [resolve(t), t]));
148
- console.log(` running all ${targets.length} files together…`);
149
- together = await verifyRun(config, journey.area, targets, budget);
150
- if (usable(together)) {
151
- const open = unclaimed(together.failures);
152
- const fresh = together.skips.filter((k) => !soloSkips.has(key(k)) && !environmentSkip(k.detail));
153
- const files = [...new Set([...open, ...fresh].map((i) => i.file))].filter((f) => byAbs.has(f));
154
- if (files.length) {
155
- console.log(` together: ${open.length} failing, ${fresh.length} newly skipped in ${files.length} file(s) — repairing`);
156
- for (const f of files) {
157
- await repairOnce(byAbs.get(f), open.filter((i) => i.file === f), fresh.filter((i) => i.file === f), note);
158
- }
159
- together = await verifyRun(config, journey.area, targets, budget);
160
- }
161
- }
162
- if (!usable(together))
163
- notRun.push(`all files together: ${together.infra ?? together.reason}`);
164
- }
165
- // 3. Judge what is left on the LAST run that covered each file. Nothing is
166
- // reported as a defect on the repair's word alone.
167
- const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
168
- const failures = finals.flatMap((o) => o.failures);
169
- // Kept so `testforge review` can judge these failures again without re-running verify.
170
- saveVerifyResults(journey.id, failures, finals.flatMap((o) => o.passes));
171
- const unresolved = unclaimed(failures).map(label);
172
- let reviewed = [];
173
- const groups = groupClaims(failures, claimOf);
174
- if (groups.length) {
175
- console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
176
- const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes), undefined, join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`));
177
- reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
178
- // A rejected claim is the test's fault until shown otherwise: one guarded
179
- // repair with the reviewer's reason; whatever still fails is unverified and
180
- // is not re-claimed as a defect.
181
- for (const r of reviews.filter((x) => x.verdict === 'test-fault' || x.verdict === 'unproven')) {
182
- const note = `An independent review found this is NOT shown to be a product defect: ${r.reason} ` +
183
- `Fix the test so it checks the behaviour correctly, and remove its "${DEFECT_MARK}" line.`;
184
- for (const f of new Set(r.tests.map((t) => t.file))) {
185
- const target = targets.find((t) => resolve(t) === f) ?? f;
186
- const mine = r.tests.filter((t) => t.file === f);
187
- if (r.verdict === 'test-fault' && (await repairOnce(target, mine, [], note))) {
188
- const again = await verifyFile(config, journey.area, target);
189
- // Only this claim's tests: a confirmed defect elsewhere in the file still fails, rightly.
190
- for (const t of again.failures) {
191
- if (mine.some((m) => m.title === t.title) || !claimOf(t))
192
- unresolved.push(`${label(t)} (after review)`);
193
- }
194
- }
195
- else {
196
- for (const t of mine)
197
- unresolved.push(`${label(t)} — review: ${r.verdict}, ${r.reason}`);
198
- }
199
- }
200
- }
201
- mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
202
- writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
203
- rmSync(join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`), { force: true });
204
- }
205
- // The checkpoints only serve an interrupted verify; a finished one starts fresh next time.
206
- rmSync(join(PROGRESS_DIR, `${journey.id}.json`), { force: true });
207
- if (notRun.length) {
208
- warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
209
- }
210
- if (unresolved.length) {
211
- warnings.push(`verify: ${unresolved.length} test(s) still fail and were not shown to be a product defect — ` +
212
- `do not report them; ${unresolved.slice(0, 5).join('; ')}`);
213
- }
214
- return { verified: !notRun.length && !unresolved.length, reviewed, warnings };
215
- }