@activate-core/test-forge 1.3.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +70 -29
- package/dist/core/docs/excerpts.js +294 -0
- package/dist/core/docs/grounding.js +55 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/test-quality.js +295 -0
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +267 -117
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/complete-file.js +77 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +140 -0
- package/dist/core/generate/harness.js +18 -34
- package/dist/core/generate/journey-memory.js +394 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +240 -82
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +118 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +146 -0
- package/dist/core/run/run-tests.js +26 -2
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — a journey read once, handed out in pieces.
|
|
3
|
+
*
|
|
4
|
+
* The journey doc is parsed into its rules, controls and test setup, the
|
|
5
|
+
* testkit into the list of helpers it exports, and both are saved next to the
|
|
6
|
+
* plan. Every later AI call receives only the rules, controls and facts its
|
|
7
|
+
* own file needs, and never the whole doc again. Measured on one small
|
|
8
|
+
* journey, re-sending the doc, the map and the facts to every file and every
|
|
9
|
+
* repair was most of 9.6M tokens; the code below needs no AI at all.
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { harnessSteps } from '../gates/test-quality.js';
|
|
13
|
+
import { dirname, join } from 'node:path';
|
|
14
|
+
import { journeyDir } from '../paths.js';
|
|
15
|
+
/** Sections that are indexes or open questions, never something to test. */
|
|
16
|
+
const NOT_RULES = /^(controls|coverage|test setup|questions for product|not requirements|pending decisions|suggested)\b/i;
|
|
17
|
+
const TAG = /\[(spec|baseline|observed)\]/;
|
|
18
|
+
const CONTROL_REF = /\bC\d+[a-z]?\b/g;
|
|
19
|
+
/** Split a journey doc into header, controls, rules and setup. Pure: no AI, no I/O. */
|
|
20
|
+
export function readRules(markdown) {
|
|
21
|
+
const lines = markdown.split('\n');
|
|
22
|
+
const header = [];
|
|
23
|
+
const setup = [];
|
|
24
|
+
const controls = {};
|
|
25
|
+
const rules = [];
|
|
26
|
+
let section = '';
|
|
27
|
+
let group = '';
|
|
28
|
+
let mode = 'header';
|
|
29
|
+
let current = null;
|
|
30
|
+
const close = () => {
|
|
31
|
+
if (!current)
|
|
32
|
+
return;
|
|
33
|
+
const r = current.rule;
|
|
34
|
+
r.text = r.text.trimEnd();
|
|
35
|
+
r.controls = [...new Set(r.text.match(CONTROL_REF) ?? [])];
|
|
36
|
+
const marked = /\]\s*\((must|should|could)\b/i.exec(r.text)?.[1]?.toLowerCase();
|
|
37
|
+
r.level = r.tag === 'spec' ? 'must' : marked;
|
|
38
|
+
current = null;
|
|
39
|
+
};
|
|
40
|
+
for (const line of lines) {
|
|
41
|
+
const heading = /^(#{2,3})\s+(.*)$/.exec(line);
|
|
42
|
+
if (heading) {
|
|
43
|
+
close();
|
|
44
|
+
const title = heading[2].trim();
|
|
45
|
+
if (heading[1] === '##') {
|
|
46
|
+
if (/^controls\b/i.test(title))
|
|
47
|
+
mode = 'controls';
|
|
48
|
+
else if (/^test setup\b/i.test(title))
|
|
49
|
+
mode = 'setup';
|
|
50
|
+
else if (NOT_RULES.test(title))
|
|
51
|
+
mode = 'skip';
|
|
52
|
+
else
|
|
53
|
+
mode = 'rules';
|
|
54
|
+
}
|
|
55
|
+
else if (mode === 'skip' || mode === 'controls' || mode === 'setup') {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const num = /^(\d+(?:\.\d+)*)\.?\s+(.*)$/.exec(title);
|
|
59
|
+
if (mode === 'rules') {
|
|
60
|
+
section = num ? `§${num[1]} ${num[2]}` : title;
|
|
61
|
+
if (heading[1] === '##')
|
|
62
|
+
group = section;
|
|
63
|
+
}
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (/^#\s/.test(line))
|
|
67
|
+
continue; // the journey title
|
|
68
|
+
if (mode === 'header') {
|
|
69
|
+
header.push(line);
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (mode === 'setup') {
|
|
73
|
+
setup.push(line);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (mode === 'controls') {
|
|
77
|
+
// "| C3 |" or a plain "| 3 |": both are control C3.
|
|
78
|
+
const row = /^\|\s*C?(\d+[a-z]?)\s*\|(.*)\|\s*$/.exec(line);
|
|
79
|
+
if (row)
|
|
80
|
+
controls[`C${row[1]}`] = row[2].split('|').map((c) => c.trim()).filter(Boolean).join(' · ');
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (mode !== 'rules')
|
|
84
|
+
continue;
|
|
85
|
+
const bullet = /^(\s*)[-*]\s+(.*)$/.exec(line);
|
|
86
|
+
if (bullet) {
|
|
87
|
+
const indent = bullet[1].length;
|
|
88
|
+
// A sub-bullet under a rule (its list of cases, its note) belongs to it.
|
|
89
|
+
if (current && indent > current.indent) {
|
|
90
|
+
current.rule.text += `\n${line}`;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
close();
|
|
94
|
+
const tag = (TAG.exec(bullet[2])?.[1] ?? '');
|
|
95
|
+
current = { rule: { id: `R${rules.length + 1}`, section, group, tag, text: bullet[2], controls: [] }, indent };
|
|
96
|
+
rules.push(current.rule);
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
// A paragraph, a table or a blank line inside a rule's block stays with it.
|
|
100
|
+
if (current && (line.trim() === '' || /^\s/.test(line) || line.startsWith('|'))) {
|
|
101
|
+
current.rule.text += `\n${line}`;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
close();
|
|
105
|
+
// Free text between rules (a section's lead-in) becomes context for the next rule's section.
|
|
106
|
+
if (line.trim())
|
|
107
|
+
rules.push({ id: `R${rules.length + 1}`, section, group, tag: '', text: line.trim(), controls: [...new Set(line.match(CONTROL_REF) ?? [])] });
|
|
108
|
+
}
|
|
109
|
+
close();
|
|
110
|
+
return { header: header.join('\n').trim(), rules, controls, setup: setup.join('\n').trim() };
|
|
111
|
+
}
|
|
112
|
+
/** One line per rule — what the planner and the gate see instead of the doc. */
|
|
113
|
+
export function ruleIndex(memory, tags = ['spec', 'baseline', 'observed']) {
|
|
114
|
+
return memory.rules
|
|
115
|
+
.filter((r) => r.tag && tags.includes(r.tag))
|
|
116
|
+
.map((r) => {
|
|
117
|
+
const oneLine = r.text.split('\n')[0].replace(TAG, '').replace(/\(source:[^)]*\)/g, '').replace(/\s+/g, ' ').trim();
|
|
118
|
+
return `${r.id} [${r.tag}] ${r.section} — ${oneLine.length > 160 ? `${oneLine.slice(0, 157)}…` : oneLine}`;
|
|
119
|
+
})
|
|
120
|
+
.join('\n');
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* The part of the journey one file needs: its own rules in full, the
|
|
124
|
+
* untagged notes and [observed] mechanics of the same sections, and the
|
|
125
|
+
* controls all of those name. Nothing else of the doc.
|
|
126
|
+
*/
|
|
127
|
+
export function journeySlice(memory, ruleIds) {
|
|
128
|
+
const own = memory.rules.filter((r) => ruleIds.includes(r.id));
|
|
129
|
+
const sections = new Set(own.map((r) => r.section));
|
|
130
|
+
const context = memory.rules.filter((r) => !ruleIds.includes(r.id) && sections.has(r.section) && (r.tag === '' || r.tag === 'observed'));
|
|
131
|
+
const controlIds = [...new Set([...own, ...context].flatMap((r) => r.controls))].sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
|
|
132
|
+
return { text: sliceText(memory, own, context, controlIds), controls: controlIds };
|
|
133
|
+
}
|
|
134
|
+
function sliceText(memory, own, context, controlIds) {
|
|
135
|
+
const bySection = (rs) => [...new Set(rs.map((r) => r.section))].map((s) => [`### ${s}`, ...rs.filter((r) => r.section === s).map((r) => `- ${r.id}: ${r.text}`)].join('\n')).join('\n\n');
|
|
136
|
+
return [
|
|
137
|
+
memory.header,
|
|
138
|
+
'',
|
|
139
|
+
controlIds.length ? ['## Controls', ...controlIds.map((c) => `- ${c}: ${memory.controls[c] ?? '(not in the Controls table)'}`)].join('\n') : '',
|
|
140
|
+
'',
|
|
141
|
+
'## Rules this file tests',
|
|
142
|
+
bySection(own),
|
|
143
|
+
...(context.length ? ['', '## Context from the same sections (not tested here, explains how the app works)', bySection(context)] : []),
|
|
144
|
+
...(memory.setup ? ['', '## Test setup', memory.setup] : []),
|
|
145
|
+
].filter((x, i, a) => !(x === '' && a[i - 1] === '')).join('\n').trim();
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* The testkit's exports, one line each: signature plus its comment. Read from
|
|
149
|
+
* the file, not asked of the AI, so every writer gets the same true list
|
|
150
|
+
* without opening the testkit itself.
|
|
151
|
+
*/
|
|
152
|
+
/** TestForge's own runtime helpers, as a testkit re-exports them (find: uiHelpers.find). */
|
|
153
|
+
const RUNTIME_HELPERS = {
|
|
154
|
+
find: "async find(page, kind, name, { exact, timeout, within }) → Promise<Locator> — ALWAYS await it before use: `const b = await kit.find(page, 'button', 'Pay')`; never kit.find(...).first() and never expect(kit.find(...)). kind: button, link, input, dropdown, checkbox, radio, toggle, tab, menuitem, dialog, heading, alert, toast, image …; name: the text the doc gives it (or a RegExp).",
|
|
155
|
+
settledClick: 'async settledClick(locator, { settleMs, tries }) — clicks once the element has stopped moving (pop-ups, growing forms).',
|
|
156
|
+
fillFrameInputs: 'async fillFrameInputs(page, framePrefix, { name: value }) — fills inputs inside an embedded iframe (card fields).',
|
|
157
|
+
watchDialogs: 'watchDialogs(page) → string[] — accepts browser alerts and records their messages.',
|
|
158
|
+
};
|
|
159
|
+
export function helperApi(harnessSource) {
|
|
160
|
+
const names = new Set();
|
|
161
|
+
const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(harnessSource);
|
|
162
|
+
if (cjs)
|
|
163
|
+
for (const n of cjs[1].split(',')) {
|
|
164
|
+
const m = /^\s*(\w+)/.exec(n);
|
|
165
|
+
if (m)
|
|
166
|
+
names.add(m[1]);
|
|
167
|
+
}
|
|
168
|
+
for (const m of harnessSource.matchAll(/^export\s+(?:async\s+)?(?:function\*?|const|let|class)\s+(\w+)/gm))
|
|
169
|
+
names.add(m[1]);
|
|
170
|
+
for (const m of harnessSource.matchAll(/^export\s*\{([^}]*)\}/gm))
|
|
171
|
+
for (const n of m[1].split(',')) {
|
|
172
|
+
const w = /(\w+)\s*$/.exec(n.trim());
|
|
173
|
+
if (w)
|
|
174
|
+
names.add(w[1]);
|
|
175
|
+
}
|
|
176
|
+
const lines = harnessSource.split('\n');
|
|
177
|
+
const out = [];
|
|
178
|
+
const steps = harnessSteps(harnessSource);
|
|
179
|
+
for (const name of names) {
|
|
180
|
+
const at = lines.findIndex((l) => new RegExp(`^\\s*(?:export\\s+)?(?:async\\s+)?(?:function\\*?\\s+${name}\\s*\\(|(?:const|let|var)\\s+${name}\\s*=)`).test(l));
|
|
181
|
+
if (at === -1) {
|
|
182
|
+
// Re-exported from TestForge's runtime (find: uiHelpers.find): its contract, not just its name.
|
|
183
|
+
out.push(RUNTIME_HELPERS[name] ? `- ${RUNTIME_HELPERS[name]}` : `- ${name}`);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (name === 'openStep' && steps.length) {
|
|
187
|
+
out.push(`- ${lines[at].trim().replace(/^export\s+/, '').replace(/\s*\{\s*\}?\s*$/, '')} — step is EXACTLY one of: ${steps.join(', ')}. Any other name does not exist. Resolves true when the step is open, else a string saying why.`);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
let sig = lines[at].trim().replace(/^export\s+/, '').replace(/\s*\{\s*\}?\s*$/, '');
|
|
191
|
+
// A constant's value is what the writer needs (TEST_IDS, a URL map): keep it whole, within reason.
|
|
192
|
+
if (/^(const|let|var)\s/.test(sig) && /[{[]\s*$/.test(lines[at])) {
|
|
193
|
+
let depth = 0;
|
|
194
|
+
const body = [];
|
|
195
|
+
for (let i = at; i < lines.length && body.join('\n').length < 1500; i++) {
|
|
196
|
+
body.push(lines[i]);
|
|
197
|
+
depth += (lines[i].match(/[{[]/g) ?? []).length - (lines[i].match(/[}\]]/g) ?? []).length;
|
|
198
|
+
if (depth <= 0)
|
|
199
|
+
break;
|
|
200
|
+
}
|
|
201
|
+
out.push(`- ${body.join('\n ').replace(/^export\s+/, '')}`);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
const doc = [];
|
|
205
|
+
for (let i = at - 1; i >= 0 && doc.length < 3; i--) {
|
|
206
|
+
const t = lines[i].trim();
|
|
207
|
+
if (!t || t === '*/' || t === '/**')
|
|
208
|
+
continue;
|
|
209
|
+
if (!/^(\/\/|\*|\/\*\*)/.test(t))
|
|
210
|
+
break;
|
|
211
|
+
const text = t.replace(/^(\/\/+|\/\*\*|\*\/?)\s?/, '').replace(/^-{4,}.*$/, '').trim();
|
|
212
|
+
if (text)
|
|
213
|
+
doc.unshift(text);
|
|
214
|
+
}
|
|
215
|
+
out.push(`- ${sig.length > 200 ? `${sig.slice(0, 197)}…` : sig}${doc.length ? ` — ${doc.join(' ')}` : ''}`);
|
|
216
|
+
}
|
|
217
|
+
return out.join('\n');
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* The code facts a file needs: every section of the grounding as it is,
|
|
221
|
+
* except SCREENS, of which only the lines for this file's controls (and the
|
|
222
|
+
* untagged ones) are kept. Facts without control tags are passed whole.
|
|
223
|
+
*/
|
|
224
|
+
export function factsSlice(facts, controlIds) {
|
|
225
|
+
if (!facts)
|
|
226
|
+
return facts;
|
|
227
|
+
const lines = facts.split('\n');
|
|
228
|
+
const tagged = /^\s*[-*]?\s*\[(C\d+[a-z]?(?:\s*,\s*C\d+[a-z]?)*)\]/;
|
|
229
|
+
if (!lines.some((l) => tagged.test(l)))
|
|
230
|
+
return facts;
|
|
231
|
+
const want = new Set(controlIds);
|
|
232
|
+
return lines
|
|
233
|
+
.filter((l) => {
|
|
234
|
+
const m = tagged.exec(l);
|
|
235
|
+
return !m || m[1].split(',').some((c) => want.has(c.trim()));
|
|
236
|
+
})
|
|
237
|
+
.join('\n');
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Write the journey's memory: its rules with source, tests, skips and history.
|
|
241
|
+
* Earlier wordings and removed rules are kept, so "why did this test change"
|
|
242
|
+
* has an answer. Pure bookkeeping, no AI.
|
|
243
|
+
*/
|
|
244
|
+
export function saveMemory(journey, docHash, memory, helpers, extra) {
|
|
245
|
+
const path = join(journeyDir(journey), 'rules.json');
|
|
246
|
+
const now = new Date().toISOString();
|
|
247
|
+
const prev = extra.prev;
|
|
248
|
+
const prevRule = new Map((prev?.rules ?? []).map((r) => [r.id, r]));
|
|
249
|
+
const renamed = new Set(extra.changes?.renamed ?? []);
|
|
250
|
+
const changed = new Set(extra.changes?.changed ?? []);
|
|
251
|
+
const skip = new Map((extra.skipped ?? []).map((x) => [x.rule, x.why]));
|
|
252
|
+
const rules = memory.rules.map((r) => {
|
|
253
|
+
const before = prevRule.get(r.id);
|
|
254
|
+
const moved = renamed.has(r.id) || changed.has(r.id);
|
|
255
|
+
const quote = /\(source:[^"“]*["“]([^"”]+)["”]/.exec(r.text)?.[1];
|
|
256
|
+
return {
|
|
257
|
+
...r,
|
|
258
|
+
category: /\]\s*\((?:must|should|could),\s*([^)]+)\)/i.exec(r.text)?.[1]?.trim(),
|
|
259
|
+
source: extra.reference || quote ? { reference: extra.reference, quote } : undefined,
|
|
260
|
+
testedBy: (extra.plan ?? []).filter((f) => f.rules?.includes(r.id)).map((f) => f.slug),
|
|
261
|
+
skipped: skip.get(r.id),
|
|
262
|
+
since: before?.since ?? now,
|
|
263
|
+
updated: moved || !before ? now : before.updated,
|
|
264
|
+
history: moved && before
|
|
265
|
+
? [...(before.history ?? []), { at: now, change: renamed.has(r.id) ? 'renamed' : 'changed', was: before.text }]
|
|
266
|
+
: before?.history,
|
|
267
|
+
};
|
|
268
|
+
});
|
|
269
|
+
const removedNow = (extra.changes?.removed ?? []).flatMap((id) => {
|
|
270
|
+
const r = prevRule.get(id);
|
|
271
|
+
return r ? [{ id, text: r.text, at: now }] : [];
|
|
272
|
+
});
|
|
273
|
+
const record = {
|
|
274
|
+
journey: journey.id,
|
|
275
|
+
name: journey.name,
|
|
276
|
+
doc: extra.doc,
|
|
277
|
+
hash: docHash,
|
|
278
|
+
reference: extra.reference,
|
|
279
|
+
updated: now,
|
|
280
|
+
header: memory.header,
|
|
281
|
+
controls: memory.controls,
|
|
282
|
+
setup: memory.setup,
|
|
283
|
+
helpers,
|
|
284
|
+
rules,
|
|
285
|
+
removed: [...(prev?.removed ?? []), ...removedNow],
|
|
286
|
+
};
|
|
287
|
+
try {
|
|
288
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
289
|
+
writeFileSync(path, JSON.stringify(record, null, 2), 'utf-8');
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
// best-effort: the memory is rebuilt from the doc when it is missing
|
|
293
|
+
}
|
|
294
|
+
return path;
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Put what the files really test into the journey's memory and its plan
|
|
298
|
+
* check: which file tests each rule, which rules cannot be tested here and
|
|
299
|
+
* why, and which have no test. Returns the one-line summary.
|
|
300
|
+
*/
|
|
301
|
+
export function recordWritten(journey, files) {
|
|
302
|
+
const dir = journeyDir(journey);
|
|
303
|
+
const path = join(dir, 'rules.json');
|
|
304
|
+
const testedBy = new Map();
|
|
305
|
+
const why = new Map();
|
|
306
|
+
const missing = new Set();
|
|
307
|
+
for (const f of files) {
|
|
308
|
+
for (const id of f.tested)
|
|
309
|
+
testedBy.set(id, [...(testedBy.get(id) ?? []), f.slug]);
|
|
310
|
+
for (const c of f.cannotTest)
|
|
311
|
+
why.set(c.id, `cannot be tested here: ${c.why}`);
|
|
312
|
+
for (const id of f.missing)
|
|
313
|
+
missing.add(id);
|
|
314
|
+
}
|
|
315
|
+
const planned = new Set(files.flatMap((f) => [...f.tested, ...f.cannotTest.map((c) => c.id), ...f.missing]));
|
|
316
|
+
try {
|
|
317
|
+
const record = JSON.parse(readFileSync(path, 'utf-8'));
|
|
318
|
+
for (const r of record.rules) {
|
|
319
|
+
if (!planned.has(r.id))
|
|
320
|
+
continue;
|
|
321
|
+
r.testedBy = testedBy.get(r.id) ?? [];
|
|
322
|
+
r.skipped = why.get(r.id) ?? (missing.has(r.id) ? 'no test was written for it' : undefined);
|
|
323
|
+
}
|
|
324
|
+
writeFileSync(path, JSON.stringify(record, null, 2), 'utf-8');
|
|
325
|
+
}
|
|
326
|
+
catch {
|
|
327
|
+
// no memory file: the summary below still reports it
|
|
328
|
+
}
|
|
329
|
+
const dropped = files.reduce((n, f) => n + f.dropped.length, 0);
|
|
330
|
+
const lines = [
|
|
331
|
+
`${testedBy.size}/${planned.size} planned rules have a test`,
|
|
332
|
+
why.size ? `${why.size} cannot be tested here (${[...why.keys()].join(', ')})` : '',
|
|
333
|
+
missing.size ? `NO TEST for ${[...missing].join(', ')}` : '',
|
|
334
|
+
dropped ? `${dropped} test(s) taken out because they could not check their rule` : '',
|
|
335
|
+
].filter(Boolean);
|
|
336
|
+
const check = join(dir, 'plan-check.md');
|
|
337
|
+
try {
|
|
338
|
+
const before = existsSync(check) ? readFileSync(check, 'utf-8').replace(/\n## Written\n[\s\S]*$/, '') : '';
|
|
339
|
+
const detail = [
|
|
340
|
+
...[...why].map(([id, w]) => `- ${id}: ${w}`),
|
|
341
|
+
...[...missing].map((id) => `- ${id}: no test was written for it`),
|
|
342
|
+
];
|
|
343
|
+
writeFileSync(check, `${before.trimEnd()}\n\n## Written\n\n${lines.join('; ')}.\n${detail.length ? `\n${detail.join('\n')}\n` : ''}`, 'utf-8');
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
// best-effort
|
|
347
|
+
}
|
|
348
|
+
return lines.join('; ');
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Rules of this journey that another journey already tests, found without the
|
|
352
|
+
* AI: the same tag and nearly the same words (≥ 85 % of the words shared).
|
|
353
|
+
* They are not planned again; the plan check names where they are tested.
|
|
354
|
+
*/
|
|
355
|
+
export function testedElsewhere(journey, memory, tested) {
|
|
356
|
+
const others = [];
|
|
357
|
+
const root = join(journeyDir(journey), '..', '..');
|
|
358
|
+
for (const area of existsSync(root) ? readdirSync(root) : []) {
|
|
359
|
+
for (const id of existsSync(join(root, area)) ? readdirSync(join(root, area)) : []) {
|
|
360
|
+
if (id === journey.id)
|
|
361
|
+
continue;
|
|
362
|
+
try {
|
|
363
|
+
others.push(JSON.parse(readFileSync(join(root, area, id, 'rules.json'), 'utf-8')));
|
|
364
|
+
}
|
|
365
|
+
catch {
|
|
366
|
+
// no memory for that journey yet
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
const words = (t) => new Set(t.replace(/\(source:[^)]*\)/g, '').toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 2));
|
|
371
|
+
const out = [];
|
|
372
|
+
for (const r of memory.rules) {
|
|
373
|
+
if (!r.tag || !tested.includes(r.tag))
|
|
374
|
+
continue;
|
|
375
|
+
const mine = words(r.text);
|
|
376
|
+
for (const o of others) {
|
|
377
|
+
const hit = o.rules.find((x) => {
|
|
378
|
+
if (x.tag !== r.tag || !x.testedBy?.length)
|
|
379
|
+
return false;
|
|
380
|
+
const theirs = words(x.text);
|
|
381
|
+
let both = 0;
|
|
382
|
+
for (const w of mine)
|
|
383
|
+
if (theirs.has(w))
|
|
384
|
+
both += 1;
|
|
385
|
+
return mine.size + theirs.size > 0 && both / (mine.size + theirs.size - both) >= 0.85;
|
|
386
|
+
});
|
|
387
|
+
if (hit) {
|
|
388
|
+
out.push({ rule: r.id, why: `already tested in ${o.journey} (${hit.id}, ${hit.testedBy.join(', ')})` });
|
|
389
|
+
break;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
return out;
|
|
394
|
+
}
|
|
@@ -9,8 +9,9 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
11
11
|
import { dirname, join } from 'node:path';
|
|
12
|
-
import { planSuite } from './suite-planner.js';
|
|
13
|
-
import {
|
|
12
|
+
import { planBySections, planSuite, testedTags } from './suite-planner.js';
|
|
13
|
+
import { loadConfig } from '../project/config.js';
|
|
14
|
+
import { journeyDir } from '../paths.js';
|
|
14
15
|
/** The line every generated file carries, so cleanup never touches a hand-written one. */
|
|
15
16
|
export const GEN_MARKER = '@generated by testforge';
|
|
16
17
|
/**
|
|
@@ -48,9 +49,9 @@ export function collisionWarning(testDir, id, ext) {
|
|
|
48
49
|
return `an existing test with different naming already covers this journey: ${clashes.join(', ')} — new file may duplicate it`;
|
|
49
50
|
}
|
|
50
51
|
/** Plan cache path for a journey, organised per area:
|
|
51
|
-
*
|
|
52
|
+
* testforge/memory/<area>/<id>/plan.json. */
|
|
52
53
|
export function planPath(journey) {
|
|
53
|
-
return join(
|
|
54
|
+
return join(journeyDir(journey), 'plan.json');
|
|
54
55
|
}
|
|
55
56
|
/**
|
|
56
57
|
* F9 — a journey doc that is still an unconfirmed stub. Deep-generating
|
|
@@ -146,7 +147,9 @@ export function cleanStaleFiles(testDir, journeyId, ext, flat, keepSlugs, dropSl
|
|
|
146
147
|
* actually matches — and it saves the planner AI call on a resume. When
|
|
147
148
|
* the doc changes, the hash changes and we re-plan.
|
|
148
149
|
*/
|
|
149
|
-
export function loadOrPlan(journey, docHash
|
|
150
|
+
export function loadOrPlan(journey, docHash, memory,
|
|
151
|
+
/** Rules not to plan, with why (another journey already tests them). */
|
|
152
|
+
preSkips = []) {
|
|
150
153
|
const p = planPath(journey);
|
|
151
154
|
if (existsSync(p)) {
|
|
152
155
|
try {
|
|
@@ -159,16 +162,120 @@ export function loadOrPlan(journey, docHash) {
|
|
|
159
162
|
// corrupt plan cache — re-plan
|
|
160
163
|
}
|
|
161
164
|
}
|
|
162
|
-
const
|
|
165
|
+
const skipped = [...preSkips];
|
|
166
|
+
const pre = new Set(preSkips.map((x) => x.rule));
|
|
167
|
+
// The doc's sections are the plan unless the project asks the AI to plan.
|
|
168
|
+
let files = memory && loadConfig().planner !== 'ai'
|
|
169
|
+
? planBySections(journey, memory, skipped)
|
|
170
|
+
: planSuite(journey, memory && pre.size ? { ...memory, rules: memory.rules.filter((r) => !pre.has(r.id)) } : memory, skipped);
|
|
171
|
+
if (memory) {
|
|
172
|
+
const gate = gatePlan(files, skipped, memory, journey.name);
|
|
173
|
+
files = gate.files;
|
|
174
|
+
try {
|
|
175
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
176
|
+
writeFileSync(join(dirname(p), 'plan-check.md'), gate.report, 'utf-8');
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
// the report is for people; the plan below is what runs
|
|
180
|
+
}
|
|
181
|
+
console.log(` ${journey.id}: ${gate.summary}`);
|
|
182
|
+
}
|
|
163
183
|
try {
|
|
164
184
|
mkdirSync(dirname(p), { recursive: true });
|
|
165
|
-
writeFileSync(p, JSON.stringify({ hash: docHash, files }, null, 2), 'utf-8');
|
|
185
|
+
writeFileSync(p, JSON.stringify({ hash: docHash, files, skipped }, null, 2), 'utf-8');
|
|
166
186
|
}
|
|
167
187
|
catch {
|
|
168
188
|
// best-effort cache
|
|
169
189
|
}
|
|
170
190
|
return files;
|
|
171
191
|
}
|
|
192
|
+
/** Rules the saved plan left out, with why. */
|
|
193
|
+
export function readPlanSkips(journey) {
|
|
194
|
+
try {
|
|
195
|
+
const c = JSON.parse(readFileSync(planPath(journey), 'utf-8'));
|
|
196
|
+
return Array.isArray(c.skipped) ? c.skipped : [];
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
return [];
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* The plan, checked before a line of test code is written: every file tests
|
|
204
|
+
* at least one rule that gets tests, no rule is tested twice, and anything not
|
|
205
|
+
* planned is listed with the reason. A file the gate drops costs nothing; the
|
|
206
|
+
* same file written, run and repaired cost most of a journey's AI budget.
|
|
207
|
+
*/
|
|
208
|
+
export function gatePlan(planned, skipped, memory, name = 'journey') {
|
|
209
|
+
const tags = testedTags();
|
|
210
|
+
const byId = new Map(memory.rules.map((r) => [r.id, r]));
|
|
211
|
+
const testable = memory.rules.filter((r) => r.tag && tags.includes(r.tag));
|
|
212
|
+
const owner = new Map();
|
|
213
|
+
const dropped = [];
|
|
214
|
+
const duplicates = [];
|
|
215
|
+
const files = [];
|
|
216
|
+
for (const f of planned) {
|
|
217
|
+
const rules = [];
|
|
218
|
+
for (const id of f.rules ?? []) {
|
|
219
|
+
const r = byId.get(id);
|
|
220
|
+
if (!r || !r.tag || !tags.includes(r.tag))
|
|
221
|
+
continue;
|
|
222
|
+
if (owner.has(id)) {
|
|
223
|
+
duplicates.push(`${id} (in ${owner.get(id)} and ${f.slug})`);
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
owner.set(id, f.slug);
|
|
227
|
+
rules.push(id);
|
|
228
|
+
}
|
|
229
|
+
if (rules.length)
|
|
230
|
+
files.push({ ...f, rules });
|
|
231
|
+
else
|
|
232
|
+
dropped.push(f.slug);
|
|
233
|
+
}
|
|
234
|
+
const skippedIds = new Set(skipped.map((s) => s.rule));
|
|
235
|
+
const missing = testable.filter((r) => !owner.has(r.id) && !skippedIds.has(r.id));
|
|
236
|
+
// A "must" (every [spec], and security, money, concurrency and error-boundary
|
|
237
|
+
// [baseline]) may be left out only with a reason the planner gave.
|
|
238
|
+
const mustMissing = missing.filter((r) => r.level === 'must');
|
|
239
|
+
const line = (id) => {
|
|
240
|
+
const r = byId.get(id);
|
|
241
|
+
return r ? `${id} [${r.tag}] ${r.section} — ${r.text.split('\n')[0].replace(/\(source:[^)]*\)/g, '').trim().slice(0, 140)}` : id;
|
|
242
|
+
};
|
|
243
|
+
const report = [
|
|
244
|
+
`# Plan check — ${name}`,
|
|
245
|
+
'',
|
|
246
|
+
`Rules that get tests (${tags.join(', ')}): ${testable.length}. Planned: ${owner.size}. Skipped with a reason: ${skippedIds.size}. Not planned: ${missing.length}.`,
|
|
247
|
+
'',
|
|
248
|
+
'## Files',
|
|
249
|
+
...files.map((f) => `- **${f.slug}** — ${f.covers}\n ${(f.rules ?? []).join(', ')}`),
|
|
250
|
+
...(skipped.length ? ['', '## Skipped by the planner', ...skipped.map((s) => `- ${line(s.rule)}\n why: ${s.why || '(no reason given)'}`)] : []),
|
|
251
|
+
...(mustMissing.length ? ['', '## MUST rules not planned', ...mustMissing.map((r) => `- ${line(r.id)}`)] : []),
|
|
252
|
+
...(missing.length > mustMissing.length ? ['', '## Not planned (no file, no reason)', ...missing.filter((r) => r.level !== 'must').map((r) => `- ${line(r.id)}`)] : []),
|
|
253
|
+
...(duplicates.length ? ['', '## Planned twice (kept in the first file)', ...duplicates.map((d) => `- ${d}`)] : []),
|
|
254
|
+
...(dropped.length ? ['', '## Files dropped (no rule that gets tests)', ...dropped.map((d) => `- ${d}`)] : []),
|
|
255
|
+
].join('\n');
|
|
256
|
+
const summary = `plan check: ${files.length} file(s), ${owner.size}/${testable.length} rules planned, ${skippedIds.size} skipped${missing.length ? `, ${missing.length} NOT planned${mustMissing.length ? ` (${mustMissing.length} MUST)` : ''}` : ''}${dropped.length ? `, ${dropped.length} file(s) dropped` : ''}${duplicates.length ? `, ${duplicates.length} duplicate(s) removed` : ''}`;
|
|
257
|
+
return { files, report, summary };
|
|
258
|
+
}
|
|
259
|
+
/** The saved plan's files, whatever doc version it was made for. */
|
|
260
|
+
export function readPlan(journey) {
|
|
261
|
+
try {
|
|
262
|
+
const c = JSON.parse(readFileSync(planPath(journey), 'utf-8'));
|
|
263
|
+
return Array.isArray(c.files) ? c.files : undefined;
|
|
264
|
+
}
|
|
265
|
+
catch {
|
|
266
|
+
return undefined;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/** Save a plan as the one for this version of the doc. */
|
|
270
|
+
export function writePlan(journey, docHash, files, skipped = readPlanSkips(journey)) {
|
|
271
|
+
try {
|
|
272
|
+
mkdirSync(dirname(planPath(journey)), { recursive: true });
|
|
273
|
+
writeFileSync(planPath(journey), JSON.stringify({ hash: docHash, files, skipped }, null, 2), 'utf-8');
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
// best-effort cache
|
|
277
|
+
}
|
|
278
|
+
}
|
|
172
279
|
/** Does this doc file / journey match a --journey selector? */
|
|
173
280
|
export function matchesJourney(file, journeyId, sel) {
|
|
174
281
|
const norm = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, '-');
|
|
@@ -10,15 +10,15 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
12
12
|
import { basename, extname, join } from 'node:path';
|
|
13
|
-
import {
|
|
13
|
+
import { RUNS_DIR } from '../paths.js';
|
|
14
14
|
import { findMarkdownFiles } from '../find-files.js';
|
|
15
15
|
import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
|
|
16
16
|
import { loadConfig, testDirForType } from '../project/config.js';
|
|
17
17
|
import { matchesJourney } from './plan-cache.js';
|
|
18
18
|
import { claimedDefects, verifyRun } from './verify.js';
|
|
19
19
|
import { groupClaims, oracleClaim, reviewDefects } from './defect-review.js';
|
|
20
|
-
const RESULTS = join(
|
|
21
|
-
const REVIEWS = join(
|
|
20
|
+
const RESULTS = join(RUNS_DIR, 'verify-results');
|
|
21
|
+
const REVIEWS = join(RUNS_DIR, 'defect-review');
|
|
22
22
|
export function saveVerifyResults(journeyId, failures, passes) {
|
|
23
23
|
mkdirSync(RESULTS, { recursive: true });
|
|
24
24
|
const saved = { at: new Date().toISOString(), failures, passes };
|