@activate-core/test-forge 1.3.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +70 -29
- package/dist/core/docs/excerpts.js +294 -0
- package/dist/core/docs/grounding.js +55 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/test-quality.js +295 -0
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +267 -117
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/complete-file.js +77 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +140 -0
- package/dist/core/generate/harness.js +18 -34
- package/dist/core/generate/journey-memory.js +394 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +240 -82
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +118 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +146 -0
- package/dist/core/run/run-tests.js +26 -2
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { readFileSync, existsSync, readdirSync } from 'node:fs';
|
|
18
18
|
import { join } from 'node:path';
|
|
19
|
-
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
20
|
-
import {
|
|
19
|
+
import { loadConfig, splitCommand, withNoTools, withReadOnlyTools } from '../project/config.js';
|
|
20
|
+
import { ruleIndex } from './journey-memory.js';
|
|
21
|
+
import { MEMORY_DIR } from '../paths.js';
|
|
21
22
|
import { runAiSync } from './ai-usage.js';
|
|
22
23
|
const MAP_PATH = 'testforge.md';
|
|
23
24
|
/**
|
|
@@ -30,13 +31,12 @@ const MAP_PATH = 'testforge.md';
|
|
|
30
31
|
export function siblingJourneys(journey, docsDir = 'docs') {
|
|
31
32
|
const lines = [];
|
|
32
33
|
const planned = new Set();
|
|
33
|
-
const dir = join(
|
|
34
|
-
for (const
|
|
35
|
-
|
|
36
|
-
if (!f.endsWith('.json') || id === journey.id)
|
|
34
|
+
const dir = join(MEMORY_DIR, journey.area);
|
|
35
|
+
for (const id of existsSync(dir) ? readdirSync(dir) : []) {
|
|
36
|
+
if (id === journey.id || !existsSync(join(dir, id, 'plan.json')))
|
|
37
37
|
continue;
|
|
38
38
|
try {
|
|
39
|
-
const plan = JSON.parse(readFileSync(join(dir,
|
|
39
|
+
const plan = JSON.parse(readFileSync(join(dir, id, 'plan.json'), 'utf-8'));
|
|
40
40
|
lines.push(`- ${id} (planned):`, ...(plan.files ?? []).map((x) => ` - ${x.title ?? ''}: ${x.covers ?? ''}`));
|
|
41
41
|
planned.add(id);
|
|
42
42
|
}
|
|
@@ -56,7 +56,7 @@ export function siblingJourneys(journey, docsDir = 'docs') {
|
|
|
56
56
|
return lines.join('\n');
|
|
57
57
|
}
|
|
58
58
|
/** Build the planning prompt for one journey. */
|
|
59
|
-
export function buildPlanPrompt(journey, map, siblings = '') {
|
|
59
|
+
export function buildPlanPrompt(journey, map, siblings = '', memory) {
|
|
60
60
|
return [
|
|
61
61
|
`You are a senior test architect. Plan how to split ONE user journey`,
|
|
62
62
|
`into a set of FOCUSED test files — one concern per file — the way a`,
|
|
@@ -64,11 +64,19 @@ export function buildPlanPrompt(journey, map, siblings = '') {
|
|
|
64
64
|
`success, failures, discount, 3D-Secure, zero-amount, price summary,`,
|
|
65
65
|
`and so on). Do NOT plan a single monolithic file.`,
|
|
66
66
|
``,
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
67
|
+
...(memory
|
|
68
|
+
? [
|
|
69
|
+
`You get the journey as its list of RULES (one line each, with an id)`,
|
|
70
|
+
`and its controls, not the whole document, and you have no tools: plan`,
|
|
71
|
+
`from this list. Every file names the rule ids it tests.`,
|
|
72
|
+
]
|
|
73
|
+
: [
|
|
74
|
+
`You are inside the project's repository with read-only tools (Read,`,
|
|
75
|
+
`Grep, Glob, LS). READ the real source for this journey — routes,`,
|
|
76
|
+
`controllers, services, models — and combine it with the journey doc`,
|
|
77
|
+
`to decide the split. Every file must map to a real behaviour that`,
|
|
78
|
+
`exists in the code or is explicitly described in the doc.`,
|
|
79
|
+
]),
|
|
72
80
|
``,
|
|
73
81
|
`Rules:`,
|
|
74
82
|
`- Let the doc's sections and the real endpoints drive the split.`,
|
|
@@ -103,7 +111,18 @@ export function buildPlanPrompt(journey, map, siblings = '') {
|
|
|
103
111
|
: []),
|
|
104
112
|
`Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
|
|
105
113
|
` [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,
|
|
106
|
-
` "covers": "<one or two sentences: exactly what this file tests>" }]`,
|
|
114
|
+
` "covers": "<one or two sentences: exactly what this file tests>"${memory ? ',\n "rules": ["R1", "R4"]' : ''} }]`,
|
|
115
|
+
...(memory
|
|
116
|
+
? [
|
|
117
|
+
`Every rule id below goes in exactly ONE file's "rules". A rule that`,
|
|
118
|
+
`another journey owns, or that cannot be checked from this journey, is`,
|
|
119
|
+
`left out and listed instead as { "skip": "R7", "why": "<short reason>" }`,
|
|
120
|
+
`in the same array. A rule that is neither planned nor skipped is`,
|
|
121
|
+
`reported as missing. Every [spec] rule and every rule marked "(must"`,
|
|
122
|
+
`(security, money and data, concurrency, error boundaries) is planned`,
|
|
123
|
+
`unless another journey owns it.`,
|
|
124
|
+
]
|
|
125
|
+
: []),
|
|
107
126
|
`slug: lowercase kebab-case, no path, no extension — and SHORT: name the`,
|
|
108
127
|
` one concern in 2–4 words (4 max), the way a human names a file. Do NOT`,
|
|
109
128
|
` restate the journey name and do NOT pile on qualifiers.`,
|
|
@@ -113,13 +132,25 @@ export function buildPlanPrompt(journey, map, siblings = '') {
|
|
|
113
132
|
`title: human-readable and specific, e.g. "Payment — Amount validation".`,
|
|
114
133
|
`The very first character of your response must be "[".`,
|
|
115
134
|
``,
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
135
|
+
...(memory
|
|
136
|
+
? [
|
|
137
|
+
`--- JOURNEY: ${journey.name} ---`,
|
|
138
|
+
memory.header,
|
|
139
|
+
``,
|
|
140
|
+
`CONTROLS:`,
|
|
141
|
+
...Object.entries(memory.controls).map(([id, c]) => `${id}: ${c}`),
|
|
142
|
+
``,
|
|
143
|
+
`RULES TO PLAN:`,
|
|
144
|
+
ruleIndex(memory, testedTags()),
|
|
145
|
+
]
|
|
146
|
+
: [`--- PROJECT MAP (data layer) ---`, map, ``, `--- JOURNEY ---`, journey.rawMarkdown]),
|
|
121
147
|
].join('\n');
|
|
122
148
|
}
|
|
149
|
+
/** Which rule tags get tests of their own; the rest only explain the app. */
|
|
150
|
+
export function testedTags() {
|
|
151
|
+
const t = loadConfig().testTags;
|
|
152
|
+
return Array.isArray(t) && t.length ? t : ['spec', 'baseline'];
|
|
153
|
+
}
|
|
123
154
|
/**
|
|
124
155
|
* Pull a JSON array out of the AI's reply, tolerating stray prose or
|
|
125
156
|
* code fences around it. Returns [] if nothing parseable is found.
|
|
@@ -154,10 +185,11 @@ function safeSlug(raw) {
|
|
|
154
185
|
* produces nothing usable, so generation never silently yields zero
|
|
155
186
|
* files.
|
|
156
187
|
*/
|
|
157
|
-
export function planSuite(journey) {
|
|
158
|
-
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8')
|
|
159
|
-
const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey));
|
|
160
|
-
const
|
|
188
|
+
export function planSuite(journey, memory, skipped = []) {
|
|
189
|
+
const map = memory || !existsSync(MAP_PATH) ? '' : readFileSync(MAP_PATH, 'utf-8');
|
|
190
|
+
const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey), memory);
|
|
191
|
+
const cmd = splitCommand(loadConfig().aiCommand);
|
|
192
|
+
const { program, args } = memory ? withNoTools(cmd) : withReadOnlyTools(cmd);
|
|
161
193
|
let raw = '';
|
|
162
194
|
try {
|
|
163
195
|
raw = runAiSync(program, args, {
|
|
@@ -175,6 +207,10 @@ export function planSuite(journey) {
|
|
|
175
207
|
if (!entry || typeof entry !== 'object')
|
|
176
208
|
continue;
|
|
177
209
|
const e = entry;
|
|
210
|
+
if (typeof e.skip === 'string') {
|
|
211
|
+
skipped.push({ rule: e.skip, why: String(e.why ?? '').trim() });
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
178
214
|
const slug = safeSlug(e.slug ?? e.title);
|
|
179
215
|
if (!slug || seen.has(slug))
|
|
180
216
|
continue;
|
|
@@ -183,6 +219,7 @@ export function planSuite(journey) {
|
|
|
183
219
|
slug,
|
|
184
220
|
title: String(e.title ?? journey.name).trim() || journey.name,
|
|
185
221
|
covers: String(e.covers ?? '').trim(),
|
|
222
|
+
...(memory && Array.isArray(e.rules) ? { rules: e.rules.map(String) } : {}),
|
|
186
223
|
});
|
|
187
224
|
}
|
|
188
225
|
if (files.length === 0) {
|
|
@@ -196,3 +233,71 @@ export function planSuite(journey) {
|
|
|
196
233
|
}
|
|
197
234
|
return files;
|
|
198
235
|
}
|
|
236
|
+
/** Sections larger than this are split by their sub-sections, then in chunks. */
|
|
237
|
+
const MAX_RULES_PER_FILE = 10;
|
|
238
|
+
/**
|
|
239
|
+
* The plan without the AI: the doc is already organised one concern per
|
|
240
|
+
* numbered section, so each section becomes one file and gets that section's
|
|
241
|
+
* rules. A section with more rules than one file should hold is split by its
|
|
242
|
+
* sub-sections, then in chunks. Rules another journey already tests come in
|
|
243
|
+
* as skips. Costs nothing.
|
|
244
|
+
*/
|
|
245
|
+
export function planBySections(journey, memory, skipped = []) {
|
|
246
|
+
const tags = testedTags();
|
|
247
|
+
const skip = new Set(skipped.map((x) => x.rule));
|
|
248
|
+
const rules = memory.rules.filter((r) => r.tag && tags.includes(r.tag) && !skip.has(r.id));
|
|
249
|
+
const byKey = (key, rs) => {
|
|
250
|
+
const m = new Map();
|
|
251
|
+
for (const r of rs)
|
|
252
|
+
m.set(key(r), [...(m.get(key(r)) ?? []), r]);
|
|
253
|
+
return m;
|
|
254
|
+
};
|
|
255
|
+
const files = [];
|
|
256
|
+
const used = new Set();
|
|
257
|
+
const STOP = new Set(['a', 'an', 'the', 'on', 'of', 'and', 'or', 'for', 'to', 'in', 'with', 'from', 'at', 'by', 'is', 'its']);
|
|
258
|
+
const slugFor = (title) => {
|
|
259
|
+
const part = / part (\d+)$/.exec(title)?.[1];
|
|
260
|
+
const words = title.replace(/^§[\d.]+\s*/, '').replace(/ part \d+$/, '').toLowerCase().split(/[^a-z0-9]+/).filter((w) => w && !STOP.has(w));
|
|
261
|
+
const base = part ? `${words.slice(0, 3).join('-')}-${part}` : safeSlug(words.join('-')) || 'rules';
|
|
262
|
+
let slug = base;
|
|
263
|
+
for (let n = 2; used.has(slug); n++)
|
|
264
|
+
slug = `${base.split('-').slice(0, 3).join('-')}-${n}`;
|
|
265
|
+
used.add(slug);
|
|
266
|
+
return slug;
|
|
267
|
+
};
|
|
268
|
+
const push = (title, rs) => {
|
|
269
|
+
const name = title.replace(/^§[\d.]+\s*/, '');
|
|
270
|
+
files.push({ slug: slugFor(title), title: `${journey.name} — ${name}`, covers: `The ${rs.length} rule(s) of ${title}.`, rules: rs.map((r) => r.id) });
|
|
271
|
+
};
|
|
272
|
+
for (const [group, rs] of byKey((r) => r.group || r.section, rules)) {
|
|
273
|
+
if (rs.length <= MAX_RULES_PER_FILE) {
|
|
274
|
+
push(group, rs);
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
// Too many for one file: its sub-sections, small ones kept together, big ones in parts.
|
|
278
|
+
let titles = [];
|
|
279
|
+
let acc = [];
|
|
280
|
+
const flush = () => {
|
|
281
|
+
if (acc.length)
|
|
282
|
+
push(titles.join(' & '), acc);
|
|
283
|
+
titles = [];
|
|
284
|
+
acc = [];
|
|
285
|
+
};
|
|
286
|
+
for (const [section, part] of byKey((r) => r.section, rs)) {
|
|
287
|
+
if (part.length > MAX_RULES_PER_FILE) {
|
|
288
|
+
flush();
|
|
289
|
+
const parts = Math.ceil(part.length / MAX_RULES_PER_FILE);
|
|
290
|
+
const size = Math.ceil(part.length / parts);
|
|
291
|
+
for (let i = 0; i < part.length; i += size)
|
|
292
|
+
push(`${section} part ${i / size + 1}`, part.slice(i, i + size));
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
if (acc.length + part.length > MAX_RULES_PER_FILE)
|
|
296
|
+
flush();
|
|
297
|
+
titles.push(section.replace(/^§[\d.]+\s*/, ''));
|
|
298
|
+
acc.push(...part);
|
|
299
|
+
}
|
|
300
|
+
flush();
|
|
301
|
+
}
|
|
302
|
+
return files;
|
|
303
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — single tests inside a generated file.
|
|
3
|
+
*
|
|
4
|
+
* Each generated test names the rule it checks in an oracle comment
|
|
5
|
+
* ("// oracle: R12 [spec] …"). That is how a change to one rule reaches only
|
|
6
|
+
* its tests: they are found, replaced, renamed or removed in place, and the
|
|
7
|
+
* rest of the file is never read by the AI or rewritten.
|
|
8
|
+
*/
|
|
9
|
+
import { matchParen } from '../gates/validate.js';
|
|
10
|
+
const TEST_CALL = /(^|[^\w.])(?:test|it)(?:\.(?:only|fixme|skip|slow|fail))?\s*\(\s*(['"`])((?:(?!\2).)*)\2/gm;
|
|
11
|
+
const RULE_ID = /\boracle:\s*((?:R\d+[\s,/&]*(?:and\s+)?)+)/g;
|
|
12
|
+
/** Every test(…) call in the file, describe blocks and hooks excluded. */
|
|
13
|
+
export function findTestBlocks(content) {
|
|
14
|
+
const out = [];
|
|
15
|
+
for (const m of content.matchAll(TEST_CALL)) {
|
|
16
|
+
const callAt = m.index + m[1].length;
|
|
17
|
+
const open = content.indexOf('(', callAt);
|
|
18
|
+
const close = matchParen(content, open);
|
|
19
|
+
if (close === -1)
|
|
20
|
+
continue;
|
|
21
|
+
let end = close + 1;
|
|
22
|
+
if (content[end] === ';')
|
|
23
|
+
end += 1;
|
|
24
|
+
// Pull in the comment lines right above the call (oracle, defect marks).
|
|
25
|
+
let start = content.lastIndexOf('\n', callAt - 1) + 1;
|
|
26
|
+
for (;;) {
|
|
27
|
+
const prevEnd = start - 1;
|
|
28
|
+
if (prevEnd <= 0)
|
|
29
|
+
break;
|
|
30
|
+
const prevStart = content.lastIndexOf('\n', prevEnd - 1) + 1;
|
|
31
|
+
if (!/^\s*\/\//.test(content.slice(prevStart, prevEnd)))
|
|
32
|
+
break;
|
|
33
|
+
start = prevStart;
|
|
34
|
+
}
|
|
35
|
+
const text = content.slice(start, end);
|
|
36
|
+
const rules = [...text.matchAll(RULE_ID)].flatMap((r) => r[1].match(/R\d+/g) ?? []);
|
|
37
|
+
out.push({ start, end, title: m[3], rules: [...new Set(rules)] });
|
|
38
|
+
}
|
|
39
|
+
return out.filter((b) => !out.some((o) => o !== b && o.start <= b.start && o.end >= b.end && o.end - o.start > b.end - b.start));
|
|
40
|
+
}
|
|
41
|
+
/** Replace (or, with '', remove) the given blocks; later ones first so offsets stay valid. */
|
|
42
|
+
export function spliceBlocks(content, edits) {
|
|
43
|
+
let out = content;
|
|
44
|
+
for (const { block, text } of [...edits].sort((a, b) => b.block.start - a.block.start)) {
|
|
45
|
+
if (text) {
|
|
46
|
+
out = out.slice(0, block.start) + text + out.slice(block.end);
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
// A removed test takes its own line and the blank line that separated it.
|
|
50
|
+
out = out.slice(0, block.start).replace(/\n[ \t]*\n[ \t]*$/, '\n') + out.slice(block.end).replace(/^[ \t]*\n/, '');
|
|
51
|
+
}
|
|
52
|
+
return out;
|
|
53
|
+
}
|
|
54
|
+
/** Add tests inside the file's outermost describe, before it closes; at the end when there is none. */
|
|
55
|
+
export function appendTests(content, tests) {
|
|
56
|
+
const d = /\btest\.describe(?:\.\w+)?\s*\(/.exec(content);
|
|
57
|
+
if (d) {
|
|
58
|
+
const close = matchParen(content, content.indexOf('(', d.index));
|
|
59
|
+
if (close !== -1) {
|
|
60
|
+
// The describe's body closes with "}" just before ")".
|
|
61
|
+
const brace = content.lastIndexOf('}', close);
|
|
62
|
+
if (brace > d.index)
|
|
63
|
+
return `${content.slice(0, brace).trimEnd()}\n\n${indent(tests.trim(), ' ')}\n${content.slice(brace)}`;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return `${content.trimEnd()}\n\n${tests.trim()}\n`;
|
|
67
|
+
}
|
|
68
|
+
const indent = (t, pad) => t.split('\n').map((l) => (l ? pad + l : l)).join('\n');
|
|
69
|
+
/** Literal replacement of each old text with the new one, only inside the given blocks. */
|
|
70
|
+
export function renameInBlocks(content, blocks, pairs) {
|
|
71
|
+
let replaced = 0;
|
|
72
|
+
const edits = blocks.map((block) => {
|
|
73
|
+
let text = content.slice(block.start, block.end);
|
|
74
|
+
for (const [from, to] of pairs) {
|
|
75
|
+
const n = text.split(from).length - 1;
|
|
76
|
+
if (n) {
|
|
77
|
+
text = text.split(from).join(to);
|
|
78
|
+
replaced += n;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return { block, text };
|
|
82
|
+
});
|
|
83
|
+
return { content: spliceBlocks(content, edits), replaced };
|
|
84
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — bring an existing suite up to a changed journey doc, in place.
|
|
3
|
+
*
|
|
4
|
+
* The doc diff says which rules were renamed, reworded, added or removed. A
|
|
5
|
+
* rename is a find-and-replace inside that rule's tests; a removed rule's
|
|
6
|
+
* tests are deleted; a reworded or added rule gets its tests written by one
|
|
7
|
+
* small AI call that sees only those rules and the file's top. Nothing else in
|
|
8
|
+
* the file is sent or rewritten. Rules in a section no file covers yet become
|
|
9
|
+
* new files in the plan, which the normal per-file step writes.
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { journeySlice, factsSlice } from './journey-memory.js';
|
|
13
|
+
import { appendTests, findTestBlocks, renameInBlocks, spliceBlocks } from './test-blocks.js';
|
|
14
|
+
import { writeTestsWithAI } from './ai-generator.js';
|
|
15
|
+
const blocksFor = (content, ids) => findTestBlocks(content).filter((b) => b.rules.some((r) => ids.includes(r)));
|
|
16
|
+
const slugOf = (section) => section.replace(/^§[\d.]+\s*/, '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').split('-').slice(0, 4).join('-') || 'new-rules';
|
|
17
|
+
export async function updateJourney(ctx) {
|
|
18
|
+
const { memory, diff } = ctx;
|
|
19
|
+
const notes = [];
|
|
20
|
+
const rule = new Map(memory.rules.map((r) => [r.id, r]));
|
|
21
|
+
const removed = new Set(diff.removed);
|
|
22
|
+
const testedId = (id) => {
|
|
23
|
+
const r = rule.get(id);
|
|
24
|
+
return !!r && !!r.tag && ctx.tested.includes(r.tag);
|
|
25
|
+
};
|
|
26
|
+
const plan = ctx.plan.map((f) => ({ ...f, rules: (f.rules ?? []).filter((id) => !removed.has(id)) }));
|
|
27
|
+
// Each added rule joins the file that already tests its section; a section
|
|
28
|
+
// no file covers becomes a new file the normal step writes.
|
|
29
|
+
const added = diff.added.filter(testedId);
|
|
30
|
+
const toFile = new Map();
|
|
31
|
+
const orphans = new Map();
|
|
32
|
+
for (const id of added) {
|
|
33
|
+
const section = rule.get(id).section;
|
|
34
|
+
const home = plan.find((f) => f.rules.some((r) => rule.get(r)?.section === section));
|
|
35
|
+
if (home)
|
|
36
|
+
toFile.set(home, [...(toFile.get(home) ?? []), id]);
|
|
37
|
+
else
|
|
38
|
+
orphans.set(section, [...(orphans.get(section) ?? []), id]);
|
|
39
|
+
}
|
|
40
|
+
for (const f of plan) {
|
|
41
|
+
const path = ctx.pathOf(f);
|
|
42
|
+
if (!existsSync(path))
|
|
43
|
+
continue;
|
|
44
|
+
const before = readFileSync(path, 'utf-8');
|
|
45
|
+
let content = before;
|
|
46
|
+
const mine = (ids) => ids.filter((id) => ctx.plan.find((p) => p.slug === f.slug)?.rules?.includes(id));
|
|
47
|
+
// Renames: literal replacement inside the renamed rule's tests only. No AI.
|
|
48
|
+
// A file written before tests named their rule gets the rename in every test.
|
|
49
|
+
const legacy = !findTestBlocks(content).some((b) => b.rules.length);
|
|
50
|
+
for (const r of diff.renamed.filter((x) => legacy || mine([x.id]).length)) {
|
|
51
|
+
const res = renameInBlocks(content, legacy ? findTestBlocks(content) : blocksFor(content, [r.id]), r.pairs);
|
|
52
|
+
content = res.content;
|
|
53
|
+
if (res.replaced)
|
|
54
|
+
notes.push(`${f.slug}: ${r.id} renamed ${r.pairs.map(([a, b]) => `"${a}" → "${b}"`).join(', ')} (${res.replaced} place(s))`);
|
|
55
|
+
}
|
|
56
|
+
// Removed rules: their tests go, unless a test also checks a rule that stays.
|
|
57
|
+
const gone = findTestBlocks(content).filter((b) => b.rules.length && b.rules.every((id) => removed.has(id)));
|
|
58
|
+
if (gone.length) {
|
|
59
|
+
content = spliceBlocks(content, gone.map((block) => ({ block, text: '' })));
|
|
60
|
+
notes.push(`${f.slug}: removed ${gone.length} test(s) of rules no longer in the doc`);
|
|
61
|
+
}
|
|
62
|
+
// Reworded rules and added rules: one small AI call for just those.
|
|
63
|
+
const changed = mine(diff.changed).filter(testedId);
|
|
64
|
+
const extra = toFile.get(f) ?? [];
|
|
65
|
+
const ids = [...changed, ...extra];
|
|
66
|
+
if (ids.length) {
|
|
67
|
+
const old = blocksFor(content, changed);
|
|
68
|
+
const firstTest = findTestBlocks(content)[0];
|
|
69
|
+
const head = content.slice(0, firstTest ? firstTest.start : Math.min(content.length, 4000)).slice(0, 4000);
|
|
70
|
+
const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(content)?.[2] ?? f.title;
|
|
71
|
+
const slice = journeySlice(memory, ids);
|
|
72
|
+
const mem = { journey: slice.text, helpers: ctx.helpers, rules: ids };
|
|
73
|
+
const oldText = old.map((b) => content.slice(b.start, b.end)).join('\n\n');
|
|
74
|
+
const base = old.length ? spliceBlocks(content, old.map((block) => ({ block, text: '' }))) : content;
|
|
75
|
+
let refusal;
|
|
76
|
+
let next = null;
|
|
77
|
+
// Only the new tests are judged: a problem the file already had is not theirs.
|
|
78
|
+
const already = ctx.check(base, path);
|
|
79
|
+
for (let attempt = 0; attempt < 2 && next === null; attempt++) {
|
|
80
|
+
let tests = '';
|
|
81
|
+
try {
|
|
82
|
+
tests = await writeTestsWithAI(ctx.journey, path, head, title, oldText, ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), mem, refusal);
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
refusal = undefined;
|
|
86
|
+
notes.push(`${f.slug}: the AI call for ${ids.join(', ')} failed — ${e.message.slice(0, 120)}`);
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
const candidate = appendTests(base, tests);
|
|
90
|
+
const why = findTestBlocks(tests).length ? ctx.check(candidate, path) : 'it contained no test(...) block';
|
|
91
|
+
refusal = why && why !== already ? why : undefined;
|
|
92
|
+
if (!refusal)
|
|
93
|
+
next = candidate;
|
|
94
|
+
}
|
|
95
|
+
if (next !== null) {
|
|
96
|
+
content = next;
|
|
97
|
+
if (changed.length)
|
|
98
|
+
notes.push(`${f.slug}: rewrote the tests of ${changed.join(', ')} (rule changed)`);
|
|
99
|
+
if (extra.length)
|
|
100
|
+
notes.push(`${f.slug}: added tests for ${extra.join(', ')}`);
|
|
101
|
+
f.rules = [...new Set([...f.rules, ...extra])];
|
|
102
|
+
}
|
|
103
|
+
else if (refusal) {
|
|
104
|
+
notes.push(`${f.slug}: tests for ${ids.join(', ')} were refused (${refusal}) — the file keeps its old tests`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (content !== before)
|
|
108
|
+
writeFileSync(path, content, 'utf-8');
|
|
109
|
+
}
|
|
110
|
+
for (const [section, ids] of orphans) {
|
|
111
|
+
let slug = slugOf(section);
|
|
112
|
+
while (plan.some((f) => f.slug === slug))
|
|
113
|
+
slug = `${slug}-2`;
|
|
114
|
+
plan.push({ slug, title: `${ctx.journey.name} — ${section.replace(/^§[\d.]+\s*/, '')}`, covers: `The rules of ${section}.`, rules: ids });
|
|
115
|
+
notes.push(`${slug}: new file for ${ids.join(', ')} (new section ${section})`);
|
|
116
|
+
}
|
|
117
|
+
return { plan: plan.filter((f) => f.rules.length), notes };
|
|
118
|
+
}
|
package/dist/core/paths.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import { join } from 'node:path';
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
3
|
/**
|
|
4
4
|
* Single source of truth for TestForge's per-project state directory and the
|
|
5
5
|
* files/sub-dirs inside it. Everything the tool persists between runs lives
|
|
@@ -9,26 +9,103 @@ import { join } from 'node:path';
|
|
|
9
9
|
* Rename in ONE place and every command — setup, discover, generate, update,
|
|
10
10
|
* coverage — follows automatically, so no path can drift out of sync.
|
|
11
11
|
*/
|
|
12
|
-
export const STATE_DIR = '
|
|
13
|
-
/**
|
|
14
|
-
* pre-migration folder if one is lying around. Not written to. */
|
|
12
|
+
export const STATE_DIR = 'testforge';
|
|
13
|
+
/** Earlier state-dir names, still skipped by source hashing and moved by migrateState(). */
|
|
15
14
|
export const LEGACY_STATE_DIR = '.testforge';
|
|
15
|
+
export const OLD_STATE_DIR = 'test-artifacts';
|
|
16
16
|
export const CACHE_PATH = join(STATE_DIR, 'cache.json');
|
|
17
17
|
export const SOURCE_PATH = join(STATE_DIR, 'source.json');
|
|
18
18
|
export const COVERAGE_PATH = join(STATE_DIR, 'coverage.json');
|
|
19
|
-
/**
|
|
20
|
-
export const
|
|
19
|
+
/** Everything TestForge remembers about each journey: <MEMORY_DIR>/<area>/<id>/ */
|
|
20
|
+
export const MEMORY_DIR = join(STATE_DIR, 'memory');
|
|
21
|
+
/** One journey's folder: rules.json, plan.json, plan-check.md, facts.json. */
|
|
22
|
+
export function journeyDir(journey) {
|
|
23
|
+
return join(MEMORY_DIR, (journey.area || 'misc').replace(/[^a-z0-9-]/gi, '-'), journey.id);
|
|
24
|
+
}
|
|
25
|
+
/** The references read (Notion pages, ticket bundles) and the comparison with the map. */
|
|
26
|
+
export const REFERENCE_DIR = join(STATE_DIR, 'reference');
|
|
27
|
+
/** What one run leaves behind, kept on this machine only. */
|
|
28
|
+
export const RUNS_DIR = join(STATE_DIR, 'runs');
|
|
21
29
|
/** Where Playwright writes its per-run output (failure reports, traces). */
|
|
22
|
-
export const RESULTS_DIR = join(
|
|
30
|
+
export const RESULTS_DIR = join(RUNS_DIR, 'playwright-results');
|
|
31
|
+
/**
|
|
32
|
+
* Move a project's state from the old layout (test-artifacts/, testforge.diff.md)
|
|
33
|
+
* into testforge/. Nothing is rebuilt and nothing already in the new place is
|
|
34
|
+
* overwritten. Safe to call on every command.
|
|
35
|
+
*/
|
|
36
|
+
export function migrateState() {
|
|
37
|
+
const moved = [];
|
|
38
|
+
const move = (from, to) => {
|
|
39
|
+
if (!existsSync(from) || existsSync(to))
|
|
40
|
+
return;
|
|
41
|
+
mkdirSync(dirname(to), { recursive: true });
|
|
42
|
+
renameSync(from, to);
|
|
43
|
+
moved.push(`${from} → ${to}`);
|
|
44
|
+
};
|
|
45
|
+
move('testforge.diff.md', join(REFERENCE_DIR, 'diff.md'));
|
|
46
|
+
const old = OLD_STATE_DIR;
|
|
47
|
+
if (!existsSync(old) || !statSync(old).isDirectory())
|
|
48
|
+
return moved;
|
|
49
|
+
for (const f of ['cache.json', 'source.json', 'coverage.json'])
|
|
50
|
+
move(join(old, f), join(STATE_DIR, f));
|
|
51
|
+
move(join(old, 'tickets-reference.md'), join(REFERENCE_DIR, 'tickets.md'));
|
|
52
|
+
for (const f of readdirSafe(join(old, 'reference')))
|
|
53
|
+
move(join(old, 'reference', f), join(REFERENCE_DIR, f));
|
|
54
|
+
for (const d of ['trace', 'playwright-results', 'verify-results', 'verify-progress', 'defect-review', 'run-data'])
|
|
55
|
+
move(join(old, d), join(RUNS_DIR, d));
|
|
56
|
+
// Per-journey files, from three folders into one per journey.
|
|
57
|
+
const perJourney = [
|
|
58
|
+
['memory', (n) => (n.endsWith('.json') ? n.slice(0, -5) : undefined), 'rules.json'],
|
|
59
|
+
['test-plans', (n) => (n.endsWith('.json') ? n.slice(0, -5) : undefined), 'plan.json'],
|
|
60
|
+
['test-plans', (n) => (n.endsWith('.gate.md') ? n.slice(0, -8) : undefined), 'plan-check.md'],
|
|
61
|
+
['grounding', (n) => (n.endsWith('.json') ? n.slice(0, -5) : undefined), 'facts.json'],
|
|
62
|
+
];
|
|
63
|
+
for (const [dir, idOf, file] of perJourney) {
|
|
64
|
+
for (const area of readdirSafe(join(old, dir))) {
|
|
65
|
+
for (const n of readdirSafe(join(old, dir, area))) {
|
|
66
|
+
const id = idOf(n);
|
|
67
|
+
if (id)
|
|
68
|
+
move(join(old, dir, area, n), join(MEMORY_DIR, area, id, file));
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
removeIfEmpty(old);
|
|
73
|
+
return moved;
|
|
74
|
+
}
|
|
75
|
+
function readdirSafe(dir) {
|
|
76
|
+
try {
|
|
77
|
+
return readdirSync(dir);
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
return [];
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Remove a directory tree that holds no files (only empty folders, or the old ignore file). */
|
|
84
|
+
function removeIfEmpty(dir) {
|
|
85
|
+
const files = (d) => readdirSafe(d).reduce((n, e) => {
|
|
86
|
+
const p = join(d, e);
|
|
87
|
+
return n + (statSync(p).isDirectory() ? files(p) : e === '.gitignore' ? 0 : 1);
|
|
88
|
+
}, 0);
|
|
89
|
+
if (existsSync(dir) && files(dir) === 0)
|
|
90
|
+
rmSync(dir, { recursive: true, force: true });
|
|
91
|
+
}
|
|
92
|
+
/** The ignore file TestForge used to write: everything local. Replaced when found. */
|
|
93
|
+
const OLD_IGNORE = '# testforge local state: regenerated as needed, so not committed.\n*\n!.gitignore\n!source.json\n';
|
|
94
|
+
const IGNORE = [
|
|
95
|
+
'# testforge state. Commit it: each journey\'s memory (rules, plan, code facts) and',
|
|
96
|
+
'# the references read cost AI to rebuild, and a teammate who pulls them starts',
|
|
97
|
+
'# from where you left off. Only runs/ (run output, AI logs) stays on this machine.',
|
|
98
|
+
'runs/',
|
|
99
|
+
'',
|
|
100
|
+
].join('\n');
|
|
23
101
|
/**
|
|
24
|
-
*
|
|
25
|
-
* .
|
|
26
|
-
* the source-of-truth choice is worth sharing with the team.
|
|
102
|
+
* The state dir is shared with the team through git, except what one run
|
|
103
|
+
* leaves behind. Written once; the old "ignore everything" file is upgraded.
|
|
27
104
|
*/
|
|
28
105
|
export function ensureStateDirIgnored() {
|
|
29
106
|
const file = join(STATE_DIR, '.gitignore');
|
|
30
|
-
if (existsSync(file))
|
|
107
|
+
if (existsSync(file) && readFileSync(file, 'utf-8') !== OLD_IGNORE)
|
|
31
108
|
return;
|
|
32
109
|
mkdirSync(STATE_DIR, { recursive: true });
|
|
33
|
-
writeFileSync(file,
|
|
110
|
+
writeFileSync(file, IGNORE, 'utf-8');
|
|
34
111
|
}
|
|
@@ -119,7 +119,7 @@ export function resolveFramework(config, area, type) {
|
|
|
119
119
|
return isApi ? 'jest' : 'playwright';
|
|
120
120
|
}
|
|
121
121
|
/** Is this command Claude's CLI (so Claude-specific flags apply)? */
|
|
122
|
-
function isClaude(program) {
|
|
122
|
+
export function isClaude(program) {
|
|
123
123
|
const base = program.split(/[\\/]/).pop() ?? program;
|
|
124
124
|
return base === 'claude';
|
|
125
125
|
}
|
|
@@ -152,9 +152,32 @@ export function withReadOnlyTools(cmd, relatedDirs = loadConfig().relatedDirs ??
|
|
|
152
152
|
return {
|
|
153
153
|
program: cmd.program,
|
|
154
154
|
// Space-separated allowlist; Write/Edit/Bash deliberately excluded.
|
|
155
|
-
args: [...cmd.args, '--allowedTools', 'Read Grep Glob LS', ...addDirArgs(relatedDirs)],
|
|
155
|
+
args: [...cmd.args, '--allowedTools', 'Read Grep Glob LS', ...LEAN, ...addDirArgs(relatedDirs)],
|
|
156
156
|
};
|
|
157
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* What a TestForge call never needs from the user's own Claude setup: their
|
|
160
|
+
* settings, hooks, CLAUDE.md memory, MCP servers and slash commands. Each of
|
|
161
|
+
* those rides along on every turn; measured, they were ~10k of the ~30k tokens
|
|
162
|
+
* a turn carried before the task began. Not for a phase that reads Notion:
|
|
163
|
+
* the connectors come in with the settings.
|
|
164
|
+
*/
|
|
165
|
+
const LEAN = ['--setting-sources', '', '--strict-mcp-config', '--disable-slash-commands'];
|
|
166
|
+
/** The whole system prompt of a no-tools call: the task itself comes on stdin. */
|
|
167
|
+
const WRITER_SYSTEM = 'You write automated test code for a QA tool. Follow the instructions in the message exactly and output only what it asks for.';
|
|
168
|
+
/**
|
|
169
|
+
* A call that has everything it needs in its prompt: no tools, no exploring,
|
|
170
|
+
* Claude Code's own system prompt replaced by one line. One turn in, one
|
|
171
|
+
* answer out. Measured: ~600 tokens of overhead per call instead of ~30k.
|
|
172
|
+
*/
|
|
173
|
+
export function withNoTools(cmd, system = WRITER_SYSTEM) {
|
|
174
|
+
if (!isClaude(cmd.program))
|
|
175
|
+
return cmd;
|
|
176
|
+
// Low effort: the thinking is in the prompt's rules and the slice it carries;
|
|
177
|
+
// what is asked for is code from them, and thinking tokens are output tokens.
|
|
178
|
+
const effort = cmd.args.includes('--effort') ? [] : ['--effort', loadConfig().aiEffort ?? 'low'];
|
|
179
|
+
return { program: cmd.program, args: [...cmd.args, '--system-prompt', system, '--tools', '', ...effort, ...LEAN] };
|
|
180
|
+
}
|
|
158
181
|
/**
|
|
159
182
|
* Read-only tools a headless exploring phase needs pre-approved. The model
|
|
160
183
|
* reaches for shell `grep`/`ls` as often as for the Grep tool; under
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
import { readFileSync, writeFileSync, existsSync, readdirSync, } from 'node:fs';
|
|
15
15
|
import { join } from 'node:path';
|
|
16
16
|
import { findMarkdownFiles } from '../find-files.js';
|
|
17
|
-
import { STATE_DIR, LEGACY_STATE_DIR } from '../paths.js';
|
|
17
|
+
import { STATE_DIR, LEGACY_STATE_DIR, OLD_STATE_DIR } from '../paths.js';
|
|
18
18
|
import { loadConfig } from './config.js';
|
|
19
19
|
/** Pull env var names from a chunk of text. */
|
|
20
20
|
export function extractEnvVars(text) {
|
|
@@ -53,6 +53,7 @@ const SKIP_DIRS = new Set([
|
|
|
53
53
|
'generated-tests',
|
|
54
54
|
STATE_DIR,
|
|
55
55
|
LEGACY_STATE_DIR,
|
|
56
|
+
OLD_STATE_DIR,
|
|
56
57
|
]);
|
|
57
58
|
const SOURCE_EXT = /\.(c|m)?[jt]sx?$/i;
|
|
58
59
|
/**
|
package/dist/core/run/ci.js
CHANGED
package/dist/core/run/clean.js
CHANGED
|
@@ -49,7 +49,7 @@ export async function runClean(opts) {
|
|
|
49
49
|
if (!opts.runId && opts.olderThanDays == null) {
|
|
50
50
|
const runs = runData.listRuns(process.cwd());
|
|
51
51
|
if (!runs.length)
|
|
52
|
-
return 'No run has recorded any data yet (
|
|
52
|
+
return 'No run has recorded any data yet (testforge/runs/run-data/ is empty).';
|
|
53
53
|
return ['Runs with recorded data:', ...runs.map((r) => ` ${r.runId} ${r.at.slice(0, 16).replace('T', ' ')} ${r.count} record(s)`),
|
|
54
54
|
'', 'Clean one with --run <id>, or every run older than N days with --older-than <N>. Add --yes to remove.'].join('\n');
|
|
55
55
|
}
|