@activate-core/test-forge 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +61 -23
- package/dist/core/docs/excerpts.js +245 -0
- package/dist/core/docs/grounding.js +49 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +202 -104
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +139 -0
- package/dist/core/generate/harness.js +3 -31
- package/dist/core/generate/journey-memory.js +325 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +169 -78
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +115 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +119 -0
- package/dist/core/run/run-tests.js +23 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — the source a journey names, collected without the AI.
|
|
3
|
+
*
|
|
4
|
+
* A journey doc already says where its behaviour lives: the [observed] rules
|
|
5
|
+
* cite files and lines, and the rules quote the texts a screen shows. Grepping
|
|
6
|
+
* for those is free; an AI doing the same with its tools took 25–32 turns, each
|
|
7
|
+
* re-sending everything read so far. The excerpts below go into ONE call.
|
|
8
|
+
*/
|
|
9
|
+
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
10
|
+
import { join, relative, resolve } from 'node:path';
|
|
11
|
+
const CODE = /\.(tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go)$/;
|
|
12
|
+
const SKIP_DIR = /^(node_modules|\.next|dist|build|coverage|vendor|storage|\.git|tests?|__tests__|__mocks__|test-artifacts|testforge|generated-tests)$/;
|
|
13
|
+
const FILE_REF = /((?:[\w.-]+\/)+[\w.-]+\.(?:tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go))(?::(\d+)(?:[-–](\d+))?)?/g;
|
|
14
|
+
const QUOTED = /"([^"\n]{4,60})"/g;
|
|
15
|
+
/** Words in a control's name that name no component in particular. */
|
|
16
|
+
const GENERIC = new Set(['button', 'buttons', 'field', 'input', 'inputs', 'header', 'footer', 'items', 'entry', 'entries', 'shown', 'guest', 'logged', 'customer', 'computer', 'phone', 'sized', 'screen', 'page', 'pages', 'where', 'opens', 'close', 'link', 'links', 'text', 'label', 'click', 'state', 'result', 'neighbour', 'neighbouring', 'other', 'while', 'their', 'there', 'which', 'every', 'itself', 'changes', 'layout']);
|
|
17
|
+
function walk(dir, out) {
|
|
18
|
+
let entries;
|
|
19
|
+
try {
|
|
20
|
+
entries = readdirSync(dir);
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
for (const e of entries) {
|
|
26
|
+
const p = join(dir, e);
|
|
27
|
+
let st;
|
|
28
|
+
try {
|
|
29
|
+
st = statSync(p);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
if (st.isDirectory()) {
|
|
35
|
+
if (!SKIP_DIR.test(e))
|
|
36
|
+
walk(p, out);
|
|
37
|
+
}
|
|
38
|
+
else if (CODE.test(e) && !/\.(test|spec)\.\w+$/.test(e) && st.size < 300_000)
|
|
39
|
+
out.push(p);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* A file named in the doc, found under the project or a related repo. Docs
|
|
44
|
+
* often shorten a path ("header/index.tsx"): that is matched as a suffix, and
|
|
45
|
+
* kept only when it points at no more than two files.
|
|
46
|
+
*/
|
|
47
|
+
function locate(ref, roots, all) {
|
|
48
|
+
for (const base of [process.cwd(), ...roots]) {
|
|
49
|
+
for (const cand of [join(base, ref), join(base, '..', ref)])
|
|
50
|
+
if (existsSync(cand) && statSync(cand).isFile())
|
|
51
|
+
return [resolve(cand)];
|
|
52
|
+
}
|
|
53
|
+
const hits = all.filter((f) => f.endsWith(`/${ref}`));
|
|
54
|
+
return hits.length <= 2 ? hits : [];
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The lines of the project's source a journey doc points at: every file it
|
|
58
|
+
* cites (around the cited lines), then every file that contains a text the
|
|
59
|
+
* doc quotes, most hits first, until `budget` characters.
|
|
60
|
+
*/
|
|
61
|
+
export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
62
|
+
const windows = new Map();
|
|
63
|
+
const add = (file, from, to) => {
|
|
64
|
+
const w = windows.get(file) ?? [];
|
|
65
|
+
w.push([Math.max(1, from), to]);
|
|
66
|
+
windows.set(file, w);
|
|
67
|
+
};
|
|
68
|
+
const order = [];
|
|
69
|
+
const files = [];
|
|
70
|
+
for (const r of roots)
|
|
71
|
+
walk(resolve(r), files);
|
|
72
|
+
for (const m of markdown.matchAll(FILE_REF)) {
|
|
73
|
+
for (const file of locate(m[1], roots, files)) {
|
|
74
|
+
const a = Number(m[2] ?? 0);
|
|
75
|
+
const b = Number(m[3] ?? a);
|
|
76
|
+
if (a)
|
|
77
|
+
add(file, a - 20, b + 20);
|
|
78
|
+
else
|
|
79
|
+
add(file, 1, 80);
|
|
80
|
+
if (!order.includes(file))
|
|
81
|
+
order.push(file);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// A file named after a control ("Escalate button" → escalateDisputeButton/) is
|
|
85
|
+
// the component itself: found by its path, before any text search.
|
|
86
|
+
const controlWords = [...markdown.matchAll(/^\|\s*C\d+[a-z]?\s*\|([^|]+)\|/gm)]
|
|
87
|
+
.flatMap((m) => m[1].replace(/["“”(][^"”)]*["”)]?/g, ' ').match(/[A-Za-z]{5,}/g) ?? [])
|
|
88
|
+
.map((w) => w.toLowerCase())
|
|
89
|
+
.filter((w) => !GENERIC.has(w));
|
|
90
|
+
const pathHits = files.filter((f) => {
|
|
91
|
+
const p = relative(process.cwd(), f).toLowerCase();
|
|
92
|
+
return [...new Set(controlWords)].some((w) => p.includes(w));
|
|
93
|
+
});
|
|
94
|
+
// How a component is reached and fed: the files that import it, up to the
|
|
95
|
+
// page that serves it (its route), and the hooks and API modules they use.
|
|
96
|
+
const importersOf = (target) => {
|
|
97
|
+
const stem = target.replace(/[\\/]index\.\w+$/, '').replace(/\.\w+$/, '');
|
|
98
|
+
const name = stem.replace(/^.*[\\/]/, '');
|
|
99
|
+
return files.filter((u) => {
|
|
100
|
+
if (u === target)
|
|
101
|
+
return false;
|
|
102
|
+
try {
|
|
103
|
+
return new RegExp(`from\\s+['"][^'"]*[\\/]${name}['"]`).test(readFileSync(u, 'utf-8'));
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
};
|
|
110
|
+
const feeds = (file) => {
|
|
111
|
+
let text = '';
|
|
112
|
+
try {
|
|
113
|
+
text = readFileSync(file, 'utf-8');
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return [];
|
|
117
|
+
}
|
|
118
|
+
const specs = [...text.matchAll(/from\s+['"]([^'"]+)['"]/g)].map((m) => m[1]).filter((sp) => /(^|[\\/])(use[A-Z]\w*|api|services?|types?|dataProvider|queries)([\\/]|$)/.test(sp));
|
|
119
|
+
return files.filter((f) => specs.some((sp) => f.replace(/\.\w+$/, '').replace(/[\\/]index$/, '').endsWith(sp.replace(/^[@~.]+[\\/]?/, '').replace(/^src[\\/]/, ''))));
|
|
120
|
+
};
|
|
121
|
+
for (const f of pathHits.slice(0, 12)) {
|
|
122
|
+
let level = [f];
|
|
123
|
+
for (let hop = 0; hop < 3 && level.length; hop++) {
|
|
124
|
+
const next = level.flatMap(importersOf).filter((u) => !order.includes(u)).slice(0, 4);
|
|
125
|
+
for (const u of next) {
|
|
126
|
+
add(u, 1, /[\\/](app|pages)[\\/]/.test(u) ? 40 : 15);
|
|
127
|
+
order.push(u);
|
|
128
|
+
for (const h of feeds(u).slice(0, 3))
|
|
129
|
+
if (!order.includes(h)) {
|
|
130
|
+
add(h, 1, 80);
|
|
131
|
+
order.push(h);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
level = next;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
for (const f of pathHits.slice(0, 12)) {
|
|
138
|
+
add(f, 1, 120);
|
|
139
|
+
if (!order.includes(f))
|
|
140
|
+
order.push(f);
|
|
141
|
+
// And where it is used: the page that renders it says how to reach it.
|
|
142
|
+
const name = /[\\/]([^\\/]+)[\\/]index\.\w+$/.exec(f)?.[1] ?? f.replace(/^.*[\\/]|\.\w+$/g, '');
|
|
143
|
+
for (const user of files) {
|
|
144
|
+
if (user === f)
|
|
145
|
+
continue;
|
|
146
|
+
let text;
|
|
147
|
+
try {
|
|
148
|
+
text = readFileSync(user, 'utf-8');
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
if (!new RegExp(`from\\s+['"][^'"]*\\b${name}['"]`).test(text))
|
|
154
|
+
continue;
|
|
155
|
+
text.split('\n').forEach((l, i) => { if (l.includes(name) || /<[A-Z]\w*/.test(l) && l.includes(name.charAt(0).toUpperCase() + name.slice(1)))
|
|
156
|
+
add(user, i + 1 - 10, i + 1 + 10); });
|
|
157
|
+
if (!order.includes(user))
|
|
158
|
+
order.push(user);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const quoted = [...markdown.matchAll(QUOTED)].map((m) => m[1].trim()).filter((t) => /[a-z]/i.test(t) && !/\.\s/.test(t));
|
|
162
|
+
const named = [...markdown.matchAll(/`([A-Za-z_$][\w$]*(?:\.[\w$]+)*)`/g)].map((m) => m[1]).filter((t) => t.length >= 4 && /[A-Z_.]/.test(t));
|
|
163
|
+
const terms = [...new Set([...quoted, ...named])];
|
|
164
|
+
if (terms.length) {
|
|
165
|
+
const hits = [];
|
|
166
|
+
for (const f of files) {
|
|
167
|
+
let text;
|
|
168
|
+
try {
|
|
169
|
+
text = readFileSync(f, 'utf-8');
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
const found = terms.filter((t) => text.includes(t));
|
|
175
|
+
if (!found.length)
|
|
176
|
+
continue;
|
|
177
|
+
const lines = text.split('\n');
|
|
178
|
+
hits.push({ file: f, terms: found, lines: lines.flatMap((l, i) => (found.some((t) => l.includes(t)) ? [i + 1] : [])) });
|
|
179
|
+
}
|
|
180
|
+
// A text found in one file says more about it than one found in twenty; one
|
|
181
|
+
// found in more than six ("Log In", "Signup") points at nothing in particular.
|
|
182
|
+
const spread = new Map(terms.map((t) => [t, hits.filter((h) => h.terms.includes(t)).length]));
|
|
183
|
+
for (const h of hits) {
|
|
184
|
+
h.terms = h.terms.filter((t) => (spread.get(t) ?? 0) <= 6);
|
|
185
|
+
h.lines = h.lines.filter((n) => h.terms.length > 0 && n > 0);
|
|
186
|
+
}
|
|
187
|
+
for (let i = hits.length - 1; i >= 0; i--)
|
|
188
|
+
if (!hits[i].terms.length)
|
|
189
|
+
hits.splice(i, 1);
|
|
190
|
+
const score = (h) => h.terms.reduce((n, t) => n + 1 / (spread.get(t) || 1), 0);
|
|
191
|
+
hits.sort((x, y) => score(y) - score(x));
|
|
192
|
+
for (const h of hits) {
|
|
193
|
+
const text = readFileSync(h.file, 'utf-8').split('\n');
|
|
194
|
+
const at = text.flatMap((l, i) => (h.terms.some((t) => l.includes(t)) ? [i + 1] : []));
|
|
195
|
+
for (const n of at)
|
|
196
|
+
add(h.file, n - 8, n + 8);
|
|
197
|
+
if (!order.includes(h.file))
|
|
198
|
+
order.push(h.file);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
const out = [];
|
|
202
|
+
let used = 0;
|
|
203
|
+
for (const file of order) {
|
|
204
|
+
const lines = readFileSync(file, 'utf-8').split('\n');
|
|
205
|
+
const merged = (windows.get(file) ?? [])
|
|
206
|
+
.map(([a, b]) => [a, Math.min(b, lines.length)])
|
|
207
|
+
.sort((x, y) => x[0] - y[0])
|
|
208
|
+
.reduce((acc, w) => {
|
|
209
|
+
const last = acc[acc.length - 1];
|
|
210
|
+
if (last && w[0] <= last[1] + 1)
|
|
211
|
+
last[1] = Math.max(last[1], w[1]);
|
|
212
|
+
else
|
|
213
|
+
acc.push([...w]);
|
|
214
|
+
return acc;
|
|
215
|
+
}, []);
|
|
216
|
+
const body = merged.map(([a, b]) => lines.slice(a - 1, b).map((l, i) => `${a + i}: ${l}`).join('\n')).join('\n…\n');
|
|
217
|
+
const block = `### ${relative(process.cwd(), file)}\n${body}\n`;
|
|
218
|
+
if (used + block.length > budget) {
|
|
219
|
+
if (used === 0)
|
|
220
|
+
out.push(block.slice(0, budget));
|
|
221
|
+
break;
|
|
222
|
+
}
|
|
223
|
+
out.push(block);
|
|
224
|
+
used += block.length;
|
|
225
|
+
}
|
|
226
|
+
return out.join('\n');
|
|
227
|
+
}
|
|
228
|
+
/** All of the source under `roots`, lower-cased with whitespace collapsed: what a label check searches. */
|
|
229
|
+
export function sourceText(roots) {
|
|
230
|
+
const files = [];
|
|
231
|
+
for (const r of roots)
|
|
232
|
+
walk(resolve(r), files);
|
|
233
|
+
return files
|
|
234
|
+
.map((f) => {
|
|
235
|
+
try {
|
|
236
|
+
return readFileSync(f, 'utf-8');
|
|
237
|
+
}
|
|
238
|
+
catch {
|
|
239
|
+
return '';
|
|
240
|
+
}
|
|
241
|
+
})
|
|
242
|
+
.join('\n')
|
|
243
|
+
.toLowerCase()
|
|
244
|
+
.replace(/\s+/g, ' ');
|
|
245
|
+
}
|
|
@@ -19,9 +19,11 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
21
21
|
import { dirname, join } from 'node:path';
|
|
22
|
-
import { loadConfig, splitCommand,
|
|
22
|
+
import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
23
|
+
import { readRules, ruleIndex } from '../generate/journey-memory.js';
|
|
24
|
+
import { sourceExcerpts } from './excerpts.js';
|
|
23
25
|
import { hashProjectSource } from './cache.js';
|
|
24
|
-
import {
|
|
26
|
+
import { journeyDir } from '../paths.js';
|
|
25
27
|
import { loadEnvFile } from '../project/config.js';
|
|
26
28
|
import { originsFrom, pathsFrom, observeApp } from './observe.js';
|
|
27
29
|
import { observeSchema } from './schema.js';
|
|
@@ -35,27 +37,31 @@ const MAP_PATH = 'testforge.md';
|
|
|
35
37
|
* that upgrades keeps serving the facts it cached under the old version —
|
|
36
38
|
* so the new facts never reach a single prompt and the upgrade does nothing.
|
|
37
39
|
*/
|
|
38
|
-
const FACTS_VERSION =
|
|
40
|
+
const FACTS_VERSION = 4;
|
|
39
41
|
/** Where a journey's grounded facts live, mirroring the plan cache layout. */
|
|
40
42
|
function groundingPath(journey) {
|
|
41
|
-
|
|
42
|
-
const slug = (journey.name || 'journey')
|
|
43
|
-
.toLowerCase()
|
|
44
|
-
.replace(/[^a-z0-9]+/g, '-')
|
|
45
|
-
.replace(/^-|-$/g, '');
|
|
46
|
-
return join(STATE_DIR, 'grounding', area, `${slug}.json`);
|
|
43
|
+
return join(journeyDir(journey), 'facts.json');
|
|
47
44
|
}
|
|
48
|
-
function buildGroundingPrompt(journey, map, srcDir) {
|
|
45
|
+
function buildGroundingPrompt(journey, map, srcDir, excerpts = '') {
|
|
49
46
|
const isUi = journey.type !== 'api';
|
|
50
47
|
return [
|
|
51
48
|
`You are a senior engineer establishing the MECHANICS of one user`,
|
|
52
49
|
`journey so tests can be written against it without every writer`,
|
|
53
50
|
`re-reading the same source.`,
|
|
54
51
|
``,
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
52
|
+
...(excerpts
|
|
53
|
+
? [
|
|
54
|
+
`The parts of this project's source that the journey names (the files`,
|
|
55
|
+
`it cites, and the files that contain the texts it quotes) are under`,
|
|
56
|
+
`SOURCE EXCERPTS below, with line numbers. You have no tools: work`,
|
|
57
|
+
`from those excerpts only.`,
|
|
58
|
+
]
|
|
59
|
+
: [
|
|
60
|
+
`You are inside the project's repository with read-only tools (Read,`,
|
|
61
|
+
`Grep, Glob, LS). Read the real code for this journey under "${srcDir}":`,
|
|
62
|
+
`start from the route/endpoint the journey names, then the`,
|
|
63
|
+
`controller/handler, the service it calls, and the models it touches.`,
|
|
64
|
+
]),
|
|
59
65
|
``,
|
|
60
66
|
`Report ONLY what you actually read. Never guess a path, a status code`,
|
|
61
67
|
`or a field name: a fact invented here is repeated into every test file`,
|
|
@@ -95,6 +101,11 @@ function buildGroundingPrompt(journey, map, srcDir) {
|
|
|
95
101
|
` label/placeholder text, on-screen text, input name/id`,
|
|
96
102
|
` attributes, and any data-testid that truly exists in the`,
|
|
97
103
|
` source. Never list a selector you did not see.`,
|
|
104
|
+
` Start every SCREENS line with the ids from the journey's Controls`,
|
|
105
|
+
` table that it serves, in brackets: "[C3,C4] header/index.tsx: link`,
|
|
106
|
+
` 'FAQs' (role link)". Each test file is given only the lines for its`,
|
|
107
|
+
` own controls, so a line without its ids reaches every file and a`,
|
|
108
|
+
` line with the wrong ids reaches none.`,
|
|
98
109
|
]
|
|
99
110
|
: []),
|
|
100
111
|
``,
|
|
@@ -102,13 +113,19 @@ function buildGroundingPrompt(journey, map, srcDir) {
|
|
|
102
113
|
`restating the journey, no advice about testing. Aim for well under 200`,
|
|
103
114
|
`lines. Output the headings and their contents only.`,
|
|
104
115
|
``,
|
|
105
|
-
`--- PROJECT MAP ---`,
|
|
106
|
-
map,
|
|
107
|
-
``,
|
|
116
|
+
...(excerpts ? [] : [`--- PROJECT MAP ---`, map, ``]),
|
|
108
117
|
`--- JOURNEY ---`,
|
|
109
|
-
journey.rawMarkdown || journey.name,
|
|
118
|
+
...(excerpts ? [compactJourney(journey.rawMarkdown || journey.name)] : [journey.rawMarkdown || journey.name]),
|
|
119
|
+
...(excerpts ? [``, `--- SOURCE EXCERPTS ---`, excerpts] : []),
|
|
110
120
|
].join('\n');
|
|
111
121
|
}
|
|
122
|
+
/** The journey as its controls and one line per rule: what the facts must serve, without the prose. */
|
|
123
|
+
function compactJourney(markdown) {
|
|
124
|
+
const m = readRules(markdown);
|
|
125
|
+
if (!m.rules.length)
|
|
126
|
+
return markdown;
|
|
127
|
+
return [m.header, ``, `CONTROLS:`, ...Object.entries(m.controls).map(([id, c]) => `${id}: ${c}`), ``, `RULES:`, ruleIndex(m)].join('\n');
|
|
128
|
+
}
|
|
112
129
|
/**
|
|
113
130
|
* Establish (or reuse) the journey's code facts. Cached against the doc hash
|
|
114
131
|
* AND the source hash: the mechanics only move when the code or the journey
|
|
@@ -117,7 +134,9 @@ function buildGroundingPrompt(journey, map, srcDir) {
|
|
|
117
134
|
* Never throws. If the AI call fails we return null and the caller falls back
|
|
118
135
|
* to telling each file to read the source itself — slower, but still correct.
|
|
119
136
|
*/
|
|
120
|
-
export async function groundJourney(journey, docHash, srcDir = 'src', markdown = ''
|
|
137
|
+
export async function groundJourney(journey, docHash, srcDir = 'src', markdown = '',
|
|
138
|
+
/** A doc edit, not a new journey: reuse the facts already read, whatever version of the doc they came from. */
|
|
139
|
+
reuseOnly = false) {
|
|
121
140
|
const path = groundingPath(journey);
|
|
122
141
|
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
123
142
|
let sourceHash = '';
|
|
@@ -131,7 +150,7 @@ export async function groundJourney(journey, docHash, srcDir = 'src', markdown =
|
|
|
131
150
|
if (existsSync(path)) {
|
|
132
151
|
try {
|
|
133
152
|
const cached = JSON.parse(readFileSync(path, 'utf-8'));
|
|
134
|
-
if (cached.
|
|
153
|
+
if (typeof cached.facts === 'string' && cached.facts.trim() && (cached.key === key || (reuseOnly && String(cached.key).startsWith(`v${FACTS_VERSION}:`)))) {
|
|
135
154
|
return cached.facts;
|
|
136
155
|
}
|
|
137
156
|
}
|
|
@@ -139,12 +158,20 @@ export async function groundJourney(journey, docHash, srcDir = 'src', markdown =
|
|
|
139
158
|
// corrupt cache — re-ground
|
|
140
159
|
}
|
|
141
160
|
}
|
|
161
|
+
if (reuseOnly)
|
|
162
|
+
return null;
|
|
142
163
|
const config = loadConfig();
|
|
143
|
-
|
|
164
|
+
// The source the doc points at, collected here, goes into one tool-less call.
|
|
165
|
+
// A doc that points at nothing findable gets no facts: its tests are written
|
|
166
|
+
// from the doc, which is the contract, rather than paying an AI to go looking.
|
|
167
|
+
const excerpts = sourceExcerpts(journey.rawMarkdown || '', [srcDir, ...(config.relatedDirs ?? [])]);
|
|
168
|
+
if (!excerpts)
|
|
169
|
+
return null;
|
|
170
|
+
const { program, args } = withNoTools(splitCommand(config.aiCommand));
|
|
144
171
|
let facts;
|
|
145
172
|
try {
|
|
146
173
|
facts = runAiSync(program, args, {
|
|
147
|
-
input: buildGroundingPrompt(journey, map, srcDir),
|
|
174
|
+
input: buildGroundingPrompt(journey, map, srcDir, excerpts),
|
|
148
175
|
encoding: 'utf-8',
|
|
149
176
|
maxBuffer: 10 * 1024 * 1024,
|
|
150
177
|
timeout: config.aiTimeoutMs ?? 300_000,
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — a reference read once, and only its relevant part handed on.
|
|
3
|
+
*
|
|
4
|
+
* Drafting one journey used to fetch the same Notion page three times (to
|
|
5
|
+
* compare, to draft, to review), each inside a long tool session. The page is
|
|
6
|
+
* now fetched by one short call, kept on disk, and cut down to the journey
|
|
7
|
+
* being drafted, its shared rules and the page's introduction. A file in the
|
|
8
|
+
* project is simply read. `--refresh` fetches again.
|
|
9
|
+
*/
|
|
10
|
+
import { createHash } from 'node:crypto';
|
|
11
|
+
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import { REFERENCE_DIR } from '../paths.js';
|
|
14
|
+
import { loadConfig, splitCommand, withReferenceTools } from '../project/config.js';
|
|
15
|
+
import { runAiSync } from '../generate/ai-usage.js';
|
|
16
|
+
const cachePath = (reference) => join(REFERENCE_DIR, `${createHash('sha1').update(reference).digest('hex').slice(0, 12)}.md`);
|
|
17
|
+
/** The reference's text: a local file as it is, a page or URL fetched once and kept. */
|
|
18
|
+
export function readReference(reference, refresh = false) {
|
|
19
|
+
if (existsSync(reference) && statSync(reference).isFile() && !/\.pdf$/i.test(reference))
|
|
20
|
+
return readFileSync(reference, 'utf-8');
|
|
21
|
+
const path = cachePath(reference);
|
|
22
|
+
if (!refresh && existsSync(path))
|
|
23
|
+
return readFileSync(path, 'utf-8');
|
|
24
|
+
const { program, args } = withReferenceTools(splitCommand(loadConfig().aiCommand));
|
|
25
|
+
const text = runAiSync(program, args, {
|
|
26
|
+
input: [
|
|
27
|
+
`Fetch this reference and print its FULL content as markdown, word for word:`,
|
|
28
|
+
` ${reference}`,
|
|
29
|
+
`It may be a Notion page, a URL or a document. Keep every heading, list,`,
|
|
30
|
+
`table and sentence; summarise nothing, add nothing. Include its sub-pages`,
|
|
31
|
+
`only if the page's own text is just a list of links to them. Print only`,
|
|
32
|
+
`the content. If you cannot read it, print one line starting "ERROR:".`,
|
|
33
|
+
].join('\n'),
|
|
34
|
+
encoding: 'utf-8',
|
|
35
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
36
|
+
}).trim();
|
|
37
|
+
if (!text || /^ERROR:/m.test(text.slice(0, 300)))
|
|
38
|
+
throw new Error(`Could not read the reference "${reference}":\n${text.slice(0, 500)}`);
|
|
39
|
+
mkdirSync(REFERENCE_DIR, { recursive: true });
|
|
40
|
+
writeFileSync(path, text, 'utf-8');
|
|
41
|
+
return text;
|
|
42
|
+
}
|
|
43
|
+
const norm = (t) => t.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
|
44
|
+
/**
|
|
45
|
+
* The part of a reference one journey needs: everything before the first
|
|
46
|
+
* section that names a journey (the page's introduction and shared rules),
|
|
47
|
+
* the sections that are not journeys, and the section whose heading names
|
|
48
|
+
* this one. A reference with no such heading is returned whole.
|
|
49
|
+
*/
|
|
50
|
+
export function sectionFor(text, journey) {
|
|
51
|
+
const lines = text.split('\n');
|
|
52
|
+
const headings = lines.flatMap((l, i) => {
|
|
53
|
+
const m = /^(#{1,4})\s+(.*)$/.exec(l);
|
|
54
|
+
return m ? [{ i, level: m[1].length, title: m[2] }] : [];
|
|
55
|
+
});
|
|
56
|
+
const want = norm(journey);
|
|
57
|
+
const hit = headings.find((h) => norm(h.title).includes(want));
|
|
58
|
+
if (!hit)
|
|
59
|
+
return text;
|
|
60
|
+
const isJourney = (t) => /\bjourney\b|^\d+[.)]\s/i.test(t);
|
|
61
|
+
const keep = new Set();
|
|
62
|
+
const span = (h) => {
|
|
63
|
+
const next = headings.find((x) => x.i > h.i && x.level <= h.level);
|
|
64
|
+
return [h.i, next ? next.i : lines.length];
|
|
65
|
+
};
|
|
66
|
+
const firstJourney = headings.find((h) => isJourney(h.title) && h.level === hit.level);
|
|
67
|
+
for (let i = 0; i < (firstJourney ? firstJourney.i : 0); i++)
|
|
68
|
+
keep.add(i);
|
|
69
|
+
for (const h of headings.filter((x) => x.level === hit.level && !isJourney(x.title))) {
|
|
70
|
+
const [a, b] = span(h);
|
|
71
|
+
for (let i = a; i < b; i++)
|
|
72
|
+
keep.add(i);
|
|
73
|
+
}
|
|
74
|
+
const [a, b] = span(hit);
|
|
75
|
+
for (let i = a; i < b; i++)
|
|
76
|
+
keep.add(i);
|
|
77
|
+
return lines.filter((_, i) => keep.has(i)).join('\n').replace(/\n{3,}/g, '\n\n').trim();
|
|
78
|
+
}
|
|
@@ -21,11 +21,11 @@
|
|
|
21
21
|
* URLs/Notion, web) access. As with discover/map this is tool-specific
|
|
22
22
|
* and tuned for the Claude CLI.
|
|
23
23
|
*/
|
|
24
|
-
import { loadConfig, splitCommand, withReferenceTools } from '../project/config.js';
|
|
24
|
+
import { loadConfig, splitCommand, withNoTools, withReferenceTools } from '../project/config.js';
|
|
25
25
|
import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
|
|
26
26
|
import { runAiSync } from '../generate/ai-usage.js';
|
|
27
27
|
/** Where the comparison report is written. */
|
|
28
|
-
export const DIFF_PATH = 'testforge
|
|
28
|
+
export const DIFF_PATH = 'testforge/reference/diff.md';
|
|
29
29
|
const MAP_PATH = 'testforge.md';
|
|
30
30
|
/** Run the configured AI command with read/write/web (+ Notion) access. */
|
|
31
31
|
function runAI(prompt) {
|
|
@@ -67,7 +67,7 @@ export function buildAnalyzePrompt(reference, docsDir) {
|
|
|
67
67
|
` that the code-derived view omits or gets wrong`,
|
|
68
68
|
` - anything the code-derived view asserts that the reference contradicts`,
|
|
69
69
|
``,
|
|
70
|
-
`Write a comparison report to "${DIFF_PATH}"
|
|
70
|
+
`Write a comparison report to "${DIFF_PATH}" with`,
|
|
71
71
|
`these sections:`,
|
|
72
72
|
` # Source-of-truth comparison`,
|
|
73
73
|
` ## Reference (what you read, and where from)`,
|
|
@@ -217,3 +217,60 @@ export function analyzeReference(reference, docsDir) {
|
|
|
217
217
|
export function draftDocsFromReference(srcDir, docsDir, reference, journey, existingDocs) {
|
|
218
218
|
return runAI(buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey, existingDocs));
|
|
219
219
|
}
|
|
220
|
+
/**
|
|
221
|
+
* Draft ONE journey's doc in a single call with no tools: the reference's
|
|
222
|
+
* part for this journey and the source lines it points at are in the prompt.
|
|
223
|
+
* The answer is the doc itself, after one AREA line.
|
|
224
|
+
*/
|
|
225
|
+
export function buildJourneyDraftPrompt(journey, section, excerpts, areas, otherDocs) {
|
|
226
|
+
return [
|
|
227
|
+
`You are a senior QA engineer writing ONE journey doc, "${journey}", so a QA`,
|
|
228
|
+
`tool can generate tests from it. The REFERENCE below is the source of truth`,
|
|
229
|
+
`for how the product is EXPECTED to behave; where the code does something`,
|
|
230
|
+
`else, document the reference (the code may be buggy).`,
|
|
231
|
+
``,
|
|
232
|
+
DEEP_DOC_GUIDANCE,
|
|
233
|
+
``,
|
|
234
|
+
`You have no tools. Everything you may use is below: the reference's part`,
|
|
235
|
+
`for this journey (with the page's shared rules), and SOURCE EXCERPTS, the`,
|
|
236
|
+
`lines of this project's code the journey's texts and files point at. Use the`,
|
|
237
|
+
`excerpts only for TECHNICAL detail (routes, field names, labels the screen`,
|
|
238
|
+
`really shows) and for [observed] rules; [spec] comes only from the reference.`,
|
|
239
|
+
`Where the excerpts do not show something, do not invent it.`,
|
|
240
|
+
``,
|
|
241
|
+
...(otherDocs.length
|
|
242
|
+
? [`Other journeys already have docs; a rule they own is not repeated here, only named:`, ...otherDocs.map((d) => ` - ${d}`), ``]
|
|
243
|
+
: []),
|
|
244
|
+
`The doc starts with this header:`,
|
|
245
|
+
`# Journey: <Name>`,
|
|
246
|
+
``,
|
|
247
|
+
`Type: web ("web" for a page/UI, "api" for an endpoint)`,
|
|
248
|
+
`Route: <path>`,
|
|
249
|
+
``,
|
|
250
|
+
`Coverage:`,
|
|
251
|
+
`- functional`,
|
|
252
|
+
`- security`,
|
|
253
|
+
``,
|
|
254
|
+
`Rules: tag every criterion as described above; [spec] only where the`,
|
|
255
|
+
`reference states it, with its quote. When the code seems to behave`,
|
|
256
|
+
`differently from a [spec] criterion, keep the criterion and add a line`,
|
|
257
|
+
`"Likely defect: <what the code does>" under it. No "# TODO", no secrets.`,
|
|
258
|
+
``,
|
|
259
|
+
`OUTPUT: first one line "AREA: <one of ${areas.join(', ')}>", then the doc`,
|
|
260
|
+
`itself, starting with "# Journey:". Nothing before or after.`,
|
|
261
|
+
``,
|
|
262
|
+
`--- REFERENCE ---`,
|
|
263
|
+
section,
|
|
264
|
+
``,
|
|
265
|
+
`--- SOURCE EXCERPTS ---`,
|
|
266
|
+
excerpts || '(no source the reference points at was found: the feature may not be built yet)',
|
|
267
|
+
].join('\n');
|
|
268
|
+
}
|
|
269
|
+
/** One no-tools call that answers with a doc; returns its area and markdown. */
|
|
270
|
+
export function draftJourneyDoc(prompt) {
|
|
271
|
+
const { program, args } = withNoTools(splitCommand(loadConfig().aiCommand));
|
|
272
|
+
const out = runAiSync(program, args, { input: prompt, encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 }).trim();
|
|
273
|
+
const area = /^AREA:\s*([\w-]+)/m.exec(out)?.[1];
|
|
274
|
+
const at = out.indexOf('# Journey:');
|
|
275
|
+
return { area, markdown: at >= 0 ? `${out.slice(at).trim()}\n` : '' };
|
|
276
|
+
}
|
package/dist/core/docs/review.js
CHANGED
|
@@ -61,6 +61,44 @@ export function buildReviewPrompt(docPath, srcDir, reference) {
|
|
|
61
61
|
`with counts.`,
|
|
62
62
|
].join('\n');
|
|
63
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* The same review in one call with no tools: the reference's part, the source
|
|
66
|
+
* excerpts and the draft are in the prompt, and the answer is the whole
|
|
67
|
+
* revised doc.
|
|
68
|
+
*/
|
|
69
|
+
export function buildLeanReviewPrompt(doc, section, excerpts, otherDocs) {
|
|
70
|
+
return [
|
|
71
|
+
`You are a senior QA engineer reviewing ONE journey doc that a first pass`,
|
|
72
|
+
`drafted. Make it COMPLETE, so no scenario a careful QA engineer, or a user`,
|
|
73
|
+
`trying to break it, would test is missing. You have no tools: the`,
|
|
74
|
+
`reference's part for this journey, the source excerpts it points at and the`,
|
|
75
|
+
`draft are below. [spec] comes only from the reference; the excerpts give`,
|
|
76
|
+
`technical detail and [observed] rules.`,
|
|
77
|
+
``,
|
|
78
|
+
`The doc must follow this guidance:`,
|
|
79
|
+
DEEP_DOC_GUIDANCE,
|
|
80
|
+
``,
|
|
81
|
+
`Check it against the METHOD above and fix what you find: every control in`,
|
|
82
|
+
`the table; every [baseline] item that fits each control, with its level`,
|
|
83
|
+
`and category, and every MUST that fits; every combination the reference`,
|
|
84
|
+
`states; every branch the excerpts show, as [observed] or a Question; every`,
|
|
85
|
+
`[spec] quoting a real reference sentence (re-tag any that does not);`,
|
|
86
|
+
`the Coverage table with no blank cell. Keep every sound criterion, the`,
|
|
87
|
+
`header and the section numbering. Never write "# TODO".`,
|
|
88
|
+
...(otherDocs.length ? [`Other journeys' docs (shared rules stay with them):`, ...otherDocs.map((d) => ` - ${d}`)] : []),
|
|
89
|
+
``,
|
|
90
|
+
`OUTPUT: the complete revised doc, starting with "# Journey:". Nothing else.`,
|
|
91
|
+
``,
|
|
92
|
+
`--- REFERENCE ---`,
|
|
93
|
+
section,
|
|
94
|
+
``,
|
|
95
|
+
`--- SOURCE EXCERPTS ---`,
|
|
96
|
+
excerpts || '(none found)',
|
|
97
|
+
``,
|
|
98
|
+
`--- THE DRAFT ---`,
|
|
99
|
+
doc,
|
|
100
|
+
].join('\n');
|
|
101
|
+
}
|
|
64
102
|
/**
|
|
65
103
|
* A [spec] line is a requirement the client wrote, so it must carry the
|
|
66
104
|
* client's words: `(source: "…")`, or `(source as above)` pointing at the quote before it. A [spec] line without that quote is the
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* 2. An external reference — a spec/doc/PDF in the repo, a website, or
|
|
10
10
|
* a Notion page capturing the human/client EXPECTED behavior.
|
|
11
11
|
*
|
|
12
|
-
* The choice is remembered in
|
|
12
|
+
* The choice is remembered in testforge/source.json so later refreshes
|
|
13
13
|
* (`update`) keep using the same source without asking again — otherwise
|
|
14
14
|
* an update would silently regenerate reference-based docs from the code.
|
|
15
15
|
*
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
9
9
|
import { basename, extname, join } from 'node:path';
|
|
10
|
-
import {
|
|
11
|
-
export const TICKETS_REFERENCE = join(
|
|
10
|
+
import { REFERENCE_DIR } from '../paths.js';
|
|
11
|
+
export const TICKETS_REFERENCE = join(REFERENCE_DIR, 'tickets.md');
|
|
12
12
|
const TEXT = /\.(md|markdown|txt|text|json|csv|html?|xml|ya?ml)$/i;
|
|
13
13
|
const ID = /\b([A-Z][A-Z0-9]+-\d+)\b/;
|
|
14
14
|
const MAX_CHARS = 20_000;
|
|
@@ -50,7 +50,7 @@ export function bundleTickets(folder) {
|
|
|
50
50
|
'',
|
|
51
51
|
...tickets.flatMap((t) => [`## ${t.id} — ${basename(t.file)}`, '', t.content.trim().slice(0, MAX_CHARS), '']),
|
|
52
52
|
].join('\n');
|
|
53
|
-
mkdirSync(
|
|
53
|
+
mkdirSync(REFERENCE_DIR, { recursive: true });
|
|
54
54
|
writeFileSync(TICKETS_REFERENCE, body, 'utf-8');
|
|
55
55
|
return { path: TICKETS_REFERENCE, ids: tickets.map((t) => t.id) };
|
|
56
56
|
}
|
|
@@ -13,7 +13,7 @@ import { COVERAGE_PATH, STATE_DIR } from '../paths.js';
|
|
|
13
13
|
* F8 — write an honest coverage manifest. For every generated file it
|
|
14
14
|
* records real (has assertions) vs deferred (pending only), so a "green"
|
|
15
15
|
* run is never mistaken for full coverage. Written to
|
|
16
|
-
*
|
|
16
|
+
* testforge/coverage.json and summarised to the console.
|
|
17
17
|
*/
|
|
18
18
|
export function writeCoverageManifest(results) {
|
|
19
19
|
const journeys = results
|
|
@@ -38,10 +38,6 @@ export function writeCoverageManifest(results) {
|
|
|
38
38
|
journey: r.journeyName,
|
|
39
39
|
real: files.filter((f) => f.real).length,
|
|
40
40
|
deferred: files.filter((f) => !f.real).length,
|
|
41
|
-
// Whether these files were RUN and passed. A file can assert plenty
|
|
42
|
-
// and still target something the app does not have, so "real" alone
|
|
43
|
-
// was never enough to call the coverage honest.
|
|
44
|
-
verified: r.verified,
|
|
45
41
|
tiers,
|
|
46
42
|
files,
|
|
47
43
|
};
|
|
@@ -55,9 +51,6 @@ export function writeCoverageManifest(results) {
|
|
|
55
51
|
writeFileSync(COVERAGE_PATH, JSON.stringify({
|
|
56
52
|
real: totalReal,
|
|
57
53
|
deferred: totalDeferred,
|
|
58
|
-
verified: journeys.filter((j) => j.verified === true).length,
|
|
59
|
-
failedVerify: journeys.filter((j) => j.verified === false).length,
|
|
60
|
-
notVerified: journeys.filter((j) => j.verified === undefined).length,
|
|
61
54
|
journeys,
|
|
62
55
|
}, null, 2), 'utf-8');
|
|
63
56
|
}
|
|
@@ -67,14 +60,6 @@ export function writeCoverageManifest(results) {
|
|
|
67
60
|
console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see ${COVERAGE_PATH}`);
|
|
68
61
|
const t = journeys.reduce((a, j) => ({ spec: a.spec + j.tiers.spec, baseline: a.baseline + j.tiers.baseline, observed: a.observed + j.tiers.observed, untagged: a.untagged + j.tiers.untagged }), { spec: 0, baseline: 0, observed: 0, untagged: 0 });
|
|
69
62
|
console.log(`Tests by the rule they check: ${t.spec} client requirement [spec], ${t.baseline} robustness [baseline], ${t.observed} current behaviour [observed]${t.untagged ? `, ${t.untagged} with no tagged rule` : ''}.`);
|
|
70
|
-
const ranGreen = journeys.filter((j) => j.verified === true).length;
|
|
71
|
-
const ranRed = journeys.filter((j) => j.verified === false).length;
|
|
72
|
-
const unrun = journeys.filter((j) => j.verified === undefined).length;
|
|
73
|
-
if (ranGreen || ranRed || unrun) {
|
|
74
|
-
console.log(`Verified: ${ranGreen} journey(s) ran green` +
|
|
75
|
-
(ranRed ? `, ${ranRed} still failing` : '') +
|
|
76
|
-
(unrun ? `, ${unrun} not run` : ''));
|
|
77
|
-
}
|
|
78
63
|
if (totalDeferred) {
|
|
79
64
|
console.log(` (deferred files are pending, NOT real coverage — regenerate or implement them.)`);
|
|
80
65
|
}
|