@activate-core/test-forge 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -3
- package/dist/cli/index.js +2 -1
- package/dist/core/docs/excerpts.js +139 -5
- package/dist/core/docs/grounding.js +78 -16
- package/dist/core/docs/reference-text.js +2 -0
- package/dist/core/gates/coverage.js +9 -0
- package/dist/core/gates/validate.js +1 -0
- package/dist/core/generate/ai-generator.js +9 -3
- package/dist/core/generate/ai-usage.js +5 -2
- package/dist/core/generate/complete-file.js +45 -7
- package/dist/core/generate/defect-review.js +2 -1
- package/dist/core/generate/harness.js +34 -0
- package/dist/core/generate/interface-file.js +12 -0
- package/dist/core/generate/journey-memory.js +7 -1
- package/dist/core/generate/passed.js +24 -0
- package/dist/core/generate/run-generate.js +117 -13
- package/dist/core/generate/test-blocks.js +34 -10
- package/dist/core/project/config.js +5 -0
- package/dist/core/prompts/blocks.js +24 -3
- package/dist/core/run/name-scan.js +120 -14
- package/dist/core/run/report.js +7 -4
- package/dist/core/run/run-tests.js +53 -4
- package/package.json +3 -2
- package/reporters/_shared.cjs +56 -3
- package/reporters/interface.cjs +14 -0
- package/reporters/jest-reporter.cjs +7 -3
- package/reporters/playwright-reporter.cjs +6 -2
- package/reporters/run-data.cjs +6 -1
package/README.md
CHANGED
|
@@ -271,9 +271,30 @@ prints a note.
|
|
|
271
271
|
|
|
272
272
|
## Running the generated tests
|
|
273
273
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
274
|
+
Run them with `testforge run` (it fills waiting names from the code first,
|
|
275
|
+
holds back files whose feature is not live yet, and prints how many tests
|
|
276
|
+
ran of how many), or with your own test runner — the generated config
|
|
277
|
+
carries the TestForge reporter either way.
|
|
278
|
+
|
|
279
|
+
**What a run remembers.** Every test that passes is recorded in
|
|
280
|
+
`testforge/passed.json`. From then on `testforge generate` never removes or
|
|
281
|
+
rewrites it — a later failure is a regression to look at, not a test to
|
|
282
|
+
rewrite. Only `generate --force` writes such a test again. A kept file's
|
|
283
|
+
other tests are reported by the checks, never removed; once the code a
|
|
284
|
+
waiting test needs arrives, only that test is written again.
|
|
285
|
+
|
|
286
|
+
**Exit code.** A run whose only non-passing tests are waiting for the
|
|
287
|
+
product or stopped by the environment exits 0: nothing failed for a product
|
|
288
|
+
reason. A real failure exits 1.
|
|
289
|
+
|
|
290
|
+
**Trying a local build in a project** (before a release):
|
|
291
|
+
|
|
292
|
+
```bash
|
|
293
|
+
# in the testforge repo
|
|
294
|
+
npm run build && npm pack
|
|
295
|
+
# in the project
|
|
296
|
+
npm i --no-save /path/to/activate-core-test-forge-<version>.tgz
|
|
297
|
+
```
|
|
277
298
|
|
|
278
299
|
A typical run:
|
|
279
300
|
|
package/dist/cli/index.js
CHANGED
|
@@ -93,7 +93,7 @@ function printUsage() {
|
|
|
93
93
|
console.log(' testforge generate --basic Use the basic generator only');
|
|
94
94
|
console.log(' testforge generate --force Regenerate everything');
|
|
95
95
|
console.log(' testforge generate --no-wait Stop when the AI hits its usage limit (default: wait for the reset and resume)');
|
|
96
|
-
console.log(' testforge generate --plan-only Plan and check each journey (testforge/
|
|
96
|
+
console.log(' testforge generate --plan-only Plan and check each journey (testforge/journeys/<area>/<id>/plan-check.md), write no tests', ' testforge generate --journey <name> --refresh-code Read the code again and rewrite the tests still waiting on names (after the feature code lands)');
|
|
97
97
|
console.log(' testforge generate --journey <name> Generate just ONE journey');
|
|
98
98
|
console.log(' testforge generate --area <area> Generate a whole area (e.g. api)');
|
|
99
99
|
console.log(' testforge discover Draft journey docs (asks: map or reference)');
|
|
@@ -249,6 +249,7 @@ async function main() {
|
|
|
249
249
|
? true
|
|
250
250
|
: undefined, // undefined -> config.waitForLimit (default on)
|
|
251
251
|
planOnly: process.argv.includes('--plan-only'),
|
|
252
|
+
refreshCode: process.argv.includes('--refresh-code'),
|
|
252
253
|
});
|
|
253
254
|
if (results.length === 0) {
|
|
254
255
|
console.log('No journey files found in testforge/docs/. Add a .md file and try again.');
|
|
@@ -62,7 +62,7 @@ function locate(ref, roots, all) {
|
|
|
62
62
|
* cites (around the cited lines), then every file that contains a text the
|
|
63
63
|
* doc quotes, most hits first, until `budget` characters.
|
|
64
64
|
*/
|
|
65
|
-
export function sourceExcerpts(markdown, roots, budget =
|
|
65
|
+
export function sourceExcerpts(markdown, roots, budget = 70_000) {
|
|
66
66
|
const windows = new Map();
|
|
67
67
|
const add = (file, from, to) => {
|
|
68
68
|
const w = windows.get(file) ?? [];
|
|
@@ -95,6 +95,54 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
95
95
|
const p = relative(process.cwd(), f).toLowerCase();
|
|
96
96
|
return [...new Set(controlWords)].some((w) => p.includes(w));
|
|
97
97
|
});
|
|
98
|
+
// A feature's own code is often named after the feature ("Smart Assignment"
|
|
99
|
+
// → app/Service/Assignment/…) and the doc names none of its classes: the
|
|
100
|
+
// journey's title finds it, and the classes the rest of the app calls are
|
|
101
|
+
// the way in (a search, a controller, a setting).
|
|
102
|
+
const titleWords = ((/^#\s*Journey:\s*(.+)$/m.exec(markdown)?.[1] ?? '').toLowerCase().match(/[a-z]{6,}/g) ?? [])
|
|
103
|
+
.filter((w) => !GENERIC.has(w));
|
|
104
|
+
if (titleWords.length) {
|
|
105
|
+
const texts = new Map();
|
|
106
|
+
const textOf = (f) => {
|
|
107
|
+
if (!texts.has(f)) {
|
|
108
|
+
try {
|
|
109
|
+
texts.set(f, readFileSync(f, 'utf-8'));
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
texts.set(f, '');
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return texts.get(f);
|
|
116
|
+
};
|
|
117
|
+
const own = files.filter((f) => {
|
|
118
|
+
const p = relative(process.cwd(), f).toLowerCase();
|
|
119
|
+
return !p.startsWith('..') && !/(^|[\\/])(tests?|__tests__|testforge|node_modules|vendor)[\\/]/.test(p) && titleWords.some((w) => p.includes(w));
|
|
120
|
+
});
|
|
121
|
+
const stem = (f) => f.replace(/^.*[\\/]/, '').replace(/\.\w+$/, '');
|
|
122
|
+
const usersOf = (f) => {
|
|
123
|
+
const name = stem(f);
|
|
124
|
+
if (name.length < 4 || name === 'index')
|
|
125
|
+
return [];
|
|
126
|
+
const re = new RegExp(`(?:\\b(?:use|import)\\s[^;\\n]*\\b${name}\\b|\\b${name}::|\\bnew\\s+\\\\?[\\w\\\\]*\\b${name}\\s*\\(|from\\s+['"][^'"]*\\b${name}['"])`);
|
|
127
|
+
return files.filter((u) => u !== f && !own.includes(u)).flatMap((u) => {
|
|
128
|
+
const lines = textOf(u).split('\n').flatMap((l, i) => (re.test(l) || (l.includes(name) && /::|->|new\s/.test(l)) ? [i + 1] : []));
|
|
129
|
+
return lines.length && re.test(textOf(u)) ? [{ file: u, lines }] : [];
|
|
130
|
+
});
|
|
131
|
+
};
|
|
132
|
+
// The feature's files most used elsewhere first: those are its entry points.
|
|
133
|
+
const ranked = own.map((f) => ({ f, users: usersOf(f) })).sort((a, b) => b.users.length - a.users.length);
|
|
134
|
+
for (const { f, users } of ranked.slice(0, 10)) {
|
|
135
|
+
add(f, 1, 60);
|
|
136
|
+
if (!order.includes(f))
|
|
137
|
+
order.push(f);
|
|
138
|
+
for (const u of users.slice(0, 6)) {
|
|
139
|
+
for (const n of u.lines.slice(0, 8))
|
|
140
|
+
add(u.file, n - 12, n + 12);
|
|
141
|
+
if (!order.includes(u.file))
|
|
142
|
+
order.push(u.file);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
98
146
|
// How a component is reached and fed: the files that import it, up to the
|
|
99
147
|
// page that serves it (its route), and the hooks and API modules they use.
|
|
100
148
|
const importersOf = (target) => {
|
|
@@ -207,7 +255,8 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
207
255
|
// word with the doc, first in the excerpts. A doc rarely names a route; a
|
|
208
256
|
// test cannot call one without its real path.
|
|
209
257
|
const words = [...new Set((markdown.toLowerCase().match(/[a-z]{5,}/g) ?? []).filter((w) => !GENERIC.has(w)))];
|
|
210
|
-
|
|
258
|
+
// A stem too short to search on ("book" from booking) keeps its word.
|
|
259
|
+
const stems = [...new Set(words.map((w) => { const st = w.replace(/(?:ings?|ed|es|s)$/, ''); return st.length >= 5 ? st : w; }))];
|
|
211
260
|
const ROUTE_FILE = /(^|[\\/])(routes?|router|urls)([\\/]|\.[a-z]+$)|\.routes?\.[a-z]+$|[\\/]routes?\.[a-z]+$/i;
|
|
212
261
|
const ROUTE_LINE = /Route::\w+\(|\brouter\.(?:get|post|put|patch|delete|all|use)\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(/;
|
|
213
262
|
// Only this package's own routes: a related repo's are another area's.
|
|
@@ -216,6 +265,7 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
216
265
|
// most 120 across the project; the prefix and group lines that complete
|
|
217
266
|
// their paths come with them. Commented-out routes are not routes.
|
|
218
267
|
const candidates = [];
|
|
268
|
+
const docLow = markdown.toLowerCase().replace(/[-_]/g, ' ');
|
|
219
269
|
for (const f of routeFiles.slice(0, 8)) {
|
|
220
270
|
let lines;
|
|
221
271
|
try {
|
|
@@ -227,20 +277,94 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
227
277
|
lines.forEach((l, i) => {
|
|
228
278
|
if (/^\s*(?:\/\/|#|\*)/.test(l))
|
|
229
279
|
return;
|
|
230
|
-
|
|
280
|
+
// The controller class is on every route it serves: only the path and method tell routes apart.
|
|
281
|
+
const low = l.toLowerCase().replace(/[\w\\]*controller(?=::class|@)/g, '');
|
|
231
282
|
if (/->prefix\(|::prefix\(|\bprefix\s*[:=]|->group\(|::group\(/i.test(l)) {
|
|
232
283
|
add(f, i + 1, i + 1);
|
|
233
284
|
return;
|
|
234
285
|
}
|
|
235
286
|
if (!ROUTE_LINE.test(l))
|
|
236
287
|
return;
|
|
237
|
-
|
|
288
|
+
// A path the doc writes as words ("booking order" for booking-order) is
|
|
289
|
+
// a route its tests call: ahead of lines that only share words.
|
|
290
|
+
const path = /['"]([\w/{}.-]*[-_][\w/{}.-]*)['"]/.exec(l)?.[1] ?? '';
|
|
291
|
+
const named = path.toLowerCase().split('/').filter((seg) => /[-_]/.test(seg) && docLow.includes(seg.replace(/[-_]/g, ' '))).length;
|
|
292
|
+
const score = stems.filter((w) => low.includes(w)).length + 10 * named;
|
|
238
293
|
if (score)
|
|
239
294
|
candidates.push({ file: f, line: i + 1, score });
|
|
240
295
|
});
|
|
241
296
|
}
|
|
242
|
-
for (const c of candidates.sort((a, b) => b.score - a.score).slice(0,
|
|
297
|
+
for (const c of candidates.sort((a, b) => b.score - a.score).slice(0, 60))
|
|
243
298
|
add(c.file, c.line, c.line);
|
|
299
|
+
// What a request must carry is in its handler: the start of the method a
|
|
300
|
+
// route names (Laravel's [Controller::class, 'method'] or 'Controller@method')
|
|
301
|
+
// holds its validation rules, so a test sends the real fields.
|
|
302
|
+
const handlers = new Set();
|
|
303
|
+
const handlerFiles = [];
|
|
304
|
+
const seenRules = new Set();
|
|
305
|
+
// Only the routes the doc names by their path: theirs are the requests its tests send.
|
|
306
|
+
for (const c of candidates.filter((x) => x.score >= 10).slice(0, 12)) {
|
|
307
|
+
let line = '';
|
|
308
|
+
try {
|
|
309
|
+
line = readFileSync(c.file, 'utf-8').split('\n')[c.line - 1] ?? '';
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
const m = /\[\s*\\?([\w\\]*?)(\w+Controller)::class\s*,\s*['"](\w+)['"]\s*\]/.exec(line) ?? /['"](\w+Controller)@(\w+)['"]/.exec(line);
|
|
315
|
+
if (!m)
|
|
316
|
+
continue;
|
|
317
|
+
const [cls, method] = m.length === 4 ? [m[2], m[3]] : [m[1], m[2]];
|
|
318
|
+
if (handlers.has(`${cls}@${method}`))
|
|
319
|
+
continue;
|
|
320
|
+
handlers.add(`${cls}@${method}`);
|
|
321
|
+
for (const f of files.filter((x) => x.endsWith(`${cls}.php`))) {
|
|
322
|
+
let text;
|
|
323
|
+
try {
|
|
324
|
+
text = readFileSync(f, 'utf-8').split('\n');
|
|
325
|
+
}
|
|
326
|
+
catch {
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
const at = text.findIndex((l) => new RegExp(`function\\s+${method}\\s*\\(`).test(l));
|
|
330
|
+
if (at === -1)
|
|
331
|
+
continue;
|
|
332
|
+
add(f, at + 1, at + 45);
|
|
333
|
+
if (!handlerFiles.includes(f))
|
|
334
|
+
handlerFiles.push(f);
|
|
335
|
+
// A handler that hands the request to a class it is given
|
|
336
|
+
// (bookingOrderV2(Request $r, OrderLogicV2 $logic)) keeps its rules there.
|
|
337
|
+
const sig = text[at].match(/\(([^)]*)\)/)?.[1] ?? '';
|
|
338
|
+
for (const dep of [...sig.matchAll(/\b([A-Z]\w+)\s+\$\w+/g)].map((d) => d[1]).filter((d) => d !== 'Request'))
|
|
339
|
+
rulesOf(dep, 2);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
// The rule methods of a class (validate, rules, getXRules), and of the
|
|
343
|
+
// classes its constructor is given, `depth` hops down.
|
|
344
|
+
function rulesOf(cls, depth) {
|
|
345
|
+
if (!depth || seenRules.has(cls))
|
|
346
|
+
return;
|
|
347
|
+
seenRules.add(cls);
|
|
348
|
+
for (const f of files.filter((x) => x.endsWith(`/${cls}.php`))) {
|
|
349
|
+
let text;
|
|
350
|
+
try {
|
|
351
|
+
text = readFileSync(f, 'utf-8').split('\n');
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
text.forEach((l, i) => {
|
|
357
|
+
if (/function\s+(?:validate|rules|\w*Rules)\s*\(/.test(l))
|
|
358
|
+
add(f, i + 1, i + 50);
|
|
359
|
+
const ctor = /function\s+__construct\s*\(([^)]*)\)/.exec(l);
|
|
360
|
+
if (ctor)
|
|
361
|
+
for (const d of ctor[1].matchAll(/\b([A-Z]\w+)\s+\$\w+/g))
|
|
362
|
+
rulesOf(d[1], depth - 1);
|
|
363
|
+
});
|
|
364
|
+
if (windows.has(f) && !handlerFiles.includes(f))
|
|
365
|
+
handlerFiles.push(f);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
244
368
|
for (const f of routeFiles.slice(0, 8)) {
|
|
245
369
|
if (candidates.some((c) => c.file === f) && windows.has(f)) {
|
|
246
370
|
const at = order.indexOf(f);
|
|
@@ -249,6 +373,16 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
249
373
|
order.unshift(f);
|
|
250
374
|
}
|
|
251
375
|
}
|
|
376
|
+
// Handlers and their rules right after the routes, ahead of the budget cut:
|
|
377
|
+
// without them a test sends fields the product does not take.
|
|
378
|
+
const firstOther = order.findIndex((f) => !routeFiles.includes(f) || !candidates.some((c) => c.file === f));
|
|
379
|
+
const handlersIn = handlerFiles.filter((f) => windows.has(f));
|
|
380
|
+
for (const f of handlersIn) {
|
|
381
|
+
const i = order.indexOf(f);
|
|
382
|
+
if (i !== -1)
|
|
383
|
+
order.splice(i, 1);
|
|
384
|
+
}
|
|
385
|
+
order.splice(firstOther === -1 ? order.length : Math.min(firstOther, order.length), 0, ...handlersIn);
|
|
252
386
|
// A page's own parts (its total, its operator card, its pickers) live in the
|
|
253
387
|
// components it imports from its own folder. Their visible texts and labels
|
|
254
388
|
// come right after it, ahead of files that only share a word with the doc.
|
|
@@ -17,12 +17,12 @@
|
|
|
17
17
|
* journey document remains the sole source of what is CORRECT, and the
|
|
18
18
|
* generation prompt says so in the same breath as it presents these facts.
|
|
19
19
|
*/
|
|
20
|
+
import { createHash } from 'node:crypto';
|
|
20
21
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
21
22
|
import { dirname, join } from 'node:path';
|
|
22
23
|
import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
23
24
|
import { readRules, ruleIndex } from '../generate/journey-memory.js';
|
|
24
25
|
import { sourceExcerpts } from './excerpts.js';
|
|
25
|
-
import { hashProjectSource } from './cache.js';
|
|
26
26
|
import { journeyDir } from '../paths.js';
|
|
27
27
|
import { loadEnvFile } from '../project/config.js';
|
|
28
28
|
import { originsFrom, pathsFrom, observeApp } from './observe.js';
|
|
@@ -37,7 +37,7 @@ const MAP_PATH = 'testforge.md';
|
|
|
37
37
|
* that upgrades keeps serving the facts it cached under the old version —
|
|
38
38
|
* so the new facts never reach a single prompt and the upgrade does nothing.
|
|
39
39
|
*/
|
|
40
|
-
const FACTS_VERSION =
|
|
40
|
+
const FACTS_VERSION = 11;
|
|
41
41
|
/** Where a journey's grounded facts live, mirroring the plan cache layout. */
|
|
42
42
|
function groundingPath(journey) {
|
|
43
43
|
return join(journeyDir(journey), 'facts.json');
|
|
@@ -107,7 +107,57 @@ function compactJourney(markdown) {
|
|
|
107
107
|
const m = readRules(markdown);
|
|
108
108
|
if (!m.rules.length)
|
|
109
109
|
return markdown;
|
|
110
|
-
|
|
110
|
+
// The setup names the requests the tests send (booking, payment): without it
|
|
111
|
+
// their routes look unused and drop out of the facts.
|
|
112
|
+
return [m.header, ``, `CONTROLS:`, ...Object.entries(m.controls).map(([id, c]) => `${id}: ${c}`), ``, `RULES:`, ruleIndex(m), ...(m.setup ? [``, `TEST SETUP:`, m.setup] : [])].join('\n');
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The code this journey reads, as the excerpts collect it (no AI): only a
|
|
116
|
+
* change to THESE files moves the journey's facts. A commit elsewhere in the
|
|
117
|
+
* project costs nothing; a feature's code arriving changes them.
|
|
118
|
+
*/
|
|
119
|
+
function journeyCode(journey, srcDir) {
|
|
120
|
+
let excerpts = '';
|
|
121
|
+
try {
|
|
122
|
+
excerpts = sourceExcerpts(journey.rawMarkdown || '', [srcDir, ...(loadConfig().relatedDirs ?? [])]);
|
|
123
|
+
}
|
|
124
|
+
catch { /* unreadable tree: no code */ }
|
|
125
|
+
return { excerpts, hash: createHash('sha1').update(excerpts).digest('hex').slice(0, 16) };
|
|
126
|
+
}
|
|
127
|
+
/** True when the journey's facts were read from code that has since changed. */
|
|
128
|
+
export function factsCodeMoved(journey, srcDir = 'src') {
|
|
129
|
+
const path = groundingPath(journey);
|
|
130
|
+
if (!existsSync(path))
|
|
131
|
+
return false;
|
|
132
|
+
try {
|
|
133
|
+
const key = String(JSON.parse(readFileSync(path, 'utf-8')).key ?? '');
|
|
134
|
+
return !key.endsWith(`:${journeyCode(journey, srcDir).hash}`);
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* True when the journey's tests were last written with no code to read (the
|
|
142
|
+
* feature was not built, or none of its code was found). Once code is there,
|
|
143
|
+
* those tests are written again from it — their "how" was a guess.
|
|
144
|
+
*/
|
|
145
|
+
export function writtenWithoutCode(journey) {
|
|
146
|
+
try {
|
|
147
|
+
return JSON.parse(readFileSync(groundingPath(journey), 'utf-8')).noFacts === true;
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/** Record that this journey's tests were written with no code to read. */
|
|
154
|
+
export function markWrittenWithoutCode(journey, srcDir = 'src') {
|
|
155
|
+
const path = groundingPath(journey);
|
|
156
|
+
try {
|
|
157
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
158
|
+
writeFileSync(path, JSON.stringify({ key: `v${FACTS_VERSION}::${journeyCode(journey, srcDir).hash}`, facts: '', noFacts: true }, null, 2), 'utf-8');
|
|
159
|
+
}
|
|
160
|
+
catch { /* best effort */ }
|
|
111
161
|
}
|
|
112
162
|
/**
|
|
113
163
|
* Establish (or reuse) the journey's code facts. Cached against the doc hash
|
|
@@ -119,21 +169,21 @@ function compactJourney(markdown) {
|
|
|
119
169
|
*/
|
|
120
170
|
export async function groundJourney(journey, docHash, srcDir = 'src', markdown = '',
|
|
121
171
|
/** A doc edit, not a new journey: reuse the facts already read, whatever version of the doc they came from. */
|
|
122
|
-
reuseOnly = false
|
|
172
|
+
reuseOnly = false,
|
|
173
|
+
/** Read the code again whatever is cached (generate --refresh-code). */
|
|
174
|
+
fresh = false) {
|
|
123
175
|
const path = groundingPath(journey);
|
|
124
176
|
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
sourceHash = hashProjectSource(srcDir);
|
|
128
|
-
}
|
|
129
|
-
catch {
|
|
130
|
-
// unreadable source tree — fall back to keying on the doc alone
|
|
131
|
-
}
|
|
177
|
+
const code = journeyCode(journey, srcDir);
|
|
178
|
+
const sourceHash = code.hash;
|
|
132
179
|
const key = `v${FACTS_VERSION}:${docHash}:${sourceHash}`;
|
|
133
|
-
if (existsSync(path)) {
|
|
180
|
+
if (existsSync(path) && !fresh) {
|
|
134
181
|
try {
|
|
135
182
|
const cached = JSON.parse(readFileSync(path, 'utf-8'));
|
|
136
|
-
|
|
183
|
+
// A doc edit reuses facts from an earlier version of the doc, but only
|
|
184
|
+
// while the code is the same: new code is exactly what the facts are for.
|
|
185
|
+
const sameCode = String(cached.key).startsWith(`v${FACTS_VERSION}:`) && String(cached.key).endsWith(`:${sourceHash}`);
|
|
186
|
+
if (typeof cached.facts === 'string' && cached.facts.trim() && (cached.key === key || (reuseOnly && sameCode))) {
|
|
137
187
|
return cached.facts;
|
|
138
188
|
}
|
|
139
189
|
}
|
|
@@ -141,15 +191,27 @@ reuseOnly = false) {
|
|
|
141
191
|
// corrupt cache — re-ground
|
|
142
192
|
}
|
|
143
193
|
}
|
|
144
|
-
if (reuseOnly)
|
|
194
|
+
if (reuseOnly && !existsSync(path))
|
|
145
195
|
return null;
|
|
196
|
+
if (reuseOnly && !fresh) {
|
|
197
|
+
// A doc edit on unchanged code reuses the facts already read, even ones an
|
|
198
|
+
// older TestForge wrote: facts in an older shape beat writing with none.
|
|
199
|
+
try {
|
|
200
|
+
const cached = JSON.parse(readFileSync(path, 'utf-8'));
|
|
201
|
+
if (String(cached.key).endsWith(`:${sourceHash}`))
|
|
202
|
+
return typeof cached.facts === 'string' && cached.facts.trim() ? cached.facts : null;
|
|
203
|
+
}
|
|
204
|
+
catch { /* corrupt: read again */ }
|
|
205
|
+
}
|
|
146
206
|
const config = loadConfig();
|
|
147
207
|
// The source the doc points at, collected here, goes into one tool-less call.
|
|
148
208
|
// A doc that points at nothing findable gets no facts: its tests are written
|
|
149
209
|
// from the doc, which is the contract, rather than paying an AI to go looking.
|
|
150
|
-
const excerpts =
|
|
151
|
-
if (!excerpts)
|
|
210
|
+
const excerpts = code.excerpts;
|
|
211
|
+
if (!excerpts) {
|
|
212
|
+
markWrittenWithoutCode(journey, srcDir);
|
|
152
213
|
return null;
|
|
214
|
+
}
|
|
153
215
|
// Excerpts only from other packages (a backend matched on a quoted text) say
|
|
154
216
|
// nothing about this area's screens: a call on them returns "unknown" for
|
|
155
217
|
// every section and costs as much as a real one.
|
|
@@ -37,6 +37,8 @@ export async function readReference(reference) {
|
|
|
37
37
|
if (!text.trim())
|
|
38
38
|
fail(reference, 'it is empty');
|
|
39
39
|
read.set(reference, text);
|
|
40
|
+
if (existsSync(reference))
|
|
41
|
+
return text; // a file is its own copy
|
|
40
42
|
try {
|
|
41
43
|
const dir = join(RUNS_DIR, 'reference');
|
|
42
44
|
mkdirSync(dir, { recursive: true });
|
|
@@ -44,6 +44,15 @@ export function writeCoverageManifest(results) {
|
|
|
44
44
|
});
|
|
45
45
|
if (!journeys.length)
|
|
46
46
|
return;
|
|
47
|
+
// Journeys this run did not touch keep their earlier entry: the manifest is
|
|
48
|
+
// the project's coverage, not this run's.
|
|
49
|
+
try {
|
|
50
|
+
const prev = JSON.parse(readFileSync(COVERAGE_PATH, 'utf-8'));
|
|
51
|
+
for (const j of prev.journeys ?? [])
|
|
52
|
+
if (!journeys.some((n) => n.journey === j.journey))
|
|
53
|
+
journeys.push(j);
|
|
54
|
+
}
|
|
55
|
+
catch { /* no earlier manifest */ }
|
|
47
56
|
const totalReal = journeys.reduce((n, j) => n + j.real, 0);
|
|
48
57
|
const totalDeferred = journeys.reduce((n, j) => n + j.deferred, 0);
|
|
49
58
|
try {
|
|
@@ -490,6 +490,7 @@ function looksStructurallyClean(content) {
|
|
|
490
490
|
t.startsWith('function ') ||
|
|
491
491
|
t.startsWith('async ') ||
|
|
492
492
|
t.startsWith('require(') ||
|
|
493
|
+
/^(?:describe|test|it|beforeAll|beforeEach)\s*\(/.test(t) ||
|
|
493
494
|
t.startsWith("'use strict'") ||
|
|
494
495
|
t.startsWith('"use strict"');
|
|
495
496
|
if (!startsWithCode)
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { spawn } from 'node:child_process';
|
|
14
14
|
import { readFileSync, existsSync } from 'node:fs';
|
|
15
|
-
import { loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
|
|
15
|
+
import { isClaude, loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
|
|
16
16
|
import { runAiSync, noteAiCall, traceArgs, traceCall } from './ai-usage.js';
|
|
17
17
|
import { API, WEB, codeOutput, evidence } from '../prompts/blocks.js';
|
|
18
18
|
import { stripStrayComments } from '../gates/validate.js';
|
|
@@ -339,7 +339,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
339
339
|
``,
|
|
340
340
|
`NAMES THE PRODUCT GIVES. The doc says WHAT happens; the product names HOW to`,
|
|
341
341
|
`reach it — a route, a button's label, a command, a table or a field.`,
|
|
342
|
-
` - When the CODE FACTS show that name,
|
|
342
|
+
` - When the CODE FACTS show that name (in ROUTES, SHAPES or DATA), use it`,
|
|
343
|
+
` as written there: the request fields a handler validates, a table's`,
|
|
344
|
+
` columns, a status value. need() is never for a name the facts show.`,
|
|
343
345
|
` - When they do not (the code is not built yet, or the facts missed it),`,
|
|
344
346
|
` write the test anyway and take the name from need('<plain name>'):`,
|
|
345
347
|
` const { need } = require('@activate-core/test-forge/reporters/interface.cjs');`,
|
|
@@ -753,6 +755,9 @@ function writerCommand(prompt, memory) {
|
|
|
753
755
|
if (!memory)
|
|
754
756
|
return { ...withReadOnlyTools(cmd), input: prompt };
|
|
755
757
|
const { system, message } = splitShared(prompt);
|
|
758
|
+
// Only the Claude CLI takes a system prompt; any other AI gets the rules at the top of its input.
|
|
759
|
+
if (system && !isClaude(cmd.program))
|
|
760
|
+
return { ...withNoTools(cmd), input: `${system}\n\n${message}` };
|
|
756
761
|
return system ? { ...withNoTools(cmd, system), input: message } : { ...withNoTools(cmd), input: prompt };
|
|
757
762
|
}
|
|
758
763
|
/**
|
|
@@ -843,7 +848,8 @@ export async function writeFilesWithAI(files) {
|
|
|
843
848
|
const config = loadConfig();
|
|
844
849
|
const cmd = splitCommand(config.aiCommand);
|
|
845
850
|
const { program, args } = system ? withNoTools(cmd, system) : withNoTools(cmd);
|
|
846
|
-
|
|
851
|
+
const input = system && !isClaude(cmd.program) ? `${system}\n\n${message}` : message;
|
|
852
|
+
return splitBatchAnswer(await runWithStdin(program, args, input, config.aiTimeoutMs), files.map((f) => f.path));
|
|
847
853
|
}
|
|
848
854
|
/** Each file's code from a batch answer, by its marker line. */
|
|
849
855
|
export function splitBatchAnswer(out, paths) {
|
|
@@ -81,6 +81,7 @@ let traceSeq = 0;
|
|
|
81
81
|
const STAGES = [
|
|
82
82
|
[/^=== REPAIR ===$/m, 'repair'],
|
|
83
83
|
[/^You are a senior QA engineer\. Generate a complete/, 'write'],
|
|
84
|
+
[/^You are a senior QA engineer writing one .* test file/, 'write'],
|
|
84
85
|
[/^You are a senior test architect/, 'plan'],
|
|
85
86
|
[/^You are a senior engineer establishing the MECHANICS/, 'grounding'],
|
|
86
87
|
[/^You review ONE claimed product defect/, 'defect-review'],
|
|
@@ -128,7 +129,9 @@ export function traceCall(prompt, raw, started, error) {
|
|
|
128
129
|
}
|
|
129
130
|
});
|
|
130
131
|
const result = events.find((e) => e.type === 'result');
|
|
131
|
-
|
|
132
|
+
// Only a Claude event stream has typed events; another AI's JSON answer is the answer itself.
|
|
133
|
+
const isStream = events.some((e) => typeof e.type === 'string');
|
|
134
|
+
const text = isStream ? String(result?.result ?? '') : raw;
|
|
132
135
|
const ru = result?.usage ?? {};
|
|
133
136
|
const tokens = ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'].reduce((n, k) => n + (Number(ru[k]) || 0), 0);
|
|
134
137
|
const cost = Number(result?.total_cost_usd) || 0;
|
|
@@ -154,7 +157,7 @@ export function traceCall(prompt, raw, started, error) {
|
|
|
154
157
|
try {
|
|
155
158
|
mkdirSync(TRACE_DIR, { recursive: true });
|
|
156
159
|
writeFileSync(join(TRACE_DIR, `${name}.prompt.md`), prompt);
|
|
157
|
-
writeFileSync(join(TRACE_DIR, `${name}.${
|
|
160
|
+
writeFileSync(join(TRACE_DIR, `${name}.${isStream ? 'stream.jsonl' : 'out.txt'}`), raw);
|
|
158
161
|
const u = result?.usage ?? {};
|
|
159
162
|
appendFileSync(join(TRACE_DIR, 'index.jsonl'), JSON.stringify({
|
|
160
163
|
seq, stage, file: file ? basename(file) : undefined,
|
|
@@ -32,6 +32,20 @@ function withoutBlocks(content, problems) {
|
|
|
32
32
|
const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
|
|
33
33
|
return blocks.length ? spliceBlocks(content, blocks.map((block) => ({ block, text: '' }))) : content;
|
|
34
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* What an answer adds to the file: its test blocks for the rules asked for.
|
|
37
|
+
* An answer that came back as a whole file (imports, an outer describe) or
|
|
38
|
+
* repeats a test the file already has would otherwise land nested or twice.
|
|
39
|
+
*/
|
|
40
|
+
export function onlyNewTests(answer, file, rules) {
|
|
41
|
+
if (!/^\s*(?:import\s|const\s+\w+\s*=\s*require\(|(?:test\.)?describe\s*\()/m.test(answer))
|
|
42
|
+
return answer;
|
|
43
|
+
const have = new Set(findTestBlocks(file).map((b) => b.title));
|
|
44
|
+
return findTestBlocks(answer)
|
|
45
|
+
.filter((b) => !have.has(b.title) && (!b.rules.length || b.rules.some((r) => rules.includes(r))))
|
|
46
|
+
.map((b) => answer.slice(b.start, b.end).replace(/^\s*\n/, ''))
|
|
47
|
+
.join('\n\n');
|
|
48
|
+
}
|
|
35
49
|
/** Is the file complete as it is? */
|
|
36
50
|
export function fileIsComplete(content, rules, steps, knownTexts) {
|
|
37
51
|
const { cov, problems } = review(content, rules, steps, knownTexts);
|
|
@@ -46,12 +60,29 @@ export async function completeFile(content, ctx) {
|
|
|
46
60
|
const drop = (ps) => { for (const p of ps)
|
|
47
61
|
if (!dropped.has(p.title))
|
|
48
62
|
dropped.set(p.title, p.reason); };
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
63
|
+
const locked = ctx.locked ?? new Set();
|
|
64
|
+
const before = new Set(findTestBlocks(content).map((b) => b.title));
|
|
65
|
+
// A test that passed is never touched. A kept file's other tests are only
|
|
66
|
+
// reported: a check added later must not quietly shrink a working suite.
|
|
67
|
+
// Only the tests asked for again (redo) go, and so do new tests just written.
|
|
68
|
+
const removable = (p) => !locked.has(p.title) && (!ctx.kept || !before.has(p.title) || !!ctx.redo?.(p.block));
|
|
69
|
+
const redoBlocks = ctx.kept && ctx.redo
|
|
70
|
+
? findTestBlocks(content).filter((b) => !locked.has(b.title) && ctx.redo(b))
|
|
71
|
+
: [];
|
|
72
|
+
const redo = redoBlocks
|
|
73
|
+
.filter((b) => !first.problems.some((p) => p.block.start === b.start))
|
|
74
|
+
.map((b) => ({ title: b.title, rules: b.rules, reason: 'it waits on a name the code now has: written again from the code', block: b }));
|
|
75
|
+
const firstDrop = [...first.problems.filter(removable), ...redo];
|
|
76
|
+
const reported = first.problems.filter((p) => !removable(p));
|
|
77
|
+
if (reported.length) {
|
|
78
|
+
notes.push(`left ${reported.length} earlier test(s) in place that a check flags (${[...new Set(reported.map((p) => shortReason(p.reason)))].slice(0, 3).join('; ')}) — not removed, they were already in the file`);
|
|
79
|
+
}
|
|
80
|
+
if (first.cov.missing.length || firstDrop.length) {
|
|
81
|
+
out = withoutNames(withoutBlocks(content, firstDrop));
|
|
82
|
+
drop(firstDrop);
|
|
52
83
|
// Only rules the doc has are written again: an id with no text makes the writer invent one.
|
|
53
84
|
const known = new Set(ctx.memory.rules.filter((r) => r.tag).map((r) => r.id));
|
|
54
|
-
const left = ruleCoverage(out, ctx.rules).missing.filter((id) => known.has(id));
|
|
85
|
+
const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])].filter((id) => known.has(id) && ctx.rules.includes(id));
|
|
55
86
|
if (left.length) {
|
|
56
87
|
const blocks = findTestBlocks(out);
|
|
57
88
|
const head = out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000);
|
|
@@ -59,8 +90,8 @@ export async function completeFile(content, ctx) {
|
|
|
59
90
|
const slice = journeySlice(ctx.memory, left);
|
|
60
91
|
calls = 1;
|
|
61
92
|
try {
|
|
62
|
-
const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(
|
|
63
|
-
const candidate = appendTests(out, tests);
|
|
93
|
+
const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(firstDrop, first.cov));
|
|
94
|
+
const candidate = appendTests(out, onlyNewTests(tests, out, left));
|
|
64
95
|
const refused = ctx.check(candidate);
|
|
65
96
|
if (refused)
|
|
66
97
|
notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
|
|
@@ -71,13 +102,20 @@ export async function completeFile(content, ctx) {
|
|
|
71
102
|
notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
|
|
72
103
|
}
|
|
73
104
|
// What the second answer still gets wrong is removed, not sent again.
|
|
74
|
-
const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts });
|
|
105
|
+
const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts }).filter(removable);
|
|
75
106
|
if (second.length) {
|
|
76
107
|
out = withoutNames(withoutBlocks(out, second));
|
|
77
108
|
drop(second);
|
|
78
109
|
}
|
|
79
110
|
}
|
|
80
111
|
}
|
|
112
|
+
// The last word, checked mechanically: every test that passed is still here.
|
|
113
|
+
const lost = [...locked].filter((t) => before.has(t) && !findTestBlocks(out).some((b) => b.title === t));
|
|
114
|
+
if (lost.length) {
|
|
115
|
+
notes.push(`kept the file as it was: the change would have lost ${lost.length} test(s) that passed, e.g. "${lost[0].slice(0, 60)}"`);
|
|
116
|
+
out = content;
|
|
117
|
+
dropped.clear();
|
|
118
|
+
}
|
|
81
119
|
// A cannot-test line that is no longer true (its rule has a test now) or never
|
|
82
120
|
// was a reason ("the locator is unknown") does not stay in the file.
|
|
83
121
|
const has = new Set(findTestBlocks(out).flatMap((b) => b.rules));
|
|
@@ -246,7 +246,8 @@ export function oracleClaim(t, content) {
|
|
|
246
246
|
return `${t.title} ${NOT_CLAIMED}`;
|
|
247
247
|
}
|
|
248
248
|
/** A [spec] line that carries the client's words, quoted or pointing at the quote above it ("source as above"). */
|
|
249
|
-
|
|
249
|
+
// A quoted source, a ticket key on its own ("source: SHOP-1") — the ticket is the source — or "same as".
|
|
250
|
+
export const HAS_SOURCE = /\((?:source(?::\s*(?:[A-Z][A-Z0-9]+-\d+:?\s*)?["“]|:\s*[A-Z][A-Z0-9]+-\d+\b|:?\s*(?:as|same)\b)|same source\b)/i;
|
|
250
251
|
/** Playwright's report of a locator that matched more than one element. */
|
|
251
252
|
const AMBIGUOUS = /strict mode violation|resolved to \d+ elements/i;
|
|
252
253
|
/** An application file:line, the only evidence a defect may rest on. */
|
|
@@ -40,6 +40,11 @@ export const HARNESS_CONTRACT = [
|
|
|
40
40
|
`- resetRateLimits(): Promise<void> (H7)`,
|
|
41
41
|
`- REQUIRED_PACKAGES: string[] (H6)`,
|
|
42
42
|
`- a 'run' role alongside the fixture roles (H8)`,
|
|
43
|
+
`- insert(table, row): Promise<id>, one row of scenario data written straight`,
|
|
44
|
+
` into the test database (H10)`,
|
|
45
|
+
`- DERIVED and AREA (H10)`,
|
|
46
|
+
`- update(table, match, values): Promise<number>, columns set on the rows`,
|
|
47
|
+
` matching every key of match (H10)`,
|
|
43
48
|
`Jest areas only`,
|
|
44
49
|
`- call(method, path, { body?, headers?, role? }):`,
|
|
45
50
|
` Promise<{ status, headers, body }>`,
|
|
@@ -177,6 +182,35 @@ export const HARNESS_CONTRACT = [
|
|
|
177
182
|
` 'unknown step "<name>" (steps: …)'. If there is no such flow, export`,
|
|
178
183
|
` STEPS = {} and have openStep resolve`,
|
|
179
184
|
` 'this product has no multi-step flow'.`,
|
|
185
|
+
``,
|
|
186
|
+
`H10. Scenario data. A scenario describes people and their state (an operator`,
|
|
187
|
+
` with a location, services, working hours and earlier jobs; a setting`,
|
|
188
|
+
` switched on). insert() and update() let a test build exactly that:`,
|
|
189
|
+
` - insert fills every NOT NULL column without a default that row leaves out`,
|
|
190
|
+
` with a value of its type (read the columns from the database), returns`,
|
|
191
|
+
` the new id, and remembers the row.`,
|
|
192
|
+
` - update remembers the previous values of the rows it changes.`,
|
|
193
|
+
` - Each test starts from a clean slate: the rows a test writes, and what it`,
|
|
194
|
+
` updates, are undone right after that test (last first), so one test's`,
|
|
195
|
+
` data never takes part in the next. The testkit does this itself when it`,
|
|
196
|
+
` is loaded: it notes where the record stands before each test and undoes`,
|
|
197
|
+
` back to that point after it (Jest: the global beforeEach/afterEach;`,
|
|
198
|
+
` Playwright: test.beforeEach/afterEach). Tests do nothing for it.`,
|
|
199
|
+
` - What a beforeAll writes stays for the whole file; disconnect() undoes it`,
|
|
200
|
+
` and anything left, so every run leaves the database as it found it.`,
|
|
201
|
+
` - Both refuse a database that is not clearly a test one, like reset() (H5).`,
|
|
202
|
+
` - Some values a scenario sets are not columns: the product works them out`,
|
|
203
|
+
` from other records (a standing from past jobs, an average per day, a`,
|
|
204
|
+
` count). Read in the code how each is worked out. insert() accepts each as`,
|
|
205
|
+
` a virtual column named _tf_<name> (lower case, underscores) and, instead`,
|
|
206
|
+
` of writing it, writes the records that give exactly that value (and are`,
|
|
207
|
+
` undone with the row). Export DERIVED: { _tf_<name>: 'how it is made' }`,
|
|
208
|
+
` for each, so tests can use them.`,
|
|
209
|
+
` - Export AREA: { latitude, longitude } (or the project's own key, when`,
|
|
210
|
+
` the product picks among records by something else): a place no record`,
|
|
211
|
+
` already in the database reaches, found by querying it. Scenarios where`,
|
|
212
|
+
` the product picks one record among many are built there, so data that`,
|
|
213
|
+
` was already in the database never takes part.`,
|
|
180
214
|
].join('\n');
|
|
181
215
|
/**
|
|
182
216
|
* Stamp written into every generated testkit: the contract it was built
|