@activate-core/test-forge 2.4.6 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -11
- package/dist/cli/index.js +12 -12
- package/dist/core/docs/cache.js +1 -1
- package/dist/core/docs/discover.js +5 -5
- package/dist/core/docs/excerpts.js +139 -5
- package/dist/core/docs/grounding.js +78 -16
- package/dist/core/docs/reference-text.js +310 -52
- package/dist/core/gates/coverage.js +9 -0
- package/dist/core/gates/validate.js +1 -0
- package/dist/core/generate/ai-generator.js +9 -3
- package/dist/core/generate/ai-usage.js +11 -4
- package/dist/core/generate/bind.js +3 -2
- package/dist/core/generate/complete-file.js +45 -7
- package/dist/core/generate/defect-review.js +2 -1
- package/dist/core/generate/harness.js +34 -0
- package/dist/core/generate/interface-file.js +12 -0
- package/dist/core/generate/journey-memory.js +7 -1
- package/dist/core/generate/passed.js +24 -0
- package/dist/core/generate/review-journey.js +2 -2
- package/dist/core/generate/run-generate.js +117 -13
- package/dist/core/generate/suite-planner.js +2 -2
- package/dist/core/generate/test-blocks.js +34 -10
- package/dist/core/paths.js +41 -6
- package/dist/core/project/config.js +5 -0
- package/dist/core/project/setup.js +9 -9
- package/dist/core/project/update.js +6 -6
- package/dist/core/prompts/blocks.js +24 -3
- package/dist/core/run/name-scan.js +120 -14
- package/dist/core/run/report.js +7 -4
- package/dist/core/run/run-tests.js +53 -4
- package/package.json +3 -2
- package/reporters/_shared.cjs +56 -3
- package/reporters/interface.cjs +14 -0
- package/reporters/jest-reporter.cjs +7 -3
- package/reporters/playwright-reporter.cjs +6 -2
- package/reporters/run-data.cjs +6 -1
package/README.md
CHANGED
|
@@ -25,7 +25,7 @@ testforge setup # one-time: inspect project, map it, draft journey docs
|
|
|
25
25
|
testforge generate # generate tests + seed files from the docs
|
|
26
26
|
````
|
|
27
27
|
|
|
28
|
-
After `setup`, review the drafted docs in `docs/` and fix anything the
|
|
28
|
+
After `setup`, review the drafted docs in `testforge/docs/` and fix anything the
|
|
29
29
|
AI got wrong — they are the source of truth for the tests. Then run
|
|
30
30
|
`generate`.
|
|
31
31
|
|
|
@@ -77,7 +77,23 @@ When a reference really is the whole product's new specification, say so:
|
|
|
77
77
|
testforge discover --reference ./docs/prd-v2.md --replace
|
|
78
78
|
```
|
|
79
79
|
|
|
80
|
-
A
|
|
80
|
+
**A reference is read fresh on every `discover`, never kept.** Only its link is
|
|
81
|
+
stored (`testforge/source.json`), so an edited Notion page is always the one
|
|
82
|
+
used. `generate` and `run` never read the reference. No AI ever re-types it:
|
|
83
|
+
|
|
84
|
+
- **Notion page:** a short call on Claude's smallest model only runs the Notion
|
|
85
|
+
fetch tool (about $0.01); the text is taken word for word from the tool's
|
|
86
|
+
result, however long the page. Child pages are found by TestForge, fetched
|
|
87
|
+
(one more call per level) and read in place.
|
|
88
|
+
- **File:** read as it is; a PDF through `pdftotext` (poppler-utils).
|
|
89
|
+
- **Any other link:** downloaded directly, no AI.
|
|
90
|
+
|
|
91
|
+
Nothing is ever quietly left out. A page Notion cuts short, a child page that
|
|
92
|
+
could not be fetched, a database on the page, a link behind a sign-in or a
|
|
93
|
+
binary file stops with an error that says what to do; images and attachments
|
|
94
|
+
on a Notion page are listed as not read.
|
|
95
|
+
|
|
96
|
+
A journey **is** its doc: `generate` reads `testforge/docs/` and treats every markdown
|
|
81
97
|
file there as one journey. So a doc you write by hand — for a feature the
|
|
82
98
|
reference does not cover yet, or one taken from a ticket — works exactly like a
|
|
83
99
|
drafted one. Write the doc, then `generate --journey <name>`.
|
|
@@ -103,7 +119,7 @@ never touched.
|
|
|
103
119
|
| -------------------- | ----------------------------------------------------------- |
|
|
104
120
|
| `testforge init` | Inspect the project and write `testforge.config.json` only. |
|
|
105
121
|
| `testforge map` | Rebuild the project map (`testforge.md`) only. |
|
|
106
|
-
| `testforge discover` | Redraft the per-journey docs in `docs/` only.
|
|
122
|
+
| `testforge discover` | Redraft the per-journey docs in `testforge/docs/` only. |
|
|
107
123
|
| `testforge discover --check --journey <name>` | Check a doc that already exists (a hand-written one too) and add what it lacks — the baseline cases its controls need, the branches the code shows. Nothing is redrafted. |
|
|
108
124
|
| `testforge env` | Rebuild `.env.testing.example` from the docs only. |
|
|
109
125
|
| `testforge run` | Run the generated tests (`--journey` / `--area` / `--env-file`). First fills the names tests wait for from the code (only where each kind of name lives, only when those lines changed); `--no-scan` skips that. |
|
|
@@ -118,8 +134,7 @@ never touched.
|
|
|
118
134
|
| `--journey <name>` | discover, generate, run | Work on one journey instead of all of them. |
|
|
119
135
|
| `--area <name>` | discover, generate, run | Work on one area (api, web, ops…). |
|
|
120
136
|
| `--force` | generate | Regenerate even when the doc has not changed. |
|
|
121
|
-
| `--plan-only` | generate | Plan and check each journey (`testforge/
|
|
122
|
-
| `--refresh` | discover | Fetch the reference again instead of the copy kept from the last run. |
|
|
137
|
+
| `--plan-only` | generate | Plan and check each journey (`testforge/journeys/<area>/<id>/plan-check.md`), write no tests. |
|
|
123
138
|
| `--no-scan` | run | Do not look for blank interface names in the code before the run. |
|
|
124
139
|
| `--refresh-docs` | update | Re-draft the docs even when nothing detectably changed — see below. |
|
|
125
140
|
| `--reference <file\|url>` | setup | Base the docs on a spec instead of the code map. |
|
|
@@ -156,7 +171,7 @@ No source changes detected.
|
|
|
156
171
|
`testforge.md`: a short list of journeys and a summary of the data layer
|
|
157
172
|
(database, models, key fields). The config is held to a closed list of
|
|
158
173
|
keys, and a framework other than `jest` or `playwright` is refused.
|
|
159
|
-
2. **discover** — writes one doc per journey into `docs/`.
|
|
174
|
+
2. **discover** — writes one doc per journey into `testforge/docs/`.
|
|
160
175
|
|
|
161
176
|
**The journey doc.** Every criterion is one line with its own id, its tier and
|
|
162
177
|
the controls it covers:
|
|
@@ -205,7 +220,7 @@ cannot read is reported as exactly that, never as "up to date".
|
|
|
205
220
|
|
|
206
221
|
## Source of truth
|
|
207
222
|
|
|
208
|
-
The journey docs in `docs/` are the source of truth for the tests —
|
|
223
|
+
The journey docs in `testforge/docs/` are the source of truth for the tests —
|
|
209
224
|
`generate` reads the docs and nothing else. So where the docs come from
|
|
210
225
|
decides everything.
|
|
211
226
|
|
|
@@ -256,9 +271,30 @@ prints a note.
|
|
|
256
271
|
|
|
257
272
|
## Running the generated tests
|
|
258
273
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
274
|
+
Run them with `testforge run` (it fills waiting names from the code first,
|
|
275
|
+
holds back files whose feature is not live yet, and prints how many tests
|
|
276
|
+
ran of how many), or with your own test runner — the generated config
|
|
277
|
+
carries the TestForge reporter either way.
|
|
278
|
+
|
|
279
|
+
**What a run remembers.** Every test that passes is recorded in
|
|
280
|
+
`testforge/passed.json`. From then on `testforge generate` never removes or
|
|
281
|
+
rewrites it — a later failure is a regression to look at, not a test to
|
|
282
|
+
rewrite. Only `generate --force` writes such a test again. A kept file's
|
|
283
|
+
other tests are reported by the checks, never removed; once the code a
|
|
284
|
+
waiting test needs arrives, only that test is written again.
|
|
285
|
+
|
|
286
|
+
**Exit code.** A run whose only non-passing tests are waiting for the
|
|
287
|
+
product or stopped by the environment exits 0: nothing failed for a product
|
|
288
|
+
reason. A real failure exits 1.
|
|
289
|
+
|
|
290
|
+
**Trying a local build in a project** (before a release):
|
|
291
|
+
|
|
292
|
+
```bash
|
|
293
|
+
# in the testforge repo
|
|
294
|
+
npm run build && npm pack
|
|
295
|
+
# in the project
|
|
296
|
+
npm i --no-save /path/to/activate-core-test-forge-<version>.tgz
|
|
297
|
+
```
|
|
262
298
|
|
|
263
299
|
A typical run:
|
|
264
300
|
|
|
@@ -391,7 +427,7 @@ Test results are kept as a workflow artifact for 14 days.
|
|
|
391
427
|
## How `generate` keeps the AI cost down
|
|
392
428
|
|
|
393
429
|
The journey doc is read once and kept as the journey's **memory**
|
|
394
|
-
(`testforge/
|
|
430
|
+
(`testforge/journeys/<area>/<id>/`): its rules with their source, level, tests and
|
|
395
431
|
history, the plan, and the code facts. No later step sends the whole doc again.
|
|
396
432
|
|
|
397
433
|
1. **Rules.** The doc is split into rules (R1, R2 …) without AI. Rule ids stay
|
package/dist/cli/index.js
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
import { VERSION } from '../core/index.js';
|
|
22
22
|
import { runGenerate } from '../core/generate/run-generate.js';
|
|
23
23
|
import { runReview } from '../core/generate/review-journey.js';
|
|
24
|
-
import { migrateState } from '../core/paths.js';
|
|
24
|
+
import { DOCS_DIR, migrateState } from '../core/paths.js';
|
|
25
25
|
import { acceptNames, bindJourneys } from '../core/generate/bind.js';
|
|
26
26
|
import { runClean } from '../core/run/clean.js';
|
|
27
27
|
import { aiUsage, describeUsage, usageByStage } from '../core/generate/ai-usage.js';
|
|
@@ -64,7 +64,6 @@ function sourceOptions() {
|
|
|
64
64
|
reference,
|
|
65
65
|
source: flagValue('--source'),
|
|
66
66
|
replace: process.argv.includes('--replace'),
|
|
67
|
-
refresh: process.argv.includes('--refresh'),
|
|
68
67
|
};
|
|
69
68
|
}
|
|
70
69
|
/**
|
|
@@ -94,11 +93,11 @@ function printUsage() {
|
|
|
94
93
|
console.log(' testforge generate --basic Use the basic generator only');
|
|
95
94
|
console.log(' testforge generate --force Regenerate everything');
|
|
96
95
|
console.log(' testforge generate --no-wait Stop when the AI hits its usage limit (default: wait for the reset and resume)');
|
|
97
|
-
console.log(' testforge generate --plan-only Plan and check each journey (testforge/
|
|
96
|
+
console.log(' testforge generate --plan-only Plan and check each journey (testforge/journeys/<area>/<id>/plan-check.md), write no tests', ' testforge generate --journey <name> --refresh-code Read the code again and rewrite the tests still waiting on names (after the feature code lands)');
|
|
98
97
|
console.log(' testforge generate --journey <name> Generate just ONE journey');
|
|
99
98
|
console.log(' testforge generate --area <area> Generate a whole area (e.g. api)');
|
|
100
99
|
console.log(' testforge discover Draft journey docs (asks: map or reference)');
|
|
101
|
-
console.log(' testforge discover --journey <name> Draft just ONE journey doc', ' testforge discover --check --journey <name> Check an existing doc (hand-written too) and add what it lacks; nothing is redrafted', ' testforge discover --journey <name> --reference <x>
|
|
100
|
+
console.log(' testforge discover --journey <name> Draft just ONE journey doc', ' testforge discover --check --journey <name> Check an existing doc (hand-written too) and add what it lacks; nothing is redrafted', ' testforge discover --journey <name> --reference <x> One journey from a reference (read fresh each time)');
|
|
102
101
|
console.log(' testforge discover --reference <file|url> Base docs on a reference');
|
|
103
102
|
console.log(' testforge discover --tickets <folder> Base docs on exported tickets (one file per ticket); each source names its ticket');
|
|
104
103
|
console.log(' (a reference only writes the journeys it');
|
|
@@ -202,22 +201,22 @@ async function main() {
|
|
|
202
201
|
console.log('Usage: testforge discover --check --journey <name>');
|
|
203
202
|
}
|
|
204
203
|
else {
|
|
205
|
-
console.log(checkExistingDoc(
|
|
204
|
+
console.log(await checkExistingDoc(DOCS_DIR, journey, '.'));
|
|
206
205
|
}
|
|
207
206
|
}
|
|
208
207
|
else if (command === 'discover') {
|
|
209
208
|
console.log('Drafting journey files with AI...');
|
|
210
|
-
console.log('This writes drafts into docs/ from your chosen source.\n');
|
|
211
|
-
const output = await runDiscover('.',
|
|
209
|
+
console.log('This writes drafts into testforge/docs/ from your chosen source.\n');
|
|
210
|
+
const output = await runDiscover('.', DOCS_DIR, sourceOptions(), flagValue('--journey'));
|
|
212
211
|
console.log(output);
|
|
213
|
-
console.log('\nDraft journeys written to docs/. Please review and edit them,');
|
|
212
|
+
console.log('\nDraft journeys written to testforge/docs/. Please review and edit them,');
|
|
214
213
|
console.log('then run `testforge generate`.');
|
|
215
214
|
}
|
|
216
215
|
else if (command === 'env') {
|
|
217
216
|
if (process.argv.includes('--diff'))
|
|
218
217
|
console.log(describeEnvDiff(envDiff('.')));
|
|
219
218
|
else
|
|
220
|
-
runEnvTemplate(
|
|
219
|
+
runEnvTemplate(DOCS_DIR, '.env.testing.example', '.', generatedScanDirs());
|
|
221
220
|
}
|
|
222
221
|
else if (command === 'run') {
|
|
223
222
|
// Anything after `--`, plus the runner flags people reach for most, goes
|
|
@@ -243,16 +242,17 @@ async function main() {
|
|
|
243
242
|
console.log(useAI
|
|
244
243
|
? 'Generating tests with AI for changed journeys...\n'
|
|
245
244
|
: 'Generating tests with the basic generator...\n');
|
|
246
|
-
const results = await runGenerate(
|
|
245
|
+
const results = await runGenerate(DOCS_DIR, useAI, force, flagValue('--journey'), flagValue('--area'), {
|
|
247
246
|
waitForLimit: process.argv.includes('--no-wait')
|
|
248
247
|
? false
|
|
249
248
|
: process.argv.includes('--wait-for-limit')
|
|
250
249
|
? true
|
|
251
250
|
: undefined, // undefined -> config.waitForLimit (default on)
|
|
252
251
|
planOnly: process.argv.includes('--plan-only'),
|
|
252
|
+
refreshCode: process.argv.includes('--refresh-code'),
|
|
253
253
|
});
|
|
254
254
|
if (results.length === 0) {
|
|
255
|
-
console.log('No journey files found in docs/. Add a .md file and try again.');
|
|
255
|
+
console.log('No journey files found in testforge/docs/. Add a .md file and try again.');
|
|
256
256
|
}
|
|
257
257
|
else {
|
|
258
258
|
for (const r of results) {
|
|
@@ -311,7 +311,7 @@ async function main() {
|
|
|
311
311
|
// just `setup`) tells the developer exactly which credentials the
|
|
312
312
|
// generated suite needs — including vars the tests themselves read.
|
|
313
313
|
console.log('');
|
|
314
|
-
runEnvTemplate(
|
|
314
|
+
runEnvTemplate(DOCS_DIR, '.env.testing.example', '.', generatedScanDirs());
|
|
315
315
|
}
|
|
316
316
|
}
|
|
317
317
|
else if (command === 'report') {
|
package/dist/core/docs/cache.js
CHANGED
|
@@ -123,7 +123,7 @@ export function hashProjectSource(rootDir) {
|
|
|
123
123
|
* not checking at all — it reports "unchanged" while the specification moves,
|
|
124
124
|
* and "changed" every time the vendor ships. Refuse it and say why.
|
|
125
125
|
*/
|
|
126
|
-
function looksLikeAppShell(url, body) {
|
|
126
|
+
export function looksLikeAppShell(url, body) {
|
|
127
127
|
let host = '';
|
|
128
128
|
try {
|
|
129
129
|
host = new URL(url).hostname.toLowerCase();
|
|
@@ -154,7 +154,7 @@ export async function runDiscover(srcDir, docsDir, opts = {}, journey) {
|
|
|
154
154
|
// reference read once and cut to this journey, drafted and reviewed in one
|
|
155
155
|
// call each with no tools.
|
|
156
156
|
if (source.mode === 'reference' && source.reference && journey) {
|
|
157
|
-
return draftOneJourney(srcDir, docsDir, source.reference, journey
|
|
157
|
+
return draftOneJourney(srcDir, docsDir, source.reference, journey);
|
|
158
158
|
}
|
|
159
159
|
if (source.mode === 'reference' && source.reference) {
|
|
160
160
|
console.log(`\nComparing the reference against testforge.md: ${source.reference}`);
|
|
@@ -277,9 +277,9 @@ async function discoverFromCode(srcDir, docsDir, journey) {
|
|
|
277
277
|
}
|
|
278
278
|
const docTitle = (text) => /^# Journey:\s*(.+)$/m.exec(text)?.[1]?.trim();
|
|
279
279
|
const kebab = (t) => t.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
280
|
-
async function draftOneJourney(srcDir, docsDir, reference, journey
|
|
280
|
+
async function draftOneJourney(srcDir, docsDir, reference, journey) {
|
|
281
281
|
const config = loadConfig();
|
|
282
|
-
const text = readReference(reference
|
|
282
|
+
const text = await readReference(reference);
|
|
283
283
|
const section = sectionFor(text, journey);
|
|
284
284
|
const excerpts = sourceExcerpts(section, [srcDir, ...(config.relatedDirs ?? [])]);
|
|
285
285
|
const docs = [...snapshotDocs(docsDir)].filter(([p, t]) => !notAJourneyDoc(p, t));
|
|
@@ -338,7 +338,7 @@ function checkAndFinish(doc, referenceText, section, excerpts, others) {
|
|
|
338
338
|
* and add what it lacks: the baseline items its controls need, the branches the
|
|
339
339
|
* code shows. Nothing is redrafted; the doc's own criteria stay as written.
|
|
340
340
|
*/
|
|
341
|
-
export function checkExistingDoc(docsDir, journey, srcDir
|
|
341
|
+
export async function checkExistingDoc(docsDir, journey, srcDir) {
|
|
342
342
|
const config = loadConfig();
|
|
343
343
|
const docs = [...snapshotDocs(docsDir)].filter(([p, t]) => !notAJourneyDoc(p, t));
|
|
344
344
|
const hit = docs.find(([p, t]) => kebab(docTitle(t) ?? '') === kebab(journey) || p.replace(/\.md$/, '').endsWith(kebab(journey)));
|
|
@@ -347,7 +347,7 @@ export function checkExistingDoc(docsDir, journey, srcDir, refresh = false) {
|
|
|
347
347
|
const [rel, doc] = hit;
|
|
348
348
|
const stored = loadSourceChoice();
|
|
349
349
|
const reference = stored?.journeys?.[rel] ?? (stored?.mode === 'reference' ? stored.reference : undefined);
|
|
350
|
-
const text = reference ? readReference(reference
|
|
350
|
+
const text = reference ? await readReference(reference) : '';
|
|
351
351
|
const section = text ? sectionFor(text, docTitle(doc) ?? journey) : '';
|
|
352
352
|
const excerpts = sourceExcerpts(section || doc, [srcDir, ...(config.relatedDirs ?? [])]);
|
|
353
353
|
const others = docs.filter(([p]) => p !== rel).map(([p, t]) => `${docTitle(t) ?? p} (${p})`);
|
|
@@ -62,7 +62,7 @@ function locate(ref, roots, all) {
|
|
|
62
62
|
* cites (around the cited lines), then every file that contains a text the
|
|
63
63
|
* doc quotes, most hits first, until `budget` characters.
|
|
64
64
|
*/
|
|
65
|
-
export function sourceExcerpts(markdown, roots, budget =
|
|
65
|
+
export function sourceExcerpts(markdown, roots, budget = 70_000) {
|
|
66
66
|
const windows = new Map();
|
|
67
67
|
const add = (file, from, to) => {
|
|
68
68
|
const w = windows.get(file) ?? [];
|
|
@@ -95,6 +95,54 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
95
95
|
const p = relative(process.cwd(), f).toLowerCase();
|
|
96
96
|
return [...new Set(controlWords)].some((w) => p.includes(w));
|
|
97
97
|
});
|
|
98
|
+
// A feature's own code is often named after the feature ("Smart Assignment"
|
|
99
|
+
// → app/Service/Assignment/…) and the doc names none of its classes: the
|
|
100
|
+
// journey's title finds it, and the classes the rest of the app calls are
|
|
101
|
+
// the way in (a search, a controller, a setting).
|
|
102
|
+
const titleWords = ((/^#\s*Journey:\s*(.+)$/m.exec(markdown)?.[1] ?? '').toLowerCase().match(/[a-z]{6,}/g) ?? [])
|
|
103
|
+
.filter((w) => !GENERIC.has(w));
|
|
104
|
+
if (titleWords.length) {
|
|
105
|
+
const texts = new Map();
|
|
106
|
+
const textOf = (f) => {
|
|
107
|
+
if (!texts.has(f)) {
|
|
108
|
+
try {
|
|
109
|
+
texts.set(f, readFileSync(f, 'utf-8'));
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
texts.set(f, '');
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return texts.get(f);
|
|
116
|
+
};
|
|
117
|
+
const own = files.filter((f) => {
|
|
118
|
+
const p = relative(process.cwd(), f).toLowerCase();
|
|
119
|
+
return !p.startsWith('..') && !/(^|[\\/])(tests?|__tests__|testforge|node_modules|vendor)[\\/]/.test(p) && titleWords.some((w) => p.includes(w));
|
|
120
|
+
});
|
|
121
|
+
const stem = (f) => f.replace(/^.*[\\/]/, '').replace(/\.\w+$/, '');
|
|
122
|
+
const usersOf = (f) => {
|
|
123
|
+
const name = stem(f);
|
|
124
|
+
if (name.length < 4 || name === 'index')
|
|
125
|
+
return [];
|
|
126
|
+
const re = new RegExp(`(?:\\b(?:use|import)\\s[^;\\n]*\\b${name}\\b|\\b${name}::|\\bnew\\s+\\\\?[\\w\\\\]*\\b${name}\\s*\\(|from\\s+['"][^'"]*\\b${name}['"])`);
|
|
127
|
+
return files.filter((u) => u !== f && !own.includes(u)).flatMap((u) => {
|
|
128
|
+
const lines = textOf(u).split('\n').flatMap((l, i) => (re.test(l) || (l.includes(name) && /::|->|new\s/.test(l)) ? [i + 1] : []));
|
|
129
|
+
return lines.length && re.test(textOf(u)) ? [{ file: u, lines }] : [];
|
|
130
|
+
});
|
|
131
|
+
};
|
|
132
|
+
// The feature's files most used elsewhere first: those are its entry points.
|
|
133
|
+
const ranked = own.map((f) => ({ f, users: usersOf(f) })).sort((a, b) => b.users.length - a.users.length);
|
|
134
|
+
for (const { f, users } of ranked.slice(0, 10)) {
|
|
135
|
+
add(f, 1, 60);
|
|
136
|
+
if (!order.includes(f))
|
|
137
|
+
order.push(f);
|
|
138
|
+
for (const u of users.slice(0, 6)) {
|
|
139
|
+
for (const n of u.lines.slice(0, 8))
|
|
140
|
+
add(u.file, n - 12, n + 12);
|
|
141
|
+
if (!order.includes(u.file))
|
|
142
|
+
order.push(u.file);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
98
146
|
// How a component is reached and fed: the files that import it, up to the
|
|
99
147
|
// page that serves it (its route), and the hooks and API modules they use.
|
|
100
148
|
const importersOf = (target) => {
|
|
@@ -207,7 +255,8 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
207
255
|
// word with the doc, first in the excerpts. A doc rarely names a route; a
|
|
208
256
|
// test cannot call one without its real path.
|
|
209
257
|
const words = [...new Set((markdown.toLowerCase().match(/[a-z]{5,}/g) ?? []).filter((w) => !GENERIC.has(w)))];
|
|
210
|
-
|
|
258
|
+
// A stem too short to search on ("book" from booking) keeps its word.
|
|
259
|
+
const stems = [...new Set(words.map((w) => { const st = w.replace(/(?:ings?|ed|es|s)$/, ''); return st.length >= 5 ? st : w; }))];
|
|
211
260
|
const ROUTE_FILE = /(^|[\\/])(routes?|router|urls)([\\/]|\.[a-z]+$)|\.routes?\.[a-z]+$|[\\/]routes?\.[a-z]+$/i;
|
|
212
261
|
const ROUTE_LINE = /Route::\w+\(|\brouter\.(?:get|post|put|patch|delete|all|use)\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(/;
|
|
213
262
|
// Only this package's own routes: a related repo's are another area's.
|
|
@@ -216,6 +265,7 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
216
265
|
// most 120 across the project; the prefix and group lines that complete
|
|
217
266
|
// their paths come with them. Commented-out routes are not routes.
|
|
218
267
|
const candidates = [];
|
|
268
|
+
const docLow = markdown.toLowerCase().replace(/[-_]/g, ' ');
|
|
219
269
|
for (const f of routeFiles.slice(0, 8)) {
|
|
220
270
|
let lines;
|
|
221
271
|
try {
|
|
@@ -227,20 +277,94 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
227
277
|
lines.forEach((l, i) => {
|
|
228
278
|
if (/^\s*(?:\/\/|#|\*)/.test(l))
|
|
229
279
|
return;
|
|
230
|
-
|
|
280
|
+
// The controller class is on every route it serves: only the path and method tell routes apart.
|
|
281
|
+
const low = l.toLowerCase().replace(/[\w\\]*controller(?=::class|@)/g, '');
|
|
231
282
|
if (/->prefix\(|::prefix\(|\bprefix\s*[:=]|->group\(|::group\(/i.test(l)) {
|
|
232
283
|
add(f, i + 1, i + 1);
|
|
233
284
|
return;
|
|
234
285
|
}
|
|
235
286
|
if (!ROUTE_LINE.test(l))
|
|
236
287
|
return;
|
|
237
|
-
|
|
288
|
+
// A path the doc writes as words ("booking order" for booking-order) is
|
|
289
|
+
// a route its tests call: ahead of lines that only share words.
|
|
290
|
+
const path = /['"]([\w/{}.-]*[-_][\w/{}.-]*)['"]/.exec(l)?.[1] ?? '';
|
|
291
|
+
const named = path.toLowerCase().split('/').filter((seg) => /[-_]/.test(seg) && docLow.includes(seg.replace(/[-_]/g, ' '))).length;
|
|
292
|
+
const score = stems.filter((w) => low.includes(w)).length + 10 * named;
|
|
238
293
|
if (score)
|
|
239
294
|
candidates.push({ file: f, line: i + 1, score });
|
|
240
295
|
});
|
|
241
296
|
}
|
|
242
|
-
for (const c of candidates.sort((a, b) => b.score - a.score).slice(0,
|
|
297
|
+
for (const c of candidates.sort((a, b) => b.score - a.score).slice(0, 60))
|
|
243
298
|
add(c.file, c.line, c.line);
|
|
299
|
+
// What a request must carry is in its handler: the start of the method a
|
|
300
|
+
// route names (Laravel's [Controller::class, 'method'] or 'Controller@method')
|
|
301
|
+
// holds its validation rules, so a test sends the real fields.
|
|
302
|
+
const handlers = new Set();
|
|
303
|
+
const handlerFiles = [];
|
|
304
|
+
const seenRules = new Set();
|
|
305
|
+
// Only the routes the doc names by their path: theirs are the requests its tests send.
|
|
306
|
+
for (const c of candidates.filter((x) => x.score >= 10).slice(0, 12)) {
|
|
307
|
+
let line = '';
|
|
308
|
+
try {
|
|
309
|
+
line = readFileSync(c.file, 'utf-8').split('\n')[c.line - 1] ?? '';
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
const m = /\[\s*\\?([\w\\]*?)(\w+Controller)::class\s*,\s*['"](\w+)['"]\s*\]/.exec(line) ?? /['"](\w+Controller)@(\w+)['"]/.exec(line);
|
|
315
|
+
if (!m)
|
|
316
|
+
continue;
|
|
317
|
+
const [cls, method] = m.length === 4 ? [m[2], m[3]] : [m[1], m[2]];
|
|
318
|
+
if (handlers.has(`${cls}@${method}`))
|
|
319
|
+
continue;
|
|
320
|
+
handlers.add(`${cls}@${method}`);
|
|
321
|
+
for (const f of files.filter((x) => x.endsWith(`${cls}.php`))) {
|
|
322
|
+
let text;
|
|
323
|
+
try {
|
|
324
|
+
text = readFileSync(f, 'utf-8').split('\n');
|
|
325
|
+
}
|
|
326
|
+
catch {
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
const at = text.findIndex((l) => new RegExp(`function\\s+${method}\\s*\\(`).test(l));
|
|
330
|
+
if (at === -1)
|
|
331
|
+
continue;
|
|
332
|
+
add(f, at + 1, at + 45);
|
|
333
|
+
if (!handlerFiles.includes(f))
|
|
334
|
+
handlerFiles.push(f);
|
|
335
|
+
// A handler that hands the request to a class it is given
|
|
336
|
+
// (bookingOrderV2(Request $r, OrderLogicV2 $logic)) keeps its rules there.
|
|
337
|
+
const sig = text[at].match(/\(([^)]*)\)/)?.[1] ?? '';
|
|
338
|
+
for (const dep of [...sig.matchAll(/\b([A-Z]\w+)\s+\$\w+/g)].map((d) => d[1]).filter((d) => d !== 'Request'))
|
|
339
|
+
rulesOf(dep, 2);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
// The rule methods of a class (validate, rules, getXRules), and of the
|
|
343
|
+
// classes its constructor is given, `depth` hops down.
|
|
344
|
+
function rulesOf(cls, depth) {
|
|
345
|
+
if (!depth || seenRules.has(cls))
|
|
346
|
+
return;
|
|
347
|
+
seenRules.add(cls);
|
|
348
|
+
for (const f of files.filter((x) => x.endsWith(`/${cls}.php`))) {
|
|
349
|
+
let text;
|
|
350
|
+
try {
|
|
351
|
+
text = readFileSync(f, 'utf-8').split('\n');
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
text.forEach((l, i) => {
|
|
357
|
+
if (/function\s+(?:validate|rules|\w*Rules)\s*\(/.test(l))
|
|
358
|
+
add(f, i + 1, i + 50);
|
|
359
|
+
const ctor = /function\s+__construct\s*\(([^)]*)\)/.exec(l);
|
|
360
|
+
if (ctor)
|
|
361
|
+
for (const d of ctor[1].matchAll(/\b([A-Z]\w+)\s+\$\w+/g))
|
|
362
|
+
rulesOf(d[1], depth - 1);
|
|
363
|
+
});
|
|
364
|
+
if (windows.has(f) && !handlerFiles.includes(f))
|
|
365
|
+
handlerFiles.push(f);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
244
368
|
for (const f of routeFiles.slice(0, 8)) {
|
|
245
369
|
if (candidates.some((c) => c.file === f) && windows.has(f)) {
|
|
246
370
|
const at = order.indexOf(f);
|
|
@@ -249,6 +373,16 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
|
|
|
249
373
|
order.unshift(f);
|
|
250
374
|
}
|
|
251
375
|
}
|
|
376
|
+
// Handlers and their rules right after the routes, ahead of the budget cut:
|
|
377
|
+
// without them a test sends fields the product does not take.
|
|
378
|
+
const firstOther = order.findIndex((f) => !routeFiles.includes(f) || !candidates.some((c) => c.file === f));
|
|
379
|
+
const handlersIn = handlerFiles.filter((f) => windows.has(f));
|
|
380
|
+
for (const f of handlersIn) {
|
|
381
|
+
const i = order.indexOf(f);
|
|
382
|
+
if (i !== -1)
|
|
383
|
+
order.splice(i, 1);
|
|
384
|
+
}
|
|
385
|
+
order.splice(firstOther === -1 ? order.length : Math.min(firstOther, order.length), 0, ...handlersIn);
|
|
252
386
|
// A page's own parts (its total, its operator card, its pickers) live in the
|
|
253
387
|
// components it imports from its own folder. Their visible texts and labels
|
|
254
388
|
// come right after it, ahead of files that only share a word with the doc.
|
|
@@ -17,12 +17,12 @@
|
|
|
17
17
|
* journey document remains the sole source of what is CORRECT, and the
|
|
18
18
|
* generation prompt says so in the same breath as it presents these facts.
|
|
19
19
|
*/
|
|
20
|
+
import { createHash } from 'node:crypto';
|
|
20
21
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
21
22
|
import { dirname, join } from 'node:path';
|
|
22
23
|
import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
23
24
|
import { readRules, ruleIndex } from '../generate/journey-memory.js';
|
|
24
25
|
import { sourceExcerpts } from './excerpts.js';
|
|
25
|
-
import { hashProjectSource } from './cache.js';
|
|
26
26
|
import { journeyDir } from '../paths.js';
|
|
27
27
|
import { loadEnvFile } from '../project/config.js';
|
|
28
28
|
import { originsFrom, pathsFrom, observeApp } from './observe.js';
|
|
@@ -37,7 +37,7 @@ const MAP_PATH = 'testforge.md';
|
|
|
37
37
|
* that upgrades keeps serving the facts it cached under the old version —
|
|
38
38
|
* so the new facts never reach a single prompt and the upgrade does nothing.
|
|
39
39
|
*/
|
|
40
|
-
const FACTS_VERSION =
|
|
40
|
+
const FACTS_VERSION = 11;
|
|
41
41
|
/** Where a journey's grounded facts live, mirroring the plan cache layout. */
|
|
42
42
|
function groundingPath(journey) {
|
|
43
43
|
return join(journeyDir(journey), 'facts.json');
|
|
@@ -107,7 +107,57 @@ function compactJourney(markdown) {
|
|
|
107
107
|
const m = readRules(markdown);
|
|
108
108
|
if (!m.rules.length)
|
|
109
109
|
return markdown;
|
|
110
|
-
|
|
110
|
+
// The setup names the requests the tests send (booking, payment): without it
|
|
111
|
+
// their routes look unused and drop out of the facts.
|
|
112
|
+
return [m.header, ``, `CONTROLS:`, ...Object.entries(m.controls).map(([id, c]) => `${id}: ${c}`), ``, `RULES:`, ruleIndex(m), ...(m.setup ? [``, `TEST SETUP:`, m.setup] : [])].join('\n');
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The code this journey reads, as the excerpts collect it (no AI): only a
|
|
116
|
+
* change to THESE files moves the journey's facts. A commit elsewhere in the
|
|
117
|
+
* project costs nothing; a feature's code arriving changes them.
|
|
118
|
+
*/
|
|
119
|
+
function journeyCode(journey, srcDir) {
|
|
120
|
+
let excerpts = '';
|
|
121
|
+
try {
|
|
122
|
+
excerpts = sourceExcerpts(journey.rawMarkdown || '', [srcDir, ...(loadConfig().relatedDirs ?? [])]);
|
|
123
|
+
}
|
|
124
|
+
catch { /* unreadable tree: no code */ }
|
|
125
|
+
return { excerpts, hash: createHash('sha1').update(excerpts).digest('hex').slice(0, 16) };
|
|
126
|
+
}
|
|
127
|
+
/** True when the journey's facts were read from code that has since changed. */
|
|
128
|
+
export function factsCodeMoved(journey, srcDir = 'src') {
|
|
129
|
+
const path = groundingPath(journey);
|
|
130
|
+
if (!existsSync(path))
|
|
131
|
+
return false;
|
|
132
|
+
try {
|
|
133
|
+
const key = String(JSON.parse(readFileSync(path, 'utf-8')).key ?? '');
|
|
134
|
+
return !key.endsWith(`:${journeyCode(journey, srcDir).hash}`);
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* True when the journey's tests were last written with no code to read (the
|
|
142
|
+
* feature was not built, or none of its code was found). Once code is there,
|
|
143
|
+
* those tests are written again from it — their "how" was a guess.
|
|
144
|
+
*/
|
|
145
|
+
export function writtenWithoutCode(journey) {
|
|
146
|
+
try {
|
|
147
|
+
return JSON.parse(readFileSync(groundingPath(journey), 'utf-8')).noFacts === true;
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/** Record that this journey's tests were written with no code to read. */
|
|
154
|
+
export function markWrittenWithoutCode(journey, srcDir = 'src') {
|
|
155
|
+
const path = groundingPath(journey);
|
|
156
|
+
try {
|
|
157
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
158
|
+
writeFileSync(path, JSON.stringify({ key: `v${FACTS_VERSION}::${journeyCode(journey, srcDir).hash}`, facts: '', noFacts: true }, null, 2), 'utf-8');
|
|
159
|
+
}
|
|
160
|
+
catch { /* best effort */ }
|
|
111
161
|
}
|
|
112
162
|
/**
|
|
113
163
|
* Establish (or reuse) the journey's code facts. Cached against the doc hash
|
|
@@ -119,21 +169,21 @@ function compactJourney(markdown) {
|
|
|
119
169
|
*/
|
|
120
170
|
export async function groundJourney(journey, docHash, srcDir = 'src', markdown = '',
|
|
121
171
|
/** A doc edit, not a new journey: reuse the facts already read, whatever version of the doc they came from. */
|
|
122
|
-
reuseOnly = false
|
|
172
|
+
reuseOnly = false,
|
|
173
|
+
/** Read the code again whatever is cached (generate --refresh-code). */
|
|
174
|
+
fresh = false) {
|
|
123
175
|
const path = groundingPath(journey);
|
|
124
176
|
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
sourceHash = hashProjectSource(srcDir);
|
|
128
|
-
}
|
|
129
|
-
catch {
|
|
130
|
-
// unreadable source tree — fall back to keying on the doc alone
|
|
131
|
-
}
|
|
177
|
+
const code = journeyCode(journey, srcDir);
|
|
178
|
+
const sourceHash = code.hash;
|
|
132
179
|
const key = `v${FACTS_VERSION}:${docHash}:${sourceHash}`;
|
|
133
|
-
if (existsSync(path)) {
|
|
180
|
+
if (existsSync(path) && !fresh) {
|
|
134
181
|
try {
|
|
135
182
|
const cached = JSON.parse(readFileSync(path, 'utf-8'));
|
|
136
|
-
|
|
183
|
+
// A doc edit reuses facts from an earlier version of the doc, but only
|
|
184
|
+
// while the code is the same: new code is exactly what the facts are for.
|
|
185
|
+
const sameCode = String(cached.key).startsWith(`v${FACTS_VERSION}:`) && String(cached.key).endsWith(`:${sourceHash}`);
|
|
186
|
+
if (typeof cached.facts === 'string' && cached.facts.trim() && (cached.key === key || (reuseOnly && sameCode))) {
|
|
137
187
|
return cached.facts;
|
|
138
188
|
}
|
|
139
189
|
}
|
|
@@ -141,15 +191,27 @@ reuseOnly = false) {
|
|
|
141
191
|
// corrupt cache — re-ground
|
|
142
192
|
}
|
|
143
193
|
}
|
|
144
|
-
if (reuseOnly)
|
|
194
|
+
if (reuseOnly && !existsSync(path))
|
|
145
195
|
return null;
|
|
196
|
+
if (reuseOnly && !fresh) {
|
|
197
|
+
// A doc edit on unchanged code reuses the facts already read, even ones an
|
|
198
|
+
// older TestForge wrote: facts in an older shape beat writing with none.
|
|
199
|
+
try {
|
|
200
|
+
const cached = JSON.parse(readFileSync(path, 'utf-8'));
|
|
201
|
+
if (String(cached.key).endsWith(`:${sourceHash}`))
|
|
202
|
+
return typeof cached.facts === 'string' && cached.facts.trim() ? cached.facts : null;
|
|
203
|
+
}
|
|
204
|
+
catch { /* corrupt: read again */ }
|
|
205
|
+
}
|
|
146
206
|
const config = loadConfig();
|
|
147
207
|
// The source the doc points at, collected here, goes into one tool-less call.
|
|
148
208
|
// A doc that points at nothing findable gets no facts: its tests are written
|
|
149
209
|
// from the doc, which is the contract, rather than paying an AI to go looking.
|
|
150
|
-
const excerpts =
|
|
151
|
-
if (!excerpts)
|
|
210
|
+
const excerpts = code.excerpts;
|
|
211
|
+
if (!excerpts) {
|
|
212
|
+
markWrittenWithoutCode(journey, srcDir);
|
|
152
213
|
return null;
|
|
214
|
+
}
|
|
153
215
|
// Excerpts only from other packages (a backend matched on a quoted text) say
|
|
154
216
|
// nothing about this area's screens: a call on them returns "unknown" for
|
|
155
217
|
// every section and costs as much as a real one.
|