@activate-core/test-forge 2.4.5 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -8
- package/dist/cli/index.js +10 -11
- package/dist/core/docs/cache.js +1 -1
- package/dist/core/docs/discover.js +5 -5
- package/dist/core/docs/grounding.js +6 -2
- package/dist/core/docs/reference-text.js +308 -52
- package/dist/core/generate/ai-usage.js +6 -2
- package/dist/core/generate/bind.js +3 -2
- package/dist/core/generate/complete-file.js +3 -1
- package/dist/core/generate/review-journey.js +2 -2
- package/dist/core/generate/run-generate.js +18 -0
- package/dist/core/generate/suite-planner.js +2 -2
- package/dist/core/paths.js +41 -6
- package/dist/core/project/setup.js +9 -9
- package/dist/core/project/update.js +6 -6
- package/dist/core/prompts/blocks.js +7 -2
- package/dist/core/run/name-scan.js +5 -3
- package/package.json +1 -1
- package/reporters/interface.cjs +29 -10
package/README.md
CHANGED
|
@@ -25,7 +25,7 @@ testforge setup # one-time: inspect project, map it, draft journey docs
|
|
|
25
25
|
testforge generate # generate tests + seed files from the docs
|
|
26
26
|
````
|
|
27
27
|
|
|
28
|
-
After `setup`, review the drafted docs in `docs/` and fix anything the
|
|
28
|
+
After `setup`, review the drafted docs in `testforge/docs/` and fix anything the
|
|
29
29
|
AI got wrong — they are the source of truth for the tests. Then run
|
|
30
30
|
`generate`.
|
|
31
31
|
|
|
@@ -77,7 +77,23 @@ When a reference really is the whole product's new specification, say so:
|
|
|
77
77
|
testforge discover --reference ./docs/prd-v2.md --replace
|
|
78
78
|
```
|
|
79
79
|
|
|
80
|
-
A
|
|
80
|
+
**A reference is read fresh on every `discover`, never kept.** Only its link is
|
|
81
|
+
stored (`testforge/source.json`), so an edited Notion page is always the one
|
|
82
|
+
used. `generate` and `run` never read the reference. No AI ever re-types it:
|
|
83
|
+
|
|
84
|
+
- **Notion page:** a short call on Claude's smallest model only runs the Notion
|
|
85
|
+
fetch tool (about $0.01); the text is taken word for word from the tool's
|
|
86
|
+
result, however long the page. Child pages are found by TestForge, fetched
|
|
87
|
+
(one more call per level) and read in place.
|
|
88
|
+
- **File:** read as it is; a PDF through `pdftotext` (poppler-utils).
|
|
89
|
+
- **Any other link:** downloaded directly, no AI.
|
|
90
|
+
|
|
91
|
+
Nothing is ever quietly left out. A page Notion cuts short, a child page that
|
|
92
|
+
could not be fetched, a database on the page, a link behind a sign-in or a
|
|
93
|
+
binary file stops with an error that says what to do; images and attachments
|
|
94
|
+
on a Notion page are listed as not read.
|
|
95
|
+
|
|
96
|
+
A journey **is** its doc: `generate` reads `testforge/docs/` and treats every markdown
|
|
81
97
|
file there as one journey. So a doc you write by hand — for a feature the
|
|
82
98
|
reference does not cover yet, or one taken from a ticket — works exactly like a
|
|
83
99
|
drafted one. Write the doc, then `generate --journey <name>`.
|
|
@@ -103,7 +119,7 @@ never touched.
|
|
|
103
119
|
| -------------------- | ----------------------------------------------------------- |
|
|
104
120
|
| `testforge init` | Inspect the project and write `testforge.config.json` only. |
|
|
105
121
|
| `testforge map` | Rebuild the project map (`testforge.md`) only. |
|
|
106
|
-
| `testforge discover` | Redraft the per-journey docs in `docs/` only.
|
|
122
|
+
| `testforge discover` | Redraft the per-journey docs in `testforge/docs/` only. |
|
|
107
123
|
| `testforge discover --check --journey <name>` | Check a doc that already exists (a hand-written one too) and add what it lacks — the baseline cases its controls need, the branches the code shows. Nothing is redrafted. |
|
|
108
124
|
| `testforge env` | Rebuild `.env.testing.example` from the docs only. |
|
|
109
125
|
| `testforge run` | Run the generated tests (`--journey` / `--area` / `--env-file`). First fills the names tests wait for from the code (only where each kind of name lives, only when those lines changed); `--no-scan` skips that. |
|
|
@@ -118,8 +134,7 @@ never touched.
|
|
|
118
134
|
| `--journey <name>` | discover, generate, run | Work on one journey instead of all of them. |
|
|
119
135
|
| `--area <name>` | discover, generate, run | Work on one area (api, web, ops…). |
|
|
120
136
|
| `--force` | generate | Regenerate even when the doc has not changed. |
|
|
121
|
-
| `--plan-only` | generate | Plan and check each journey (`testforge/
|
|
122
|
-
| `--refresh` | discover | Fetch the reference again instead of the copy kept from the last run. |
|
|
137
|
+
| `--plan-only` | generate | Plan and check each journey (`testforge/journeys/<area>/<id>/plan-check.md`), write no tests. |
|
|
123
138
|
| `--no-scan` | run | Do not look for blank interface names in the code before the run. |
|
|
124
139
|
| `--refresh-docs` | update | Re-draft the docs even when nothing detectably changed — see below. |
|
|
125
140
|
| `--reference <file\|url>` | setup | Base the docs on a spec instead of the code map. |
|
|
@@ -156,7 +171,7 @@ No source changes detected.
|
|
|
156
171
|
`testforge.md`: a short list of journeys and a summary of the data layer
|
|
157
172
|
(database, models, key fields). The config is held to a closed list of
|
|
158
173
|
keys, and a framework other than `jest` or `playwright` is refused.
|
|
159
|
-
2. **discover** — writes one doc per journey into `docs/`.
|
|
174
|
+
2. **discover** — writes one doc per journey into `testforge/docs/`.
|
|
160
175
|
|
|
161
176
|
**The journey doc.** Every criterion is one line with its own id, its tier and
|
|
162
177
|
the controls it covers:
|
|
@@ -205,7 +220,7 @@ cannot read is reported as exactly that, never as "up to date".
|
|
|
205
220
|
|
|
206
221
|
## Source of truth
|
|
207
222
|
|
|
208
|
-
The journey docs in `docs/` are the source of truth for the tests —
|
|
223
|
+
The journey docs in `testforge/docs/` are the source of truth for the tests —
|
|
209
224
|
`generate` reads the docs and nothing else. So where the docs come from
|
|
210
225
|
decides everything.
|
|
211
226
|
|
|
@@ -391,7 +406,7 @@ Test results are kept as a workflow artifact for 14 days.
|
|
|
391
406
|
## How `generate` keeps the AI cost down
|
|
392
407
|
|
|
393
408
|
The journey doc is read once and kept as the journey's **memory**
|
|
394
|
-
(`testforge/
|
|
409
|
+
(`testforge/journeys/<area>/<id>/`): its rules with their source, level, tests and
|
|
395
410
|
history, the plan, and the code facts. No later step sends the whole doc again.
|
|
396
411
|
|
|
397
412
|
1. **Rules.** The doc is split into rules (R1, R2 …) without AI. Rule ids stay
|
package/dist/cli/index.js
CHANGED
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
import { VERSION } from '../core/index.js';
|
|
22
22
|
import { runGenerate } from '../core/generate/run-generate.js';
|
|
23
23
|
import { runReview } from '../core/generate/review-journey.js';
|
|
24
|
-
import { migrateState } from '../core/paths.js';
|
|
24
|
+
import { DOCS_DIR, migrateState } from '../core/paths.js';
|
|
25
25
|
import { acceptNames, bindJourneys } from '../core/generate/bind.js';
|
|
26
26
|
import { runClean } from '../core/run/clean.js';
|
|
27
27
|
import { aiUsage, describeUsage, usageByStage } from '../core/generate/ai-usage.js';
|
|
@@ -64,7 +64,6 @@ function sourceOptions() {
|
|
|
64
64
|
reference,
|
|
65
65
|
source: flagValue('--source'),
|
|
66
66
|
replace: process.argv.includes('--replace'),
|
|
67
|
-
refresh: process.argv.includes('--refresh'),
|
|
68
67
|
};
|
|
69
68
|
}
|
|
70
69
|
/**
|
|
@@ -98,7 +97,7 @@ function printUsage() {
|
|
|
98
97
|
console.log(' testforge generate --journey <name> Generate just ONE journey');
|
|
99
98
|
console.log(' testforge generate --area <area> Generate a whole area (e.g. api)');
|
|
100
99
|
console.log(' testforge discover Draft journey docs (asks: map or reference)');
|
|
101
|
-
console.log(' testforge discover --journey <name> Draft just ONE journey doc', ' testforge discover --check --journey <name> Check an existing doc (hand-written too) and add what it lacks; nothing is redrafted', ' testforge discover --journey <name> --reference <x>
|
|
100
|
+
console.log(' testforge discover --journey <name> Draft just ONE journey doc', ' testforge discover --check --journey <name> Check an existing doc (hand-written too) and add what it lacks; nothing is redrafted', ' testforge discover --journey <name> --reference <x> One journey from a reference (read fresh each time)');
|
|
102
101
|
console.log(' testforge discover --reference <file|url> Base docs on a reference');
|
|
103
102
|
console.log(' testforge discover --tickets <folder> Base docs on exported tickets (one file per ticket); each source names its ticket');
|
|
104
103
|
console.log(' (a reference only writes the journeys it');
|
|
@@ -202,22 +201,22 @@ async function main() {
|
|
|
202
201
|
console.log('Usage: testforge discover --check --journey <name>');
|
|
203
202
|
}
|
|
204
203
|
else {
|
|
205
|
-
console.log(checkExistingDoc(
|
|
204
|
+
console.log(await checkExistingDoc(DOCS_DIR, journey, '.'));
|
|
206
205
|
}
|
|
207
206
|
}
|
|
208
207
|
else if (command === 'discover') {
|
|
209
208
|
console.log('Drafting journey files with AI...');
|
|
210
|
-
console.log('This writes drafts into docs/ from your chosen source.\n');
|
|
211
|
-
const output = await runDiscover('.',
|
|
209
|
+
console.log('This writes drafts into testforge/docs/ from your chosen source.\n');
|
|
210
|
+
const output = await runDiscover('.', DOCS_DIR, sourceOptions(), flagValue('--journey'));
|
|
212
211
|
console.log(output);
|
|
213
|
-
console.log('\nDraft journeys written to docs/. Please review and edit them,');
|
|
212
|
+
console.log('\nDraft journeys written to testforge/docs/. Please review and edit them,');
|
|
214
213
|
console.log('then run `testforge generate`.');
|
|
215
214
|
}
|
|
216
215
|
else if (command === 'env') {
|
|
217
216
|
if (process.argv.includes('--diff'))
|
|
218
217
|
console.log(describeEnvDiff(envDiff('.')));
|
|
219
218
|
else
|
|
220
|
-
runEnvTemplate(
|
|
219
|
+
runEnvTemplate(DOCS_DIR, '.env.testing.example', '.', generatedScanDirs());
|
|
221
220
|
}
|
|
222
221
|
else if (command === 'run') {
|
|
223
222
|
// Anything after `--`, plus the runner flags people reach for most, goes
|
|
@@ -243,7 +242,7 @@ async function main() {
|
|
|
243
242
|
console.log(useAI
|
|
244
243
|
? 'Generating tests with AI for changed journeys...\n'
|
|
245
244
|
: 'Generating tests with the basic generator...\n');
|
|
246
|
-
const results = await runGenerate(
|
|
245
|
+
const results = await runGenerate(DOCS_DIR, useAI, force, flagValue('--journey'), flagValue('--area'), {
|
|
247
246
|
waitForLimit: process.argv.includes('--no-wait')
|
|
248
247
|
? false
|
|
249
248
|
: process.argv.includes('--wait-for-limit')
|
|
@@ -252,7 +251,7 @@ async function main() {
|
|
|
252
251
|
planOnly: process.argv.includes('--plan-only'),
|
|
253
252
|
});
|
|
254
253
|
if (results.length === 0) {
|
|
255
|
-
console.log('No journey files found in docs/. Add a .md file and try again.');
|
|
254
|
+
console.log('No journey files found in testforge/docs/. Add a .md file and try again.');
|
|
256
255
|
}
|
|
257
256
|
else {
|
|
258
257
|
for (const r of results) {
|
|
@@ -311,7 +310,7 @@ async function main() {
|
|
|
311
310
|
// just `setup`) tells the developer exactly which credentials the
|
|
312
311
|
// generated suite needs — including vars the tests themselves read.
|
|
313
312
|
console.log('');
|
|
314
|
-
runEnvTemplate(
|
|
313
|
+
runEnvTemplate(DOCS_DIR, '.env.testing.example', '.', generatedScanDirs());
|
|
315
314
|
}
|
|
316
315
|
}
|
|
317
316
|
else if (command === 'report') {
|
package/dist/core/docs/cache.js
CHANGED
|
@@ -123,7 +123,7 @@ export function hashProjectSource(rootDir) {
|
|
|
123
123
|
* not checking at all — it reports "unchanged" while the specification moves,
|
|
124
124
|
* and "changed" every time the vendor ships. Refuse it and say why.
|
|
125
125
|
*/
|
|
126
|
-
function looksLikeAppShell(url, body) {
|
|
126
|
+
export function looksLikeAppShell(url, body) {
|
|
127
127
|
let host = '';
|
|
128
128
|
try {
|
|
129
129
|
host = new URL(url).hostname.toLowerCase();
|
|
@@ -154,7 +154,7 @@ export async function runDiscover(srcDir, docsDir, opts = {}, journey) {
|
|
|
154
154
|
// reference read once and cut to this journey, drafted and reviewed in one
|
|
155
155
|
// call each with no tools.
|
|
156
156
|
if (source.mode === 'reference' && source.reference && journey) {
|
|
157
|
-
return draftOneJourney(srcDir, docsDir, source.reference, journey
|
|
157
|
+
return draftOneJourney(srcDir, docsDir, source.reference, journey);
|
|
158
158
|
}
|
|
159
159
|
if (source.mode === 'reference' && source.reference) {
|
|
160
160
|
console.log(`\nComparing the reference against testforge.md: ${source.reference}`);
|
|
@@ -277,9 +277,9 @@ async function discoverFromCode(srcDir, docsDir, journey) {
|
|
|
277
277
|
}
|
|
278
278
|
const docTitle = (text) => /^# Journey:\s*(.+)$/m.exec(text)?.[1]?.trim();
|
|
279
279
|
const kebab = (t) => t.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
280
|
-
async function draftOneJourney(srcDir, docsDir, reference, journey
|
|
280
|
+
async function draftOneJourney(srcDir, docsDir, reference, journey) {
|
|
281
281
|
const config = loadConfig();
|
|
282
|
-
const text = readReference(reference
|
|
282
|
+
const text = await readReference(reference);
|
|
283
283
|
const section = sectionFor(text, journey);
|
|
284
284
|
const excerpts = sourceExcerpts(section, [srcDir, ...(config.relatedDirs ?? [])]);
|
|
285
285
|
const docs = [...snapshotDocs(docsDir)].filter(([p, t]) => !notAJourneyDoc(p, t));
|
|
@@ -338,7 +338,7 @@ function checkAndFinish(doc, referenceText, section, excerpts, others) {
|
|
|
338
338
|
* and add what it lacks: the baseline items its controls need, the branches the
|
|
339
339
|
* code shows. Nothing is redrafted; the doc's own criteria stay as written.
|
|
340
340
|
*/
|
|
341
|
-
export function checkExistingDoc(docsDir, journey, srcDir
|
|
341
|
+
export async function checkExistingDoc(docsDir, journey, srcDir) {
|
|
342
342
|
const config = loadConfig();
|
|
343
343
|
const docs = [...snapshotDocs(docsDir)].filter(([p, t]) => !notAJourneyDoc(p, t));
|
|
344
344
|
const hit = docs.find(([p, t]) => kebab(docTitle(t) ?? '') === kebab(journey) || p.replace(/\.md$/, '').endsWith(kebab(journey)));
|
|
@@ -347,7 +347,7 @@ export function checkExistingDoc(docsDir, journey, srcDir, refresh = false) {
|
|
|
347
347
|
const [rel, doc] = hit;
|
|
348
348
|
const stored = loadSourceChoice();
|
|
349
349
|
const reference = stored?.journeys?.[rel] ?? (stored?.mode === 'reference' ? stored.reference : undefined);
|
|
350
|
-
const text = reference ? readReference(reference
|
|
350
|
+
const text = reference ? await readReference(reference) : '';
|
|
351
351
|
const section = text ? sectionFor(text, docTitle(doc) ?? journey) : '';
|
|
352
352
|
const excerpts = sourceExcerpts(section || doc, [srcDir, ...(config.relatedDirs ?? [])]);
|
|
353
353
|
const others = docs.filter(([p]) => p !== rel).map(([p, t]) => `${docTitle(t) ?? p} (${p})`);
|
|
@@ -37,7 +37,7 @@ const MAP_PATH = 'testforge.md';
|
|
|
37
37
|
* that upgrades keeps serving the facts it cached under the old version —
|
|
38
38
|
* so the new facts never reach a single prompt and the upgrade does nothing.
|
|
39
39
|
*/
|
|
40
|
-
const FACTS_VERSION =
|
|
40
|
+
const FACTS_VERSION = 7;
|
|
41
41
|
/** Where a journey's grounded facts live, mirroring the plan cache layout. */
|
|
42
42
|
function groundingPath(journey) {
|
|
43
43
|
return join(journeyDir(journey), 'facts.json');
|
|
@@ -69,7 +69,11 @@ function buildGroundingPrompt(journey, map, srcDir, excerpts = '') {
|
|
|
69
69
|
` token still populates the request user through a pre-auth hook; and the`,
|
|
70
70
|
` field that ownership is scoped on.`,
|
|
71
71
|
`SHAPES: for each route, the request payload, the response body as its`,
|
|
72
|
-
` serializer builds it, and every status code the handler can return
|
|
72
|
+
` serializer builds it, and every status code the handler can return. For a`,
|
|
73
|
+
` response a screen shows as a list or a detail, also list every field the`,
|
|
74
|
+
` screen reads to show an item (with its type), the values a list keeps or`,
|
|
75
|
+
` drops it by (a status the list filters on), and which of those fields the`,
|
|
76
|
+
` screen renders as text, so a test can stub one item and find it on screen.`,
|
|
73
77
|
`DATA: the tables or collections read or written, the fields that matter, and`,
|
|
74
78
|
` their enum values. For each table a test must seed, copy from the`,
|
|
75
79
|
` migrations, schema dump or model declarations: every NOT NULL column without`,
|
|
@@ -1,63 +1,319 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* testforge — a reference read
|
|
2
|
+
* testforge — a reference read fresh, once per command, word for word, and
|
|
3
|
+
* only its relevant part handed on. No AI ever re-types a reference:
|
|
3
4
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* - a file in the project is read as it is (a PDF through pdftotext);
|
|
6
|
+
* - a Notion page is fetched by a short AI call that only runs the Notion
|
|
7
|
+
* fetch tool (one more per level of child pages); the text is taken from
|
|
8
|
+
* the tools' results in the event stream (or the file the CLI saves a large
|
|
9
|
+
* one to);
|
|
10
|
+
* - any other link is downloaded directly.
|
|
11
|
+
*
|
|
12
|
+
* Anything that would leave part of a reference out (a page Notion cut short,
|
|
13
|
+
* a child page not fetched, a database, a login page) is an error, never a
|
|
14
|
+
* quietly shorter reference. Nothing is kept between commands, so an edited
|
|
15
|
+
* page is always the one used; the last text read is written under
|
|
16
|
+
* testforge/runs/reference/ to look at, never read back.
|
|
9
17
|
*/
|
|
10
18
|
import { createHash } from 'node:crypto';
|
|
11
|
-
import {
|
|
19
|
+
import { spawnSync } from 'node:child_process';
|
|
20
|
+
import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
21
|
+
import { tmpdir } from 'node:os';
|
|
12
22
|
import { join } from 'node:path';
|
|
13
|
-
import {
|
|
14
|
-
import { loadConfig, splitCommand
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
``,
|
|
35
|
-
`If you cannot read it, print one line starting "ERROR:".`,
|
|
36
|
-
`If it is too long to print in one answer, stop at the end of a heading's`,
|
|
37
|
-
`section and print a last line "CONTINUE AFTER: <that heading>". You will be`,
|
|
38
|
-
`asked for the rest.`,
|
|
39
|
-
].join('\n'),
|
|
40
|
-
encoding: 'utf-8',
|
|
41
|
-
maxBuffer: 10 * 1024 * 1024,
|
|
42
|
-
}).trim();
|
|
43
|
-
// A long page comes in parts; each part says where the next one starts.
|
|
44
|
-
const parts = [];
|
|
45
|
-
let after;
|
|
46
|
-
for (let k = 0; k < 12; k++) {
|
|
47
|
-
const part = ask(after);
|
|
48
|
-
const cont = /\n?CONTINUE AFTER:\s*(.+)\s*$/.exec(part);
|
|
49
|
-
parts.push(cont ? part.slice(0, cont.index).trimEnd() : part);
|
|
50
|
-
if (!cont || cont[1].trim() === after)
|
|
51
|
-
break;
|
|
52
|
-
after = cont[1].trim();
|
|
23
|
+
import { RUNS_DIR } from '../paths.js';
|
|
24
|
+
import { isClaude, loadConfig, splitCommand } from '../project/config.js';
|
|
25
|
+
import { runAiWithStream } from '../generate/ai-usage.js';
|
|
26
|
+
import { looksLikeAppShell } from './cache.js';
|
|
27
|
+
const read = new Map();
|
|
28
|
+
/** The reference's text, word for word: a file, a folder of files, a Notion page with its child pages, or a link. */
|
|
29
|
+
export async function readReference(reference) {
|
|
30
|
+
const known = read.get(reference);
|
|
31
|
+
if (known !== undefined)
|
|
32
|
+
return known;
|
|
33
|
+
const text = existsSync(reference) ? readLocal(reference)
|
|
34
|
+
: notionId(reference) && !/^https?:\/\//i.test(reference) || isNotionUrl(reference) ? readNotion(reference)
|
|
35
|
+
: /^https?:\/\//i.test(reference) ? await readUrl(reference)
|
|
36
|
+
: fail(reference, 'it is neither a file in this project nor a link');
|
|
37
|
+
if (!text.trim())
|
|
38
|
+
fail(reference, 'it is empty');
|
|
39
|
+
read.set(reference, text);
|
|
40
|
+
try {
|
|
41
|
+
const dir = join(RUNS_DIR, 'reference');
|
|
42
|
+
mkdirSync(dir, { recursive: true });
|
|
43
|
+
writeFileSync(join(dir, `${createHash('sha1').update(reference).digest('hex').slice(0, 12)}.md`), text, 'utf-8');
|
|
53
44
|
}
|
|
54
|
-
|
|
55
|
-
if (!text || /^ERROR:/m.test(text.slice(0, 300)))
|
|
56
|
-
throw new Error(`Could not read the reference "${reference}":\n${text.slice(0, 500)}`);
|
|
57
|
-
mkdirSync(REFERENCE_DIR, { recursive: true });
|
|
58
|
-
writeFileSync(path, text, 'utf-8');
|
|
45
|
+
catch { /* only a copy to look at */ }
|
|
59
46
|
return text;
|
|
60
47
|
}
|
|
48
|
+
function fail(reference, why) {
|
|
49
|
+
throw new Error(`Could not read the reference "${reference}": ${why}.`);
|
|
50
|
+
}
|
|
51
|
+
// ---- files ----
|
|
52
|
+
function readLocal(path) {
|
|
53
|
+
if (statSync(path).isDirectory()) {
|
|
54
|
+
return readdirSync(path).sort()
|
|
55
|
+
.filter((f) => /\.(md|markdown|txt|pdf)$/i.test(f) && statSync(join(path, f)).isFile())
|
|
56
|
+
.map((f) => `# ${f}\n\n${readLocal(join(path, f))}`)
|
|
57
|
+
.join('\n\n');
|
|
58
|
+
}
|
|
59
|
+
if (/\.pdf$/i.test(path))
|
|
60
|
+
return pdfText(path, path);
|
|
61
|
+
const buf = readFileSync(path);
|
|
62
|
+
if (buf.includes(0))
|
|
63
|
+
fail(path, 'it is not a text file — save it as .md, .txt or .pdf');
|
|
64
|
+
return buf.toString('utf-8');
|
|
65
|
+
}
|
|
66
|
+
function pdfText(file, reference) {
|
|
67
|
+
const r = spawnSync('pdftotext', [file, '-'], { encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024 });
|
|
68
|
+
if (r.error)
|
|
69
|
+
fail(reference, 'reading a PDF needs pdftotext (poppler-utils: `apt install poppler-utils`, `brew install poppler`)');
|
|
70
|
+
if (r.status !== 0)
|
|
71
|
+
fail(reference, `pdftotext failed: ${String(r.stderr).trim().slice(0, 200)}`);
|
|
72
|
+
if (!r.stdout.trim())
|
|
73
|
+
fail(reference, 'the PDF has no text layer (a scan) — export the text instead');
|
|
74
|
+
return r.stdout;
|
|
75
|
+
}
|
|
76
|
+
// ---- links ----
|
|
77
|
+
async function readUrl(url) {
|
|
78
|
+
let res;
|
|
79
|
+
try {
|
|
80
|
+
res = await fetch(url, { redirect: 'follow' });
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
return fail(url, e.message);
|
|
84
|
+
}
|
|
85
|
+
if (!res.ok)
|
|
86
|
+
fail(url, `the link returned ${res.status} — if it needs a sign-in, export it as .md, .txt or .pdf and pass the file`);
|
|
87
|
+
const type = res.headers.get('content-type') ?? '';
|
|
88
|
+
if (/pdf/i.test(type) || /\.pdf(\?|$)/i.test(url)) {
|
|
89
|
+
const file = join(mkdtempSync(join(tmpdir(), 'testforge-')), 'reference.pdf');
|
|
90
|
+
writeFileSync(file, Buffer.from(await res.arrayBuffer()));
|
|
91
|
+
return pdfText(file, url);
|
|
92
|
+
}
|
|
93
|
+
const body = await res.text();
|
|
94
|
+
if (!/html/i.test(type))
|
|
95
|
+
return body;
|
|
96
|
+
const shell = looksLikeAppShell(url, body);
|
|
97
|
+
if (shell)
|
|
98
|
+
fail(url, `${shell} — export it as .md, .txt or .pdf and pass the file`);
|
|
99
|
+
return htmlText(body);
|
|
100
|
+
}
|
|
101
|
+
/** A web page's readable text, its headings and list items kept as Markdown. */
|
|
102
|
+
export function htmlText(html) {
|
|
103
|
+
return entities(html
|
|
104
|
+
.replace(/<(script|style|noscript|svg|head)\b[\s\S]*?<\/\1>/gi, '')
|
|
105
|
+
.replace(/<h([1-6])\b[^>]*>/gi, (_, n) => `\n\n${'#'.repeat(Number(n))} `)
|
|
106
|
+
.replace(/<li\b[^>]*>/gi, '\n- ')
|
|
107
|
+
.replace(/<\/t[dh]>/gi, ' | ')
|
|
108
|
+
.replace(/<(?:br|\/p|\/div|\/h[1-6]|\/tr|\/ul|\/ol|\/section|\/article)\b[^>]*>/gi, '\n')
|
|
109
|
+
.replace(/<[^>]+>/g, ''))
|
|
110
|
+
.replace(/[ \t]+/g, ' ')
|
|
111
|
+
.replace(/\n\s*\n\s*\n+/g, '\n\n')
|
|
112
|
+
.trim();
|
|
113
|
+
}
|
|
114
|
+
function entities(s) {
|
|
115
|
+
return s.replace(/ /g, ' ').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'|'/g, "'")
|
|
116
|
+
.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))).replace(/&/g, '&');
|
|
117
|
+
}
|
|
118
|
+
// ---- Notion ----
|
|
119
|
+
const isNotionUrl = (s) => /^https?:\/\/([\w-]+\.)*notion\.(so|site|com)\//i.test(s);
|
|
120
|
+
/** A Notion page's 32-hex id, from a URL or a bare id. */
|
|
121
|
+
export function notionId(s) {
|
|
122
|
+
const path = s.split(/[?#]/)[0].replace(/-/g, '');
|
|
123
|
+
return /([0-9a-f]{32})(?![0-9a-f])/i.exec(path)?.[1]?.toLowerCase();
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* The fetch only runs tools, and the pages are taken from the tools' results,
|
|
127
|
+
* not the AI's words: Claude's smallest model does it for about a fifteenth
|
|
128
|
+
* of the cost of the configured one.
|
|
129
|
+
*/
|
|
130
|
+
export function fetchModel(cmd) {
|
|
131
|
+
if (!isClaude(cmd.program))
|
|
132
|
+
return cmd;
|
|
133
|
+
const args = cmd.args.filter((a, k) => a !== '--model' && cmd.args[k - 1] !== '--model' && !a.startsWith('--model='));
|
|
134
|
+
return { program: cmd.program, args: [...args, '--model', 'haiku'] };
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* One haiku call per level of child pages (one call for a page without any):
|
|
138
|
+
* the AI is given the exact pages to fetch, and the code — not the AI — finds
|
|
139
|
+
* the child pages in what came back.
|
|
140
|
+
*/
|
|
141
|
+
function readNotion(reference) {
|
|
142
|
+
const cmd = splitCommand(loadConfig().aiCommand);
|
|
143
|
+
if (!isClaude(cmd.program))
|
|
144
|
+
fail(reference, 'a Notion page is read through the Claude CLI and its Notion connection; set aiCommand to claude, or export the page as .md and pass the file');
|
|
145
|
+
const { program, args } = fetchModel(cmd);
|
|
146
|
+
const tools = ['--allowedTools', 'mcp__claude_ai_Notion__notion-fetch mcp__notion__notion-fetch',
|
|
147
|
+
'--disallowedTools', 'Bash Read Grep Glob Write Edit NotebookEdit WebFetch WebSearch Task Agent'];
|
|
148
|
+
const pages = new Map();
|
|
149
|
+
const root = notionId(reference) ?? reference;
|
|
150
|
+
let want = [reference];
|
|
151
|
+
let aiSaid = '';
|
|
152
|
+
for (let round = 0; want.length && round < 12; round++) {
|
|
153
|
+
const { text, raw } = runAiWithStream(program, [...args, ...tools], {
|
|
154
|
+
input: [
|
|
155
|
+
`Fetch each of these Notion pages with the Notion fetch tool (load it with`,
|
|
156
|
+
`ToolSearch "select:mcp__claude_ai_Notion__notion-fetch" if it is not loaded),`,
|
|
157
|
+
`each once, and nothing else:`,
|
|
158
|
+
...want.map((u) => ` ${u}`),
|
|
159
|
+
`Do not print any page: they are read from the tool results. A result the`,
|
|
160
|
+
`tool saved to a file instead of returning counts as fetched. Reply with the`,
|
|
161
|
+
`single word DONE, or one line starting "ERROR:" if a page cannot be fetched.`,
|
|
162
|
+
].join('\n'),
|
|
163
|
+
encoding: 'utf-8',
|
|
164
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
165
|
+
});
|
|
166
|
+
// The AI's word is not the verdict: what the tool results hold is.
|
|
167
|
+
if (/^ERROR:/m.test(text.slice(0, 300)))
|
|
168
|
+
aiSaid = text.trim().slice(0, 300);
|
|
169
|
+
const before = new Set(pages.keys());
|
|
170
|
+
for (const [id, p] of notionPages(raw, reference))
|
|
171
|
+
if (!pages.has(id))
|
|
172
|
+
pages.set(id, p);
|
|
173
|
+
const got = [...pages.keys()].filter((id) => !before.has(id));
|
|
174
|
+
if (!got.length)
|
|
175
|
+
break; // nothing new: assembly names what is missing
|
|
176
|
+
// Next: every child page the reference reaches that is not read yet.
|
|
177
|
+
want = [...new Set(reachable(pages, root).filter((u) => { const id = notionId(u); return id && !pages.has(id); }))];
|
|
178
|
+
}
|
|
179
|
+
try {
|
|
180
|
+
return assemble(pages, reference);
|
|
181
|
+
}
|
|
182
|
+
catch (e) {
|
|
183
|
+
throw aiSaid ? new Error(`${e.message}\n The AI said: ${aiSaid}`) : e;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const CHILD = /<page\s+url="([^"]+)"[^>]*>([\s\S]*?)<\/page>/g;
|
|
187
|
+
/** The child-page links of every page the reference reaches through pages already read. */
|
|
188
|
+
function reachable(pages, root) {
|
|
189
|
+
const out = [];
|
|
190
|
+
const seen = new Set();
|
|
191
|
+
const walk = (id) => {
|
|
192
|
+
if (seen.has(id))
|
|
193
|
+
return;
|
|
194
|
+
seen.add(id);
|
|
195
|
+
const page = pages.get(id);
|
|
196
|
+
if (!page)
|
|
197
|
+
return;
|
|
198
|
+
for (const m of page.body.matchAll(CHILD)) {
|
|
199
|
+
out.push(m[1]);
|
|
200
|
+
const c = notionId(m[1]);
|
|
201
|
+
if (c)
|
|
202
|
+
walk(c);
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
walk(root);
|
|
206
|
+
return out;
|
|
207
|
+
}
|
|
208
|
+
/** The whole reference from a CLI event stream (one round). Throws when any part would be missing. */
|
|
209
|
+
export function notionText(raw, reference) {
|
|
210
|
+
return assemble(notionPages(raw, reference), reference);
|
|
211
|
+
}
|
|
212
|
+
/** The page with each child page in the place its link stood. Throws when any part would be missing. */
|
|
213
|
+
function assemble(pages, reference) {
|
|
214
|
+
const root = notionId(reference);
|
|
215
|
+
const top = root ? pages.get(root) : undefined;
|
|
216
|
+
if (!top)
|
|
217
|
+
fail(reference, 'the Notion page was not fetched — check that Claude\'s Notion connection can open it');
|
|
218
|
+
const missing = [];
|
|
219
|
+
const cut = [];
|
|
220
|
+
const expand = (page, name, seen) => {
|
|
221
|
+
if (page.cut)
|
|
222
|
+
cut.push(name);
|
|
223
|
+
return page.body.replace(CHILD, (_, url, title) => {
|
|
224
|
+
const id = notionId(url);
|
|
225
|
+
const child = id ? pages.get(id) : undefined;
|
|
226
|
+
if (!id || seen.has(id))
|
|
227
|
+
return title;
|
|
228
|
+
if (!child) {
|
|
229
|
+
missing.push(title.trim() || url);
|
|
230
|
+
return title;
|
|
231
|
+
}
|
|
232
|
+
const childName = child.title || title.trim();
|
|
233
|
+
return `\n# ${childName}\n\n${expand(child, childName, new Set([...seen, id])).trim()}\n`;
|
|
234
|
+
});
|
|
235
|
+
};
|
|
236
|
+
const body = expand(top, top.title || reference, new Set([root]));
|
|
237
|
+
if (cut.length)
|
|
238
|
+
fail(reference, `Notion returned ${cut.map((c) => `"${c}"`).join(', ')} with part of it left out (a page too large for one fetch). Split it into child pages — each child page is read whole`);
|
|
239
|
+
if (missing.length)
|
|
240
|
+
fail(reference, `its child page(s) could not be fetched: ${missing.join(', ')}. Check that Claude's Notion connection can open them, then run the command again`);
|
|
241
|
+
const unread = [...new Set([...body.matchAll(/<(database|folder|data-source)\b[^>]*>([^<]*)/g)].map((m) => `${m[1]} "${m[2].trim()}"`))];
|
|
242
|
+
if (unread.length)
|
|
243
|
+
fail(reference, `it holds ${unread.join(', ')}, whose rows are not part of the page text. Copy what the tests need into the page, or export it as .md and pass the file`);
|
|
244
|
+
const files = (body.match(/<(?:pdf|file|audio|video|image)\b|!\[[^\]]*\]\(/g) ?? []).length;
|
|
245
|
+
if (files)
|
|
246
|
+
console.warn(` ${reference}: ${files} image(s)/attachment(s) on the page are not text and are not read.`);
|
|
247
|
+
const out = plainNotion(body).replace(/\n{3,}/g, '\n\n').trim();
|
|
248
|
+
return top.title ? `# ${top.title}\n\n${out}` : out;
|
|
249
|
+
}
|
|
250
|
+
/** Every Notion page fetched in a CLI event stream, by id. */
|
|
251
|
+
function notionPages(raw, reference) {
|
|
252
|
+
const tools = new Map();
|
|
253
|
+
const pages = new Map();
|
|
254
|
+
for (const line of raw.split('\n')) {
|
|
255
|
+
if (!line.startsWith('{'))
|
|
256
|
+
continue;
|
|
257
|
+
let e;
|
|
258
|
+
try {
|
|
259
|
+
e = JSON.parse(line);
|
|
260
|
+
}
|
|
261
|
+
catch {
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
for (const c of e.message?.content ?? []) {
|
|
265
|
+
if (c.type === 'tool_use' && c.id && c.name)
|
|
266
|
+
tools.set(c.id, { name: c.name, id: typeof c.input?.id === 'string' ? c.input.id : undefined });
|
|
267
|
+
if (c.type !== 'tool_result')
|
|
268
|
+
continue;
|
|
269
|
+
const tool = tools.get(c.tool_use_id ?? '');
|
|
270
|
+
if (!tool || !/notion.*fetch/i.test(tool.name))
|
|
271
|
+
continue;
|
|
272
|
+
let body = typeof c.content === 'string' ? c.content
|
|
273
|
+
: Array.isArray(c.content) ? c.content.map((x) => x.text ?? '').join('') : '';
|
|
274
|
+
// A result over the CLI's limit is saved whole to a file; the result names it.
|
|
275
|
+
const saved = /saved to (\/\S+?\.txt)\b/.exec(body)?.[1];
|
|
276
|
+
if (saved) {
|
|
277
|
+
if (!existsSync(saved))
|
|
278
|
+
fail(reference, `the CLI saved a Notion page to ${saved}, which is gone`);
|
|
279
|
+
body = readFileSync(saved, 'utf-8');
|
|
280
|
+
}
|
|
281
|
+
let j = {};
|
|
282
|
+
try {
|
|
283
|
+
j = JSON.parse(body);
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
continue;
|
|
287
|
+
} // an error message, not a page
|
|
288
|
+
const text = String(j.text ?? '');
|
|
289
|
+
const open = text.indexOf('<content>');
|
|
290
|
+
if (open < 0)
|
|
291
|
+
continue;
|
|
292
|
+
const close = text.lastIndexOf('</content>');
|
|
293
|
+
const id = notionId(String(j.url ?? '')) ?? notionId(tool.id ?? '');
|
|
294
|
+
if (!id)
|
|
295
|
+
continue;
|
|
296
|
+
pages.set(id, {
|
|
297
|
+
title: String(j.title ?? '').trim(),
|
|
298
|
+
body: text.slice(open + '<content>'.length, close < 0 ? undefined : close),
|
|
299
|
+
cut: close < 0 || j.truncated === true || Number(j.unknown_block_count) > 0,
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return pages;
|
|
304
|
+
}
|
|
305
|
+
/** Notion's tags as Markdown, so quotes match the page as people read it. */
|
|
306
|
+
export function plainNotion(s) {
|
|
307
|
+
return entities(s
|
|
308
|
+
.replace(/<\/?code>/g, '`')
|
|
309
|
+
.replace(/<\/?(?:b|strong)>/g, '**')
|
|
310
|
+
.replace(/<\/?(?:i|em)>/g, '*')
|
|
311
|
+
.replace(/<br\s*\/?>/g, '\n')
|
|
312
|
+
.replace(/<\/t[dh]>\s*/g, ' | ')
|
|
313
|
+
.replace(/<\/tr>/g, '\n')
|
|
314
|
+
.replace(/<date\s+start="([^"]+)"(?:\s+end="([^"]+)")?[^>]*\/?>/g, (_, a, b) => (b ? `${a} to ${b}` : a))
|
|
315
|
+
.replace(/<\/?[a-z][\w-]*(?:\s[^>]*)?\/?>/g, ''));
|
|
316
|
+
}
|
|
61
317
|
const norm = (t) => t.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
|
62
318
|
/**
|
|
63
319
|
* The part of a reference one journey needs: everything before the first
|
|
@@ -53,12 +53,16 @@ export function describeUsage(u) {
|
|
|
53
53
|
}
|
|
54
54
|
/** execFileSync for an AI CLI, counted. Same signature as the call it replaces. */
|
|
55
55
|
export function runAiSync(program, args, options) {
|
|
56
|
+
return runAiWithStream(program, args, options).text;
|
|
57
|
+
}
|
|
58
|
+
/** The same call, also returning the raw event stream (tool results included) for a caller that needs it. */
|
|
59
|
+
export function runAiWithStream(program, args, options) {
|
|
56
60
|
const started = Date.now();
|
|
57
61
|
let out = '';
|
|
58
62
|
try {
|
|
59
63
|
// The event stream is larger than the answer alone: give it room.
|
|
60
64
|
out = execFileSync(program, traceArgs(program, args), { ...options, maxBuffer: Math.max(options.maxBuffer ?? 0, 64 * 1024 * 1024) });
|
|
61
|
-
return traceCall(String(options.input ?? ''), out, started);
|
|
65
|
+
return { text: traceCall(String(options.input ?? ''), out, started), raw: out };
|
|
62
66
|
}
|
|
63
67
|
catch (e) {
|
|
64
68
|
traceCall(String(options.input ?? ''), String(e.stdout ?? ''), started, String(e));
|
|
@@ -85,7 +89,7 @@ const STAGES = [
|
|
|
85
89
|
[/^You are a senior QA engineer creating per-journey documentation/, 'discover'],
|
|
86
90
|
[/^You are a senior QA engineer writing ONE journey doc/, 'discover'],
|
|
87
91
|
[/^You are a senior QA engineer writing the journey doc for/, 'discover'],
|
|
88
|
-
[/^(?:Fetch this reference
|
|
92
|
+
[/^(?:Fetch this reference|Continue printing this reference)/, 'reference-fetch'],
|
|
89
93
|
[/^You are a senior QA engineer reviewing ONE journey doc/, 'doc-review'],
|
|
90
94
|
[/^You check ONE journey doc/, 'doc-review'],
|
|
91
95
|
[/^You are setting up TestForge in this repository/, 'init'],
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* tests take the code's wording, and the next `generate` records it in the
|
|
9
9
|
* journey's memory as a rename. No AI.
|
|
10
10
|
*/
|
|
11
|
+
import { DOCS_DIR } from '../paths.js';
|
|
11
12
|
import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
12
13
|
import { basename, extname, join, relative } from 'node:path';
|
|
13
14
|
import { findMarkdownFiles } from '../find-files.js';
|
|
@@ -59,7 +60,7 @@ export function bindJourneys(opts = {}) {
|
|
|
59
60
|
const haystack = sourceText(roots);
|
|
60
61
|
let texts;
|
|
61
62
|
const texter = () => (texts ??= shownTexts(rawSource(roots)));
|
|
62
|
-
return journeys(opts.docsDir ??
|
|
63
|
+
return journeys(opts.docsDir ?? DOCS_DIR, opts.journey).map((j) => ({
|
|
63
64
|
journey: j.id,
|
|
64
65
|
files: checkReadiness(testFilesIn(j.dir), haystack, texter).map((r) => ({ ...r, file: relative(process.cwd(), r.file) })),
|
|
65
66
|
}));
|
|
@@ -71,7 +72,7 @@ export function bindJourneys(opts = {}) {
|
|
|
71
72
|
*/
|
|
72
73
|
export function acceptNames(pairs, opts = {}) {
|
|
73
74
|
const notes = [];
|
|
74
|
-
for (const j of journeys(opts.docsDir ??
|
|
75
|
+
for (const j of journeys(opts.docsDir ?? DOCS_DIR, opts.journey)) {
|
|
75
76
|
const files = [j.doc, ...testFilesIn(j.dir)];
|
|
76
77
|
let touched = 0;
|
|
77
78
|
for (const f of files) {
|
|
@@ -49,7 +49,9 @@ export async function completeFile(content, ctx) {
|
|
|
49
49
|
if (first.cov.missing.length || first.problems.length) {
|
|
50
50
|
out = withoutNames(withoutBlocks(content, first.problems));
|
|
51
51
|
drop(first.problems);
|
|
52
|
-
|
|
52
|
+
// Only rules the doc has are written again: an id with no text makes the writer invent one.
|
|
53
|
+
const known = new Set(ctx.memory.rules.filter((r) => r.tag).map((r) => r.id));
|
|
54
|
+
const left = ruleCoverage(out, ctx.rules).missing.filter((id) => known.has(id));
|
|
53
55
|
if (left.length) {
|
|
54
56
|
const blocks = findTestBlocks(out);
|
|
55
57
|
const head = out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000);
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
12
12
|
import { basename, extname, join } from 'node:path';
|
|
13
|
-
import { RUNS_DIR } from '../paths.js';
|
|
13
|
+
import { RUNS_DIR, DOCS_DIR } from '../paths.js';
|
|
14
14
|
import { findMarkdownFiles } from '../find-files.js';
|
|
15
15
|
import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
|
|
16
16
|
import { loadConfig, testDirForType } from '../project/config.js';
|
|
@@ -59,7 +59,7 @@ const testFiles = (dir) => {
|
|
|
59
59
|
};
|
|
60
60
|
export async function runReview(opts = {}) {
|
|
61
61
|
const config = loadConfig();
|
|
62
|
-
const docsDir = opts.docsDir ??
|
|
62
|
+
const docsDir = opts.docsDir ?? DOCS_DIR;
|
|
63
63
|
const out = [];
|
|
64
64
|
for (const file of findMarkdownFiles(docsDir)) {
|
|
65
65
|
const markdown = readFileSync(join(docsDir, file), 'utf-8');
|
|
@@ -573,6 +573,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
573
573
|
const planned = useAI
|
|
574
574
|
? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
|
|
575
575
|
: [{ slug: journey.id, title: journey.name, covers: '' }];
|
|
576
|
+
// A plan kept from an earlier version of the doc may name rules the doc no
|
|
577
|
+
// longer has (reworded into new ids, or removed). A writer asked to test a
|
|
578
|
+
// rule it is given no text for invents one, so those ids are dropped here,
|
|
579
|
+
// and a file left with no rule is not written.
|
|
580
|
+
if (memory) {
|
|
581
|
+
const known = new Set(memory.rules.filter((r) => r.tag).map((r) => r.id));
|
|
582
|
+
for (let k = planned.length - 1; k >= 0; k--) {
|
|
583
|
+
const pf = planned[k];
|
|
584
|
+
if (!pf.rules?.length)
|
|
585
|
+
continue;
|
|
586
|
+
const gone = pf.rules.filter((id) => !known.has(id));
|
|
587
|
+
if (!gone.length)
|
|
588
|
+
continue;
|
|
589
|
+
pf.rules = pf.rules.filter((id) => known.has(id));
|
|
590
|
+
if (!pf.rules.length)
|
|
591
|
+
planned.splice(k, 1);
|
|
592
|
+
}
|
|
593
|
+
}
|
|
576
594
|
// A plan-only run leaves the memory as the tests were last written, so the
|
|
577
595
|
// next generate still sees what changed in the doc and updates those tests.
|
|
578
596
|
if (memory && !opts.planOnly) {
|
|
@@ -18,7 +18,7 @@ import { readFileSync, existsSync, readdirSync } from 'node:fs';
|
|
|
18
18
|
import { join } from 'node:path';
|
|
19
19
|
import { loadConfig, splitCommand, withNoTools, withReadOnlyTools } from '../project/config.js';
|
|
20
20
|
import { ruleIndex } from './journey-memory.js';
|
|
21
|
-
import { MEMORY_DIR } from '../paths.js';
|
|
21
|
+
import { MEMORY_DIR, DOCS_DIR } from '../paths.js';
|
|
22
22
|
import { runAiSync } from './ai-usage.js';
|
|
23
23
|
const MAP_PATH = 'testforge.md';
|
|
24
24
|
/**
|
|
@@ -28,7 +28,7 @@ const MAP_PATH = 'testforge.md';
|
|
|
28
28
|
* well over half of all tests. With the siblings in view, each behaviour is
|
|
29
29
|
* planned once, in the journey that owns it.
|
|
30
30
|
*/
|
|
31
|
-
export function siblingJourneys(journey, docsDir =
|
|
31
|
+
export function siblingJourneys(journey, docsDir = DOCS_DIR) {
|
|
32
32
|
const lines = [];
|
|
33
33
|
const planned = new Set();
|
|
34
34
|
const dir = join(MEMORY_DIR, journey.area);
|
package/dist/core/paths.js
CHANGED
|
@@ -16,13 +16,15 @@ export const OLD_STATE_DIR = 'test-artifacts';
|
|
|
16
16
|
export const CACHE_PATH = join(STATE_DIR, 'cache.json');
|
|
17
17
|
export const SOURCE_PATH = join(STATE_DIR, 'source.json');
|
|
18
18
|
export const COVERAGE_PATH = join(STATE_DIR, 'coverage.json');
|
|
19
|
-
/**
|
|
20
|
-
export const
|
|
19
|
+
/** The journey docs: <DOCS_DIR>/<area>/<journey>.md, and shared notes under _reference/. */
|
|
20
|
+
export const DOCS_DIR = join(STATE_DIR, 'docs');
|
|
21
|
+
/** Everything TestForge keeps about each journey: <MEMORY_DIR>/<area>/<id>/ */
|
|
22
|
+
export const MEMORY_DIR = join(STATE_DIR, 'journeys');
|
|
21
23
|
/** One journey's folder: rules.json, plan.json, plan-check.md, facts.json. */
|
|
22
24
|
export function journeyDir(journey) {
|
|
23
25
|
return join(MEMORY_DIR, (journey.area || 'misc').replace(/[^a-z0-9-]/gi, '-'), journey.id);
|
|
24
26
|
}
|
|
25
|
-
/**
|
|
27
|
+
/** Ticket bundles and the comparison with the map. Reference pages are read fresh, never kept here. */
|
|
26
28
|
export const REFERENCE_DIR = join(STATE_DIR, 'reference');
|
|
27
29
|
/** What one run leaves behind, kept on this machine only. */
|
|
28
30
|
export const RUNS_DIR = join(STATE_DIR, 'runs');
|
|
@@ -43,6 +45,13 @@ export function migrateState() {
|
|
|
43
45
|
moved.push(`${from} → ${to}`);
|
|
44
46
|
};
|
|
45
47
|
move('testforge.diff.md', join(REFERENCE_DIR, 'diff.md'));
|
|
48
|
+
move(join(STATE_DIR, 'memory'), MEMORY_DIR);
|
|
49
|
+
// Reference pages were kept as <12 hex>.md; they are read fresh now.
|
|
50
|
+
for (const f of readdirSafe(REFERENCE_DIR)) {
|
|
51
|
+
if (/^[0-9a-f]{12}\.md$/.test(f))
|
|
52
|
+
move(join(REFERENCE_DIR, f), join(RUNS_DIR, 'reference', f));
|
|
53
|
+
}
|
|
54
|
+
moveDocs(moved);
|
|
46
55
|
const old = OLD_STATE_DIR;
|
|
47
56
|
if (!existsSync(old) || !statSync(old).isDirectory())
|
|
48
57
|
return moved;
|
|
@@ -72,6 +81,32 @@ export function migrateState() {
|
|
|
72
81
|
removeIfEmpty(old);
|
|
73
82
|
return moved;
|
|
74
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Journey docs from the project's docs/ into testforge/docs/: only what
|
|
86
|
+
* TestForge wrote (an <area>/ file opening with "# Journey:", and _reference/),
|
|
87
|
+
* never the project's own documents. Nothing already there is overwritten.
|
|
88
|
+
*/
|
|
89
|
+
function moveDocs(moved) {
|
|
90
|
+
const from = 'docs';
|
|
91
|
+
for (const area of readdirSafe(from)) {
|
|
92
|
+
const dir = join(from, area);
|
|
93
|
+
if (!statSync(dir).isDirectory())
|
|
94
|
+
continue;
|
|
95
|
+
for (const f of readdirSafe(dir)) {
|
|
96
|
+
const src = join(dir, f);
|
|
97
|
+
const to = join(DOCS_DIR, area, f);
|
|
98
|
+
if (existsSync(to) || !statSync(src).isFile() || !f.endsWith('.md'))
|
|
99
|
+
continue;
|
|
100
|
+
if (area !== '_reference' && !/^#\s*Journey:/m.test(readFileSync(src, 'utf-8')))
|
|
101
|
+
continue;
|
|
102
|
+
mkdirSync(dirname(to), { recursive: true });
|
|
103
|
+
renameSync(src, to);
|
|
104
|
+
moved.push(`${src} → ${to}`);
|
|
105
|
+
}
|
|
106
|
+
removeIfEmpty(dir);
|
|
107
|
+
}
|
|
108
|
+
removeIfEmpty(from);
|
|
109
|
+
}
|
|
75
110
|
function readdirSafe(dir) {
|
|
76
111
|
try {
|
|
77
112
|
return readdirSync(dir);
|
|
@@ -92,9 +127,9 @@ function removeIfEmpty(dir) {
|
|
|
92
127
|
/** The ignore file TestForge used to write: everything local. Replaced when found. */
|
|
93
128
|
const OLD_IGNORE = '# testforge local state: regenerated as needed, so not committed.\n*\n!.gitignore\n!source.json\n';
|
|
94
129
|
const IGNORE = [
|
|
95
|
-
'# testforge state. Commit it: each journey\'s
|
|
96
|
-
'#
|
|
97
|
-
'#
|
|
130
|
+
'# testforge state. Commit it: the journey docs and each journey\'s rules, plan and',
|
|
131
|
+
'# code facts cost AI to rebuild, and a teammate who pulls them starts from where',
|
|
132
|
+
'# you left off. Only runs/ (run output, AI logs) stays on this machine.',
|
|
98
133
|
'runs/',
|
|
99
134
|
'',
|
|
100
135
|
].join('\n');
|
|
@@ -4,14 +4,14 @@
|
|
|
4
4
|
* Runs the one-time project preparation in order, with progress so the
|
|
5
5
|
* user can see what is happening at each stage:
|
|
6
6
|
* 1. config + map — one call writes testforge.config.json and testforge.md
|
|
7
|
-
* 2. discover — draft a doc per journey into docs/
|
|
7
|
+
* 2. discover — draft a doc per journey into testforge/docs/
|
|
8
8
|
*
|
|
9
9
|
* Each step is also available as its own command for re-running a
|
|
10
10
|
* single stage; setup just chains them for a smooth first run.
|
|
11
11
|
*/
|
|
12
12
|
import { existsSync } from 'node:fs';
|
|
13
13
|
import { hashProjectSource, loadCache, saveCache } from '../docs/cache.js';
|
|
14
|
-
import { CACHE_PATH } from '../paths.js';
|
|
14
|
+
import { CACHE_PATH, DOCS_DIR } from '../paths.js';
|
|
15
15
|
import { checkSetupConfig, runSetupCall } from './init.js';
|
|
16
16
|
import { loadConfig } from './config.js';
|
|
17
17
|
import { runDiscover } from '../docs/discover.js';
|
|
@@ -21,13 +21,13 @@ export async function runSetup(srcDir, opts = {}) {
|
|
|
21
21
|
// to `update` instead, which only refreshes what actually changed.
|
|
22
22
|
const alreadySetUp = existsSync('testforge.config.json') &&
|
|
23
23
|
existsSync('testforge.md') &&
|
|
24
|
-
existsSync(
|
|
24
|
+
existsSync(DOCS_DIR);
|
|
25
25
|
if (alreadySetUp) {
|
|
26
26
|
console.log('This project is already set up for testforge.');
|
|
27
|
-
console.log(' - testforge.config.json, testforge.md and docs/ exist.\n');
|
|
27
|
+
console.log(' - testforge.config.json, testforge.md and testforge/docs/ exist.\n');
|
|
28
28
|
console.log("You don't need to run setup again. Instead:");
|
|
29
29
|
console.log(' testforge update refresh only what changed');
|
|
30
|
-
console.log(' testforge generate (re)generate tests from docs/');
|
|
30
|
+
console.log(' testforge generate (re)generate tests from testforge/docs/');
|
|
31
31
|
return;
|
|
32
32
|
}
|
|
33
33
|
console.log('Setting up testforge for this project.\n');
|
|
@@ -45,8 +45,8 @@ export async function runSetup(srcDir, opts = {}) {
|
|
|
45
45
|
// Step 3 — discover. This is where the source-of-truth barrier lives:
|
|
46
46
|
// the docs may be based on testforge.md or on an external reference.
|
|
47
47
|
console.log('Step 2/2: Drafting a doc for each journey...');
|
|
48
|
-
await runDiscover(srcDir,
|
|
49
|
-
console.log(' done — docs/\n');
|
|
48
|
+
await runDiscover(srcDir, DOCS_DIR, opts);
|
|
49
|
+
console.log(' done — testforge/docs/\n');
|
|
50
50
|
// Remember the source state so the first `update` can detect "no change".
|
|
51
51
|
const cache = loadCache(CACHE_PATH);
|
|
52
52
|
cache['__source__'] = hashProjectSource(srcDir);
|
|
@@ -54,12 +54,12 @@ export async function runSetup(srcDir, opts = {}) {
|
|
|
54
54
|
// Build an env template so the developer knows which credentials each
|
|
55
55
|
// area/journey needs for running the tests.
|
|
56
56
|
console.log('Building env template...');
|
|
57
|
-
runEnvTemplate(
|
|
57
|
+
runEnvTemplate(DOCS_DIR, '.env.testing.example', srcDir);
|
|
58
58
|
console.log('');
|
|
59
59
|
console.log('Setup complete.');
|
|
60
60
|
if (!loadConfig().dashboardUrl) {
|
|
61
61
|
console.log('To publish runs to the TestForge dashboard, set "dashboardUrl" in testforge.config.json and TESTFORGE_KEY in the env file.');
|
|
62
62
|
}
|
|
63
|
-
console.log('Review the drafts in docs/, fix anything the AI got wrong,');
|
|
63
|
+
console.log('Review the drafts in testforge/docs/, fix anything the AI got wrong,');
|
|
64
64
|
console.log('then run `testforge generate` to create tests and seed files.');
|
|
65
65
|
}
|
|
@@ -15,7 +15,7 @@ import { runGenerate } from '../generate/run-generate.js';
|
|
|
15
15
|
import { hashProjectSource, hashReference, loadCache, saveCache } from '../docs/cache.js';
|
|
16
16
|
import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
17
17
|
import { runEnvTemplate } from './env-template.js';
|
|
18
|
-
import { CACHE_PATH } from '../paths.js';
|
|
18
|
+
import { CACHE_PATH, DOCS_DIR } from '../paths.js';
|
|
19
19
|
const SOURCE_KEY = '__source__';
|
|
20
20
|
const REFERENCE_KEY = '__reference__';
|
|
21
21
|
export async function runUpdate(srcDir, opts = {}) {
|
|
@@ -69,7 +69,7 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
69
69
|
// Reuse the source of truth chosen at setup (map or a reference)
|
|
70
70
|
// without prompting — an update is a refresh, not a re-decision.
|
|
71
71
|
console.log('Updating journey docs...');
|
|
72
|
-
await runDiscover(srcDir,
|
|
72
|
+
await runDiscover(srcDir, DOCS_DIR, { noPrompt: true });
|
|
73
73
|
// Journeys that arrived on their own document — a ticket's attached spec —
|
|
74
74
|
// are refreshed from THAT document. The project specification does not
|
|
75
75
|
// describe them, so refreshing them from it would quietly replace a written
|
|
@@ -78,11 +78,11 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
78
78
|
for (const [doc, ref] of ownRefs) {
|
|
79
79
|
const name = doc.replace(/\.md$/, '').split('/').pop();
|
|
80
80
|
console.log(` refreshing ${name} from its own reference: ${ref}`);
|
|
81
|
-
await runDiscover(srcDir,
|
|
81
|
+
await runDiscover(srcDir, DOCS_DIR, { reference: ref, noPrompt: true }, name);
|
|
82
82
|
}
|
|
83
|
-
console.log(' done — docs/\n');
|
|
83
|
+
console.log(' done — testforge/docs/\n');
|
|
84
84
|
console.log('Regenerating tests for changed journeys...');
|
|
85
|
-
const results = await runGenerate(
|
|
85
|
+
const results = await runGenerate(DOCS_DIR, true, false);
|
|
86
86
|
const made = results.filter((r) => r.status !== 'skipped').length;
|
|
87
87
|
const skipped = results.filter((r) => r.status === 'skipped').length;
|
|
88
88
|
console.log(` done — ${made} updated, ${skipped} unchanged.\n`);
|
|
@@ -91,7 +91,7 @@ export async function runUpdate(srcDir, opts = {}) {
|
|
|
91
91
|
cache[REFERENCE_KEY] = refState.hash;
|
|
92
92
|
saveCache(CACHE_PATH, cache);
|
|
93
93
|
console.log('Refreshing env template...');
|
|
94
|
-
runEnvTemplate(
|
|
94
|
+
runEnvTemplate(DOCS_DIR, '.env.testing.example', srcDir);
|
|
95
95
|
console.log('');
|
|
96
96
|
console.log('Update complete.');
|
|
97
97
|
}
|
|
@@ -277,14 +277,19 @@ export const WEB = `WEB (Playwright)
|
|
|
277
277
|
- A state the test cannot create (a cancelled booking, an expired code, a
|
|
278
278
|
banned account, an empty list) is stubbed with page.route(). Shape the stub
|
|
279
279
|
like the response in <code_facts>, and put "(stubbed state)" in the test
|
|
280
|
-
title.
|
|
280
|
+
title. A stubbed item carries every field the screen reads to show it,
|
|
281
|
+
including the value the list filters on (its status), and the test finds it
|
|
282
|
+
on screen by a field the screen renders as text, never by one it does not
|
|
283
|
+
show. Match the stub by exact host, port, path prefix or a request
|
|
281
284
|
predicate, never by a substring that also matches the app's own pages.
|
|
282
285
|
- Waiting: prefer retrying assertions. Register waitForResponse or
|
|
283
286
|
waitForRequest before the action that triggers it. After a login, sign-out
|
|
284
287
|
or navigation, wait for the control you will use, then assert the URL the
|
|
285
288
|
app chooses. A page that never renders is reported as that, not as the rule
|
|
286
289
|
failing.
|
|
287
|
-
- A missing element is a failure, never a skip.
|
|
290
|
+
- A missing element is a failure, never a skip. kit.find fails when the
|
|
291
|
+
element is not there, so to check that something is not shown use
|
|
292
|
+
page.getByRole(…) or page.getByText(…) with toHaveCount(0) or toBeHidden().
|
|
288
293
|
- For API calls inside a UI test, call the request fixture directly
|
|
289
294
|
(request.get, request.post). For a separate context with its own headers,
|
|
290
295
|
use kit.apiContext(request, role). Never call request.newContext() or
|
|
@@ -16,7 +16,6 @@ import { dirname, join, relative } from 'node:path';
|
|
|
16
16
|
import { STATE_DIR } from '../paths.js';
|
|
17
17
|
import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
18
18
|
import { runAiSync } from '../generate/ai-usage.js';
|
|
19
|
-
import { cleanOutput } from '../generate/ai-generator.js';
|
|
20
19
|
import { readInterface, recordInterface } from '../generate/interface-file.js';
|
|
21
20
|
const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
|
|
22
21
|
const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
|
|
@@ -56,7 +55,7 @@ const SHAPE = {
|
|
|
56
55
|
field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]/,
|
|
57
56
|
// …or a menu entry's label (label: 'Orders').
|
|
58
57
|
// …or a JSX text child on a line of its own (<Button>\n Escalate\n</Button>).
|
|
59
|
-
label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]|\blabel\s*:\s*['"`]|^\s*[A-Z][\w'’ ]{1,40}\s*$/,
|
|
58
|
+
label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]|\blabel\s*:\s*['"`]|^\s*[A-Z][\w'’ ]{1,40}\s*$|\/>\s*[A-Z][\w'’ ]{1,40}\s*$/,
|
|
60
59
|
};
|
|
61
60
|
/** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
|
|
62
61
|
function candidateLines(name, kind, files, root) {
|
|
@@ -216,7 +215,10 @@ export function scanNames(blank, root = '.') {
|
|
|
216
215
|
try {
|
|
217
216
|
const { program, args } = withNoTools(splitCommand(loadConfig().aiCommand), 'Answer only with the JSON object asked for.');
|
|
218
217
|
calls = 1;
|
|
219
|
-
|
|
218
|
+
// The answer is JSON, not code: read it as it is (cleanOutput keeps only code-like lines and drops it).
|
|
219
|
+
const raw = String(runAiSync(program, args, { input: prompt, encoding: 'utf-8', maxBuffer: 4 * 1024 * 1024, timeout: 120_000 }));
|
|
220
|
+
if (raw.indexOf('{') < 0)
|
|
221
|
+
throw new Error(`the answer was not JSON: ${raw.slice(0, 80)}`);
|
|
220
222
|
const json = JSON.parse(raw.slice(raw.indexOf('{'), raw.lastIndexOf('}') + 1));
|
|
221
223
|
const keep = {};
|
|
222
224
|
for (const a of ask) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/reporters/interface.cjs
CHANGED
|
@@ -57,7 +57,25 @@ function callerFile() {
|
|
|
57
57
|
return undefined;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
// A test is declared with its title first; test.skip(condition, 'why') inside a test is not one.
|
|
61
|
+
/** Index of the ")" closing the "(" at `open`, skipping strings, templates and comments; -1 if none. */
|
|
62
|
+
function closingParen(src, open) {
|
|
63
|
+
let depth = 0;
|
|
64
|
+
for (let i = open; i < src.length; i++) {
|
|
65
|
+
const c = src[i];
|
|
66
|
+
if (c === '/' && src[i + 1] === '/') { i = src.indexOf('\n', i); if (i < 0) return -1; continue; }
|
|
67
|
+
if (c === '/' && src[i + 1] === '*') { i = src.indexOf('*/', i + 2); if (i < 0) return -1; i += 1; continue; }
|
|
68
|
+
if (c === '"' || c === "'" || c === '`') {
|
|
69
|
+
for (i += 1; i < src.length && src[i] !== c; i++) if (src[i] === '\\') i += 1;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (c === '(') depth += 1;
|
|
73
|
+
else if (c === ')') { depth -= 1; if (depth === 0) return i; }
|
|
74
|
+
}
|
|
75
|
+
return -1;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const TEST_START = /^\s*(?:test|it)(?:\.(?:only|skip|fixme|fail|slow|concurrent))?\s*\(\s*['"`]/;
|
|
61
79
|
|
|
62
80
|
/**
|
|
63
81
|
* Which tests of a file use the constant a blank name was assigned to, worked
|
|
@@ -74,15 +92,16 @@ function usersOf(file, name) {
|
|
|
74
92
|
const v = decl[1];
|
|
75
93
|
const lines = src.split('\n');
|
|
76
94
|
const declLine = src.slice(0, decl.index).split('\n').length - 1;
|
|
77
|
-
// Each test: from its test( line to the
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
95
|
+
// Each test: from its test( line to the bracket that closes the call, so a
|
|
96
|
+
// helper written between two tests belongs to neither of them.
|
|
97
|
+
const starts = [];
|
|
98
|
+
let at = 0;
|
|
99
|
+
lines.forEach((l, i) => { if (TEST_START.test(l)) starts.push({ i, pos: at + l.indexOf('(') }); at += l.length + 1; });
|
|
100
|
+
const lineOf = (pos) => src.slice(0, pos).split('\n').length - 1;
|
|
101
|
+
const tests = starts.map(({ i, pos }) => {
|
|
102
|
+
const close = closingParen(src, pos);
|
|
103
|
+
const end = close < 0 ? lines.length : lineOf(close) + 1;
|
|
104
|
+
return { start: i, end, text: lines.slice(i, end).join('\n'), title: (/\(\s*(['"`])((?:(?!\1).)*)\1/.exec(lines[i]) || [])[2] };
|
|
86
105
|
});
|
|
87
106
|
const uses = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
|
|
88
107
|
const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
|