@activate-core/test-forge 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/cli/index.js +60 -2
- package/dist/core/docs/discover.js +2 -2
- package/dist/core/docs/grounding.js +2 -2
- package/dist/core/docs/reference.js +8 -2
- package/dist/core/docs/review.js +43 -0
- package/dist/core/docs/tickets.js +56 -0
- package/dist/core/gates/coverage.js +11 -3
- package/dist/core/gates/validate.js +109 -0
- package/dist/core/generate/ai-generator.js +52 -3
- package/dist/core/generate/ai-session.js +8 -0
- package/dist/core/generate/ai-usage.js +39 -0
- package/dist/core/generate/defect-review.js +154 -11
- package/dist/core/generate/harness.js +50 -2
- package/dist/core/generate/required-data.js +75 -0
- package/dist/core/generate/review-journey.js +104 -0
- package/dist/core/generate/run-generate.js +26 -4
- package/dist/core/generate/seed-generator.js +2 -2
- package/dist/core/generate/suite-planner.js +10 -2
- package/dist/core/generate/verify-journey.js +46 -5
- package/dist/core/generate/verify.js +9 -4
- package/dist/core/project/env-diff.js +122 -0
- package/dist/core/project/env-template.js +1 -1
- package/dist/core/project/init.js +2 -2
- package/dist/core/project/map.js +2 -2
- package/dist/core/run/ci.js +94 -0
- package/dist/core/run/clean.js +111 -0
- package/dist/core/run/report.js +19 -17
- package/package.json +1 -1
- package/reporters/_shared.cjs +45 -3
- package/reporters/jest-reporter.cjs +5 -2
- package/reporters/playwright-reporter.cjs +10 -2
- package/reporters/run-data.cjs +66 -0
- package/reporters/ui-helpers.cjs +56 -0
package/README.md
CHANGED
|
@@ -285,6 +285,22 @@ the project map list the env var names each journey needs.
|
|
|
285
285
|
|
|
286
286
|
The test runner produces its own pass/fail report.
|
|
287
287
|
|
|
288
|
+
### In CI (GitHub Actions)
|
|
289
|
+
|
|
290
|
+
```bash
|
|
291
|
+
npx testforge ci --github # run only when started by hand
|
|
292
|
+
npx testforge ci --github --schedule weekdays # also Mon–Fri 06:00 UTC (daily, weekly or a cron line too)
|
|
293
|
+
npx testforge ci --github --env-name staging --force
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
The workflow runs `testforge run` when started by hand (and on the schedule, if given), against a
|
|
297
|
+
deployed environment, and publishes each run to the dashboard. It reads its
|
|
298
|
+
values from repository secrets with the same names as your env file
|
|
299
|
+
(TestForge reads only the names, never the values), plus `TESTFORGE_KEY`,
|
|
300
|
+
and the dashboard address from the repository variable
|
|
301
|
+
`TESTFORGE_DASHBOARD_URL`. Point the base URL secrets at the deployed app.
|
|
302
|
+
Test results are kept as a workflow artifact for 14 days.
|
|
303
|
+
|
|
288
304
|
## Configuration
|
|
289
305
|
|
|
290
306
|
`testforge setup` writes `testforge.config.json` for you. You can edit it:
|
package/dist/cli/index.js
CHANGED
|
@@ -20,6 +20,12 @@
|
|
|
20
20
|
*/
|
|
21
21
|
import { VERSION } from '../core/index.js';
|
|
22
22
|
import { runGenerate } from '../core/generate/run-generate.js';
|
|
23
|
+
import { runReview } from '../core/generate/review-journey.js';
|
|
24
|
+
import { runClean } from '../core/run/clean.js';
|
|
25
|
+
import { aiUsage, describeUsage } from '../core/generate/ai-usage.js';
|
|
26
|
+
import { envDiff, describeEnvDiff } from '../core/project/env-diff.js';
|
|
27
|
+
import { bundleTickets } from '../core/docs/tickets.js';
|
|
28
|
+
import { writeGithubWorkflow } from '../core/run/ci.js';
|
|
23
29
|
import { runTests } from '../core/run/run-tests.js';
|
|
24
30
|
import { runDiscover } from '../core/docs/discover.js';
|
|
25
31
|
import { runMap } from '../core/project/map.js';
|
|
@@ -45,8 +51,15 @@ function flagValue(name) {
|
|
|
45
51
|
}
|
|
46
52
|
/** Source-of-truth options shared by setup and discover. */
|
|
47
53
|
function sourceOptions() {
|
|
54
|
+
const tickets = flagValue('--tickets');
|
|
55
|
+
let reference = flagValue('--reference');
|
|
56
|
+
if (!reference && tickets) {
|
|
57
|
+
const bundle = bundleTickets(tickets);
|
|
58
|
+
console.log(`Using ${bundle.ids.length} exported ticket(s) from ${tickets} as the reference: ${bundle.ids.slice(0, 8).join(', ')}${bundle.ids.length > 8 ? ', …' : ''}`);
|
|
59
|
+
reference = bundle.path;
|
|
60
|
+
}
|
|
48
61
|
return {
|
|
49
|
-
reference
|
|
62
|
+
reference,
|
|
50
63
|
source: flagValue('--source'),
|
|
51
64
|
replace: process.argv.includes('--replace'),
|
|
52
65
|
};
|
|
@@ -84,6 +97,7 @@ function printUsage() {
|
|
|
84
97
|
console.log(' testforge discover Draft journey docs (asks: map or reference)');
|
|
85
98
|
console.log(' testforge discover --journey <name> Draft just ONE journey doc');
|
|
86
99
|
console.log(' testforge discover --reference <file|url> Base docs on a reference');
|
|
100
|
+
console.log(' testforge discover --tickets <folder> Base docs on exported tickets (one file per ticket); each source names its ticket');
|
|
87
101
|
console.log(' (a reference only writes the journeys it');
|
|
88
102
|
console.log(' describes — add --replace to let it');
|
|
89
103
|
console.log(' redraft the whole project)');
|
|
@@ -91,6 +105,10 @@ function printUsage() {
|
|
|
91
105
|
console.log(' testforge init Inspect project and write config');
|
|
92
106
|
console.log(' testforge update Re-check and update only what changed');
|
|
93
107
|
console.log(' testforge env Write .env.testing.example from docs');
|
|
108
|
+
console.log(' testforge env --diff Only the variables the suite reads that the env file does not set');
|
|
109
|
+
console.log(' testforge ci --github Write a GitHub Actions workflow that runs the suite against a deployed environment');
|
|
110
|
+
console.log(' testforge review [--journey j] [--rerun] Judge the last failures again (defect / test fault / environment …), no repair');
|
|
111
|
+
console.log(' testforge clean [--run <id> | --older-than <days>] [--yes] Remove what earlier runs created (dry run without --yes)');
|
|
94
112
|
console.log(' testforge run [--area a] [--journey j] Run the generated tests (Jest/Playwright)');
|
|
95
113
|
console.log(' testforge run --env-file <file> Run against another server (e.g. .env.dev / .env.staging)');
|
|
96
114
|
console.log(' testforge run --headed | --ui | --debug Watch the browser / open the Playwright UI');
|
|
@@ -116,6 +134,33 @@ async function main() {
|
|
|
116
134
|
console.log(output);
|
|
117
135
|
console.log('\nProject map written to testforge.md');
|
|
118
136
|
}
|
|
137
|
+
else if (command === 'ci') {
|
|
138
|
+
if (!process.argv.includes('--github')) {
|
|
139
|
+
console.log('Usage: testforge ci --github [--schedule none|daily|weekdays|weekly|"<cron>"] [--env-name <label>] [--force]');
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
console.log(writeGithubWorkflow({ envName: flagValue('--env-name'), force: process.argv.includes('--force'), schedule: flagValue('--schedule') }));
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
else if (command === 'clean') {
|
|
146
|
+
const days = flagValue('--older-than');
|
|
147
|
+
console.log(await runClean({ runId: flagValue('--run'), olderThanDays: days != null ? Number(days) : undefined, yes: process.argv.includes('--yes') }));
|
|
148
|
+
}
|
|
149
|
+
else if (command === 'review') {
|
|
150
|
+
console.log('Reviewing the last failures of each journey (no repair)...\n');
|
|
151
|
+
const summaries = await runReview({ journey: flagValue('--journey'), rerun: process.argv.includes('--rerun') });
|
|
152
|
+
if (!summaries.length)
|
|
153
|
+
console.log('No generated journey matched.');
|
|
154
|
+
for (const r of summaries) {
|
|
155
|
+
if (r.skipped)
|
|
156
|
+
console.log(` ${r.journey}: not reviewed — ${r.skipped}`);
|
|
157
|
+
else if (!r.failures)
|
|
158
|
+
console.log(` ${r.journey}: no failures to review`);
|
|
159
|
+
else
|
|
160
|
+
console.log(` ${r.journey}: ${r.failures} failure(s) → ` + Object.entries(r.verdicts).map(([k, n]) => `${n} ${k}`).join(', '));
|
|
161
|
+
}
|
|
162
|
+
console.log('\nVerdicts written to test-artifacts/defect-review/. The next run publishes them with each failed test.');
|
|
163
|
+
}
|
|
119
164
|
else if (command === 'discover') {
|
|
120
165
|
console.log('Drafting journey files with AI...');
|
|
121
166
|
console.log('This writes drafts into docs/ from your chosen source.\n');
|
|
@@ -125,7 +170,10 @@ async function main() {
|
|
|
125
170
|
console.log('then run `testforge generate`.');
|
|
126
171
|
}
|
|
127
172
|
else if (command === 'env') {
|
|
128
|
-
|
|
173
|
+
if (process.argv.includes('--diff'))
|
|
174
|
+
console.log(describeEnvDiff(envDiff('.')));
|
|
175
|
+
else
|
|
176
|
+
runEnvTemplate('docs', '.env.testing.example', '.', generatedScanDirs());
|
|
129
177
|
}
|
|
130
178
|
else if (command === 'run') {
|
|
131
179
|
// Anything after `--`, plus the runner flags people reach for most, goes
|
|
@@ -202,6 +250,16 @@ async function main() {
|
|
|
202
250
|
const seedFailures = results.filter((r) => r.seedStatus === 'failed').length;
|
|
203
251
|
console.log(`\nDone. ${generated} generated, ${skipped} skipped (unchanged)` +
|
|
204
252
|
(notices ? `, ${notices} notice(s) above.` : '.'));
|
|
253
|
+
const spent = aiUsage();
|
|
254
|
+
if (spent.calls)
|
|
255
|
+
console.log(`AI this run: ${describeUsage(spent)}.`);
|
|
256
|
+
// What the journeys just written still need from the env file.
|
|
257
|
+
const touched = results.filter((r) => r.testDir && r.status !== 'skipped').map((r) => basename(r.testDir));
|
|
258
|
+
if (touched.length) {
|
|
259
|
+
const d = envDiff('.', touched);
|
|
260
|
+
if (d.journeys.some((j) => j.missing.some((v) => !v.hasDefault)))
|
|
261
|
+
console.log('\n' + describeEnvDiff({ ...d, journeys: d.journeys.map((j) => ({ ...j, missing: j.missing.filter((v) => !v.hasDefault) })) }));
|
|
262
|
+
}
|
|
205
263
|
if (seedFailures > 0) {
|
|
206
264
|
console.log(`Warning: ${seedFailures} seed file(s) could not be generated — see notes above.`);
|
|
207
265
|
}
|
|
@@ -14,7 +14,6 @@
|
|
|
14
14
|
* tools) rather than stuffing all source code into the prompt, which
|
|
15
15
|
* keeps this simple and works for projects of any size.
|
|
16
16
|
*/
|
|
17
|
-
import { execFileSync } from 'node:child_process';
|
|
18
17
|
import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, renameSync, } from 'node:fs';
|
|
19
18
|
import { join, relative, sep, basename } from 'node:path';
|
|
20
19
|
import { notAJourneyDoc } from './parser.js';
|
|
@@ -24,6 +23,7 @@ import { promptConfirm } from '../prompt.js';
|
|
|
24
23
|
import { loadSourceChoice, resolveDocsSource, saveJourneyReferences, saveSourceChoice, } from './source-of-truth.js';
|
|
25
24
|
import { analyzeReference, draftDocsFromReference, DIFF_PATH, } from './reference.js';
|
|
26
25
|
import { reviewDocs } from './review.js';
|
|
26
|
+
import { runAiSync } from '../generate/ai-usage.js';
|
|
27
27
|
/** Scope line: one journey, or all of them. */
|
|
28
28
|
function scopeLines(journey) {
|
|
29
29
|
return journey
|
|
@@ -125,7 +125,7 @@ function runCodeDiscover(srcDir, docsDir, journey) {
|
|
|
125
125
|
// are tool-specific; for Claude we add acceptEdits. Other tools may
|
|
126
126
|
// need different setup (see README).
|
|
127
127
|
const { program, args } = withEditTools(splitCommand(loadConfig().aiCommand));
|
|
128
|
-
const output =
|
|
128
|
+
const output = runAiSync(program, args, {
|
|
129
129
|
input: prompt,
|
|
130
130
|
encoding: 'utf-8',
|
|
131
131
|
maxBuffer: 10 * 1024 * 1024,
|
|
@@ -17,7 +17,6 @@
|
|
|
17
17
|
* journey document remains the sole source of what is CORRECT, and the
|
|
18
18
|
* generation prompt says so in the same breath as it presents these facts.
|
|
19
19
|
*/
|
|
20
|
-
import { execFileSync } from 'node:child_process';
|
|
21
20
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
22
21
|
import { dirname, join } from 'node:path';
|
|
23
22
|
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
@@ -26,6 +25,7 @@ import { STATE_DIR } from '../paths.js';
|
|
|
26
25
|
import { loadEnvFile } from '../project/config.js';
|
|
27
26
|
import { originsFrom, pathsFrom, observeApp } from './observe.js';
|
|
28
27
|
import { observeSchema } from './schema.js';
|
|
28
|
+
import { runAiSync } from '../generate/ai-usage.js';
|
|
29
29
|
const MAP_PATH = 'testforge.md';
|
|
30
30
|
/**
|
|
31
31
|
* Bumped whenever the SHAPE of a facts block changes.
|
|
@@ -143,7 +143,7 @@ export async function groundJourney(journey, docHash, srcDir = 'src', markdown =
|
|
|
143
143
|
const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
|
|
144
144
|
let facts;
|
|
145
145
|
try {
|
|
146
|
-
facts =
|
|
146
|
+
facts = runAiSync(program, args, {
|
|
147
147
|
input: buildGroundingPrompt(journey, map, srcDir),
|
|
148
148
|
encoding: 'utf-8',
|
|
149
149
|
maxBuffer: 10 * 1024 * 1024,
|
|
@@ -21,16 +21,16 @@
|
|
|
21
21
|
* URLs/Notion, web) access. As with discover/map this is tool-specific
|
|
22
22
|
* and tuned for the Claude CLI.
|
|
23
23
|
*/
|
|
24
|
-
import { execFileSync } from 'node:child_process';
|
|
25
24
|
import { loadConfig, splitCommand, withReferenceTools } from '../project/config.js';
|
|
26
25
|
import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
|
|
26
|
+
import { runAiSync } from '../generate/ai-usage.js';
|
|
27
27
|
/** Where the comparison report is written. */
|
|
28
28
|
export const DIFF_PATH = 'testforge.diff.md';
|
|
29
29
|
const MAP_PATH = 'testforge.md';
|
|
30
30
|
/** Run the configured AI command with read/write/web (+ Notion) access. */
|
|
31
31
|
function runAI(prompt) {
|
|
32
32
|
const { program, args } = withReferenceTools(splitCommand(loadConfig().aiCommand));
|
|
33
|
-
const output =
|
|
33
|
+
const output = runAiSync(program, args, {
|
|
34
34
|
input: prompt,
|
|
35
35
|
encoding: 'utf-8',
|
|
36
36
|
maxBuffer: 10 * 1024 * 1024,
|
|
@@ -109,6 +109,12 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
|
|
|
109
109
|
DEEP_DOC_GUIDANCE,
|
|
110
110
|
``,
|
|
111
111
|
`Reference: ${reference}`,
|
|
112
|
+
...(reference.endsWith('tickets-reference.md')
|
|
113
|
+
? [
|
|
114
|
+
`The reference is a bundle of exported tickets, one section per ticket id.`,
|
|
115
|
+
`Every [spec] source names its ticket: (source: <TICKET-ID>: "<quoted sentence>").`,
|
|
116
|
+
]
|
|
117
|
+
: []),
|
|
112
118
|
...(loadConfig().decisions
|
|
113
119
|
? [
|
|
114
120
|
`NOT YET REQUIREMENTS: the project keeps its open product decisions and`,
|
package/dist/core/docs/review.js
CHANGED
|
@@ -10,9 +10,11 @@
|
|
|
10
10
|
* tiers that claim more than their source says. The reviewer edits the file
|
|
11
11
|
* in place and never deletes a sound criterion.
|
|
12
12
|
*/
|
|
13
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
13
14
|
import { join } from 'node:path';
|
|
14
15
|
import { loadConfig, splitCommand, withEditTools, withReferenceTools } from '../project/config.js';
|
|
15
16
|
import { runWithStdin } from '../generate/ai-generator.js';
|
|
17
|
+
import { HAS_SOURCE } from '../generate/defect-review.js';
|
|
16
18
|
import { pool } from '../generate/run-generate.js';
|
|
17
19
|
import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
|
|
18
20
|
/** Build the prompt that reviews and completes ONE journey doc. */
|
|
@@ -59,6 +61,46 @@ export function buildReviewPrompt(docPath, srcDir, reference) {
|
|
|
59
61
|
`with counts.`,
|
|
60
62
|
].join('\n');
|
|
61
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* A [spec] line is a requirement the client wrote, so it must carry the
|
|
66
|
+
* client's words: `(source: "…")`, or `(source as above)` pointing at the quote before it. A [spec] line without that quote is the
|
|
67
|
+
* drafter's reading, and it is re-tagged [observed] — never given an invented
|
|
68
|
+
* quote. Fenced code blocks are left alone.
|
|
69
|
+
*/
|
|
70
|
+
export function enforceSpecSources(markdown) {
|
|
71
|
+
let inFence = false;
|
|
72
|
+
let retagged = 0;
|
|
73
|
+
const lines = markdown.split('\n').map((line) => {
|
|
74
|
+
if (/^\s*```/.test(line))
|
|
75
|
+
inFence = !inFence;
|
|
76
|
+
if (inFence || !/\[spec\]/i.test(line) || HAS_SOURCE.test(line))
|
|
77
|
+
return line;
|
|
78
|
+
retagged += 1;
|
|
79
|
+
return line.replace(/\[spec\]/gi, '[observed]');
|
|
80
|
+
});
|
|
81
|
+
return { markdown: lines.join('\n'), retagged };
|
|
82
|
+
}
|
|
83
|
+
/** Apply the source-quote rule to docs on disk; returns how many lines were re-tagged. */
|
|
84
|
+
export function enforceSpecSourcesOnDisk(docsDir, docs) {
|
|
85
|
+
let total = 0;
|
|
86
|
+
for (const doc of docs) {
|
|
87
|
+
const path = join(docsDir, doc);
|
|
88
|
+
let text;
|
|
89
|
+
try {
|
|
90
|
+
text = readFileSync(path, 'utf-8');
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const { markdown, retagged } = enforceSpecSources(text);
|
|
96
|
+
if (!retagged)
|
|
97
|
+
continue;
|
|
98
|
+
writeFileSync(path, markdown, 'utf-8');
|
|
99
|
+
console.log(` ${doc}: ${retagged} [spec] line(s) without a quoted source re-tagged [observed]`);
|
|
100
|
+
total += retagged;
|
|
101
|
+
}
|
|
102
|
+
return total;
|
|
103
|
+
}
|
|
62
104
|
/**
|
|
63
105
|
* Review each doc (paths relative to docsDir) in its own AI pass, a few at
|
|
64
106
|
* a time. A failed review leaves that doc as drafted and is reported, never
|
|
@@ -81,4 +123,5 @@ export async function reviewDocs(docsDir, docs, srcDir, reference) {
|
|
|
81
123
|
console.log(` ${doc}: review failed, kept as drafted (${err.message.split('\n')[0]})`);
|
|
82
124
|
}
|
|
83
125
|
});
|
|
126
|
+
enforceSpecSourcesOnDisk(docsDir, docs);
|
|
84
127
|
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `--tickets <folder>`: a folder of exported tickets (one file per ticket, any
|
|
3
|
+
* text format) used as the reference. The files are bundled, in ticket order,
|
|
4
|
+
* into one reference document under the state folder, each under a heading
|
|
5
|
+
* that carries its ticket id, so every quoted source can name its ticket.
|
|
6
|
+
* Nothing is fetched from a tracker: the person exports what is in scope.
|
|
7
|
+
*/
|
|
8
|
+
import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
9
|
+
import { basename, extname, join } from 'node:path';
|
|
10
|
+
import { STATE_DIR } from '../paths.js';
|
|
11
|
+
export const TICKETS_REFERENCE = join(STATE_DIR, 'tickets-reference.md');
|
|
12
|
+
const TEXT = /\.(md|markdown|txt|text|json|csv|html?|xml|ya?ml)$/i;
|
|
13
|
+
const ID = /\b([A-Z][A-Z0-9]+-\d+)\b/;
|
|
14
|
+
const MAX_CHARS = 20_000;
|
|
15
|
+
/** The ticket id a file stands for: in its name, else in its first lines, else its name. */
|
|
16
|
+
export function ticketId(file, content) {
|
|
17
|
+
return ID.exec(basename(file))?.[1] ?? ID.exec(content.split('\n').slice(0, 5).join('\n'))?.[1] ?? basename(file, extname(file));
|
|
18
|
+
}
|
|
19
|
+
const order = (a, b) => {
|
|
20
|
+
const pa = /^(.*)-(\d+)$/.exec(a), pb = /^(.*)-(\d+)$/.exec(b);
|
|
21
|
+
return pa && pb && pa[1] === pb[1] ? Number(pa[2]) - Number(pb[2]) : a.localeCompare(b);
|
|
22
|
+
};
|
|
23
|
+
/** Bundle every text file under the folder into the tickets reference. Returns its path and the ids. */
|
|
24
|
+
export function bundleTickets(folder) {
|
|
25
|
+
const files = [];
|
|
26
|
+
const walk = (d) => {
|
|
27
|
+
for (const e of readdirSync(d, { withFileTypes: true })) {
|
|
28
|
+
const p = join(d, e.name);
|
|
29
|
+
if (e.isDirectory())
|
|
30
|
+
walk(p);
|
|
31
|
+
else if (TEXT.test(e.name))
|
|
32
|
+
files.push(p);
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
if (!statSync(folder).isDirectory())
|
|
36
|
+
throw new Error(`--tickets needs a folder of exported tickets: ${folder} is not one`);
|
|
37
|
+
walk(folder);
|
|
38
|
+
if (!files.length)
|
|
39
|
+
throw new Error(`--tickets: no text files in ${folder} (export each ticket as .md, .txt, .json, .html, …)`);
|
|
40
|
+
const tickets = files
|
|
41
|
+
.map((f) => { const content = readFileSync(f, 'utf-8'); return { id: ticketId(f, content), file: f, content }; })
|
|
42
|
+
.sort((a, b) => order(a.id, b.id));
|
|
43
|
+
const body = [
|
|
44
|
+
'# Tickets',
|
|
45
|
+
'',
|
|
46
|
+
`Exported tickets used as the reference (${tickets.length}). Each section below is one ticket,`,
|
|
47
|
+
'headed by its id. Every [spec] criterion drawn from them names its ticket in its',
|
|
48
|
+
'quoted source. Where tickets disagree, the journey lists it as a question for',
|
|
49
|
+
'product instead of picking one.',
|
|
50
|
+
'',
|
|
51
|
+
...tickets.flatMap((t) => [`## ${t.id} — ${basename(t.file)}`, '', t.content.trim().slice(0, MAX_CHARS), '']),
|
|
52
|
+
].join('\n');
|
|
53
|
+
mkdirSync(STATE_DIR, { recursive: true });
|
|
54
|
+
writeFileSync(TICKETS_REFERENCE, body, 'utf-8');
|
|
55
|
+
return { path: TICKETS_REFERENCE, ids: tickets.map((t) => t.id) };
|
|
56
|
+
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* as full coverage.
|
|
8
8
|
*/
|
|
9
9
|
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
10
|
-
import { analyzeFile } from './validate.js';
|
|
10
|
+
import { analyzeFile, testsByTier } from './validate.js';
|
|
11
11
|
import { COVERAGE_PATH, STATE_DIR } from '../paths.js';
|
|
12
12
|
/**
|
|
13
13
|
* F8 — write an honest coverage manifest. For every generated file it
|
|
@@ -21,14 +21,19 @@ export function writeCoverageManifest(results) {
|
|
|
21
21
|
.map((r) => {
|
|
22
22
|
const files = r.testFiles.map((t) => {
|
|
23
23
|
let a = { activeTests: 0, pending: 0, assertions: 0, real: false };
|
|
24
|
+
let tiers = { spec: 0, baseline: 0, observed: 0, untagged: 0 };
|
|
24
25
|
try {
|
|
25
|
-
|
|
26
|
+
const content = readFileSync(t.path, 'utf-8');
|
|
27
|
+
a = analyzeFile(content);
|
|
28
|
+
tiers = testsByTier(content);
|
|
26
29
|
}
|
|
27
30
|
catch {
|
|
28
31
|
/* file unreadable — count as empty */
|
|
29
32
|
}
|
|
30
|
-
return { file: t.path, status: t.status, ...a };
|
|
33
|
+
return { file: t.path, status: t.status, ...a, tiers };
|
|
31
34
|
});
|
|
35
|
+
// How many tests rest on a client requirement, and how many on robustness or what the app does.
|
|
36
|
+
const tiers = files.reduce((acc, f) => ({ spec: acc.spec + f.tiers.spec, baseline: acc.baseline + f.tiers.baseline, observed: acc.observed + f.tiers.observed, untagged: acc.untagged + f.tiers.untagged }), { spec: 0, baseline: 0, observed: 0, untagged: 0 });
|
|
32
37
|
return {
|
|
33
38
|
journey: r.journeyName,
|
|
34
39
|
real: files.filter((f) => f.real).length,
|
|
@@ -37,6 +42,7 @@ export function writeCoverageManifest(results) {
|
|
|
37
42
|
// and still target something the app does not have, so "real" alone
|
|
38
43
|
// was never enough to call the coverage honest.
|
|
39
44
|
verified: r.verified,
|
|
45
|
+
tiers,
|
|
40
46
|
files,
|
|
41
47
|
};
|
|
42
48
|
});
|
|
@@ -59,6 +65,8 @@ export function writeCoverageManifest(results) {
|
|
|
59
65
|
/* best-effort */
|
|
60
66
|
}
|
|
61
67
|
console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see ${COVERAGE_PATH}`);
|
|
68
|
+
const t = journeys.reduce((a, j) => ({ spec: a.spec + j.tiers.spec, baseline: a.baseline + j.tiers.baseline, observed: a.observed + j.tiers.observed, untagged: a.untagged + j.tiers.untagged }), { spec: 0, baseline: 0, observed: 0, untagged: 0 });
|
|
69
|
+
console.log(`Tests by the rule they check: ${t.spec} client requirement [spec], ${t.baseline} robustness [baseline], ${t.observed} current behaviour [observed]${t.untagged ? `, ${t.untagged} with no tagged rule` : ''}.`);
|
|
62
70
|
const ranGreen = journeys.filter((j) => j.verified === true).length;
|
|
63
71
|
const ranRed = journeys.filter((j) => j.verified === false).length;
|
|
64
72
|
const unrun = journeys.filter((j) => j.verified === undefined).length;
|
|
@@ -176,6 +176,28 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
176
176
|
return `relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`;
|
|
177
177
|
}
|
|
178
178
|
}
|
|
179
|
+
// F18 — no fixed sleeps. A wait for a set time either wastes it or is too
|
|
180
|
+
// short on a slower machine; wait for the event or the state instead.
|
|
181
|
+
if (ctx.framework === 'playwright' && ctx.forbidPlaceholders && /\bwaitForTimeout\s*\(/.test(content)) {
|
|
182
|
+
return 'page.waitForTimeout() is not used: wait for the element, response or URL the test needs (expect(...).toBeVisible(), page.waitForResponse(), expect(page).toHaveURL())';
|
|
183
|
+
}
|
|
184
|
+
// F19 — a long test timeout needs a reason. A 240 s budget hides a test that
|
|
185
|
+
// is stuck; the cap makes it fail early with its real error.
|
|
186
|
+
if (ctx.maxTestTimeoutMs) {
|
|
187
|
+
const long = longTimeoutsWithoutReason(content, ctx.maxTestTimeoutMs);
|
|
188
|
+
if (long.length) {
|
|
189
|
+
return `test.setTimeout(${long[0]}) is above ${ctx.maxTestTimeoutMs} ms with no "// timeout: <why>" comment: wait for what the test needs instead, or state why it takes that long`;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
// F20 — one criterion, a few tests. Thirty inputs against a rule that only
|
|
193
|
+
// checks "not empty" prove nothing the first did not; they cost run time and
|
|
194
|
+
// turn one defect into thirty red rows.
|
|
195
|
+
if (ctx.maxTestsPerRule && ctx.forbidPlaceholders) {
|
|
196
|
+
const over = testsPerCriterion(content).find((c) => c.tests > ctx.maxTestsPerRule);
|
|
197
|
+
if (over) {
|
|
198
|
+
return `${over.tests} tests check the same criterion (${over.criterion.slice(0, 80)}…): keep at most ${ctx.maxTestsPerRule}, one per branch, and put further inputs in one parametrised test`;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
179
201
|
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
180
202
|
// as a bare "fixme", indistinguishable from the tool giving up. Runs on
|
|
181
203
|
// content normalisePending() has already given every reason it could find.
|
|
@@ -409,3 +431,90 @@ export function unresolvedImports(content, filePath) {
|
|
|
409
431
|
&& !RESOLVE_EXTS.slice(1).some((e) => existsSync(join(target, 'index' + e)));
|
|
410
432
|
});
|
|
411
433
|
}
|
|
434
|
+
/**
|
|
435
|
+
* Helpers written into more than one file of a journey: the same top-level
|
|
436
|
+
* function (same body, whitespace aside) defined in two or more files. Such a
|
|
437
|
+
* copy diverges, and one mistake in it fails every file that carries it — the
|
|
438
|
+
* Stripe card-fill helper copied into nine files was exactly that. One-liners
|
|
439
|
+
* (a locator, a URL builder) are left alone.
|
|
440
|
+
*/
|
|
441
|
+
export function duplicatedHelpers(files, minChars = 120) {
|
|
442
|
+
const seen = new Map();
|
|
443
|
+
const START = /^(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(|^(?:export\s+)?const\s+(\w+)\s*=\s*(?:async\s*)?(?:\([^)]*\)|\w+)\s*(?::[^=]+)?=>/;
|
|
444
|
+
for (const f of files) {
|
|
445
|
+
const lines = f.content.split('\n');
|
|
446
|
+
for (let i = 0; i < lines.length; i++) {
|
|
447
|
+
const m = START.exec(lines[i]);
|
|
448
|
+
if (!m)
|
|
449
|
+
continue;
|
|
450
|
+
// The definition runs to the line where its braces close again.
|
|
451
|
+
let depth = 0, j = i, body = '';
|
|
452
|
+
for (; j < lines.length; j++) {
|
|
453
|
+
body += lines[j] + '\n';
|
|
454
|
+
for (const ch of lines[j])
|
|
455
|
+
depth += ch === '{' ? 1 : ch === '}' ? -1 : 0;
|
|
456
|
+
if (depth <= 0 && j > i - 1 && (lines[j].includes('}') || lines[j].trimEnd().endsWith(';')))
|
|
457
|
+
break;
|
|
458
|
+
}
|
|
459
|
+
const norm = body.replace(/\s+/g, '');
|
|
460
|
+
if (norm.length < minChars)
|
|
461
|
+
continue;
|
|
462
|
+
const key = norm;
|
|
463
|
+
const entry = seen.get(key) ?? { name: m[1] ?? m[2], files: new Set() };
|
|
464
|
+
entry.files.add(f.path);
|
|
465
|
+
seen.set(key, entry);
|
|
466
|
+
i = j;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
return [...seen.values()].filter((e) => e.files.size > 1).map((e) => ({ name: e.name, files: [...e.files].sort() }));
|
|
470
|
+
}
|
|
471
|
+
/** test.setTimeout(N) values above the cap whose line, or the line above, carries no "// timeout:" reason. */
|
|
472
|
+
export function longTimeoutsWithoutReason(content, capMs) {
|
|
473
|
+
const lines = content.split('\n');
|
|
474
|
+
const out = [];
|
|
475
|
+
lines.forEach((line, i) => {
|
|
476
|
+
const m = /\btest\.setTimeout\(\s*([\d_]+)\s*\)/.exec(line);
|
|
477
|
+
if (!m)
|
|
478
|
+
return;
|
|
479
|
+
const ms = Number(m[1].replace(/_/g, ''));
|
|
480
|
+
const reason = /\/\/\s*timeout:/i.test(line) || /\/\/\s*timeout:/i.test(lines[i - 1] ?? '');
|
|
481
|
+
if (ms > capMs && !reason)
|
|
482
|
+
out.push(ms);
|
|
483
|
+
});
|
|
484
|
+
return out;
|
|
485
|
+
}
|
|
486
|
+
/** Each test's criterion: the first "// oracle:" line inside it, with its tier. */
|
|
487
|
+
function testCriteria(content) {
|
|
488
|
+
const lines = content.split('\n');
|
|
489
|
+
const starts = [];
|
|
490
|
+
lines.forEach((l, i) => { if (/^\s*(?:test|it)\s*\(/.test(l))
|
|
491
|
+
starts.push(i); });
|
|
492
|
+
return starts.map((start, n) => {
|
|
493
|
+
const end = starts[n + 1] ?? lines.length;
|
|
494
|
+
for (let i = start; i < end; i++) {
|
|
495
|
+
const m = /\/\/\s*oracle:\s*(.+)$/.exec(lines[i]);
|
|
496
|
+
if (m) {
|
|
497
|
+
const tier = (/\[(spec|baseline|observed)\]/i.exec(m[1])?.[1]?.toLowerCase() ?? 'untagged');
|
|
498
|
+
// The criterion is the quoted rule when there is one, else the oracle text.
|
|
499
|
+
const quoted = /["“]([^"”]{12,})["”]/.exec(m[1]);
|
|
500
|
+
return { criterion: (quoted ? quoted[1] : m[1]).trim().toLowerCase(), tier };
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
return { criterion: '', tier: 'untagged' };
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
/** How many tests of a file check each criterion (tests with no oracle are not counted). */
|
|
507
|
+
export function testsPerCriterion(content) {
|
|
508
|
+
const counts = new Map();
|
|
509
|
+
for (const c of testCriteria(content))
|
|
510
|
+
if (c.criterion)
|
|
511
|
+
counts.set(c.criterion, (counts.get(c.criterion) ?? 0) + 1);
|
|
512
|
+
return [...counts].map(([criterion, tests]) => ({ criterion, tests })).sort((a, b) => b.tests - a.tests);
|
|
513
|
+
}
|
|
514
|
+
/** Tests of a file by the tier of the criterion they check. */
|
|
515
|
+
export function testsByTier(content) {
|
|
516
|
+
const out = { spec: 0, baseline: 0, observed: 0, untagged: 0 };
|
|
517
|
+
for (const c of testCriteria(content))
|
|
518
|
+
out[c.tier] += 1;
|
|
519
|
+
return out;
|
|
520
|
+
}
|
|
@@ -10,15 +10,18 @@
|
|
|
10
10
|
* We call it in print mode (`claude -p`) and feed the prompt via
|
|
11
11
|
* stdin so long prompts and special characters are handled safely.
|
|
12
12
|
*/
|
|
13
|
-
import {
|
|
13
|
+
import { spawn } from 'node:child_process';
|
|
14
14
|
import { readFileSync, existsSync } from 'node:fs';
|
|
15
15
|
import { loadConfig, splitCommand, withReadOnlyTools, resolveFramework, } from '../project/config.js';
|
|
16
|
+
import { runAiSync, noteAiCall } from './ai-usage.js';
|
|
16
17
|
const MAP_PATH = 'testforge.md';
|
|
17
18
|
/**
|
|
18
19
|
* Spawn a command, feed `input` on stdin, resolve with its stdout.
|
|
19
20
|
* Async (unlike execFileSync) so many calls can run concurrently.
|
|
20
21
|
*/
|
|
21
22
|
export function runWithStdin(program, args, input, timeoutMs = 300_000) {
|
|
23
|
+
const started = Date.now();
|
|
24
|
+
let produced = '';
|
|
22
25
|
return new Promise((resolve, reject) => {
|
|
23
26
|
const child = spawn(program, args, {
|
|
24
27
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
@@ -47,6 +50,7 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
|
|
|
47
50
|
child.on('error', (e) => finish(() => reject(e)));
|
|
48
51
|
child.on('close', (code) => {
|
|
49
52
|
finish(() => {
|
|
53
|
+
produced = out;
|
|
50
54
|
if (code === 0)
|
|
51
55
|
resolve(out);
|
|
52
56
|
else {
|
|
@@ -60,7 +64,7 @@ export function runWithStdin(program, args, input, timeoutMs = 300_000) {
|
|
|
60
64
|
});
|
|
61
65
|
});
|
|
62
66
|
child.stdin.end(input);
|
|
63
|
-
});
|
|
67
|
+
}).finally(() => noteAiCall(input.length, produced.length, Date.now() - started));
|
|
64
68
|
}
|
|
65
69
|
/** Build the prompt we send to the AI for one journey. */
|
|
66
70
|
export function buildPrompt(journey, map, testPath, focus, harnessPath, transport, grounding) {
|
|
@@ -234,6 +238,38 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
234
238
|
` (change decoded bytes, not an encoded character), and the valid control`,
|
|
235
239
|
` is asserted too where you can.`,
|
|
236
240
|
``,
|
|
241
|
+
`USE THE TESTKIT'S HELPERS, NEVER YOUR OWN COPY. A helper written into several`,
|
|
242
|
+
`files diverges, and one mistake in it fails every file at once.`,
|
|
243
|
+
` - kit.settledClick(locator) for a button inside a pop-up or a form that`,
|
|
244
|
+
` grows while you fill it (payment, address, any modal with a scrollbar);`,
|
|
245
|
+
` kit.fillFrameInputs(page, framePrefix, { name: value }) for inputs inside`,
|
|
246
|
+
` an embedded iframe; kit.watchDialogs(page) to accept and record alerts.`,
|
|
247
|
+
` - Anything else two files of this journey would both need belongs in the`,
|
|
248
|
+
` testkit, not copied into each file. Small one-line locators are fine.`,
|
|
249
|
+
``,
|
|
250
|
+
`A LATER STEP OF A FLOW IS OPENED THROUGH THE TESTKIT. Never page.goto() a`,
|
|
251
|
+
`later step of a multi-step flow by its URL alone: call`,
|
|
252
|
+
`kit.openStep(page, step, input) and skip with its reason when it does not`,
|
|
253
|
+
`resolve true. Only a test that checks the guard itself (what a visitor who`,
|
|
254
|
+
`skipped the earlier steps sees) opens the bare URL.`,
|
|
255
|
+
``,
|
|
256
|
+
`DATA A TEST CREATES. Other runs use the same accounts, so what they left`,
|
|
257
|
+
`behind is already there when this test starts.`,
|
|
258
|
+
` - A test that creates a record (a booking, an order, a vehicle, an`,
|
|
259
|
+
` address, a payment) does it as the testkit's "run" role, makes every`,
|
|
260
|
+
` identifying value with kit.unique(label) (a registration, a name, a`,
|
|
261
|
+
` note), and records what it made with kit.created(...) unless a testkit`,
|
|
262
|
+
` helper already did.`,
|
|
263
|
+
` - It asserts on what IT created — found by its own id or its unique value —`,
|
|
264
|
+
` never on a count of an account-wide list, "the first" or "the only" item,`,
|
|
265
|
+
` or a list being empty. "Exactly one booking for this request" is checked`,
|
|
266
|
+
` among the records that carry this test's unique value.`,
|
|
267
|
+
` - A test that only reads data uses the fixture role and the seeded ids.`,
|
|
268
|
+
``,
|
|
269
|
+
`AT MOST ${maxPerRule()} TESTS FOR ONE CRITERION. Two tests whose oracle quotes the same`,
|
|
270
|
+
`criterion must reach different branches; more inputs for that criterion go`,
|
|
271
|
+
`into ONE parametrised test (a loop over rows), not more tests.`,
|
|
272
|
+
``,
|
|
237
273
|
`COVER EACH OUTCOME ONCE. A suite is judged by the defects it can catch, not`,
|
|
238
274
|
`by its size; every redundant test costs run time and buries the ones that`,
|
|
239
275
|
`matter.`,
|
|
@@ -504,7 +540,7 @@ export function generateWithAI(journey, testPath, focus, grounding) {
|
|
|
504
540
|
// still can't create files — it reads, then prints the code.
|
|
505
541
|
const config = loadConfig();
|
|
506
542
|
const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
|
|
507
|
-
const output =
|
|
543
|
+
const output = runAiSync(program, args, {
|
|
508
544
|
input: prompt,
|
|
509
545
|
encoding: 'utf-8',
|
|
510
546
|
maxBuffer: 10 * 1024 * 1024,
|
|
@@ -521,7 +557,11 @@ export function generateWithAI(journey, testPath, focus, grounding) {
|
|
|
521
557
|
* `claude` process). Same prompt, same read-only exploration.
|
|
522
558
|
*/
|
|
523
559
|
/** Marks a test whose failure the repair judged to be the application's. */
|
|
560
|
+
/** The most tests one file may spend on one criterion (config maxTestsPerRule, default 4). */
|
|
561
|
+
export const maxPerRule = () => loadConfig().maxTestsPerRule ?? 4;
|
|
524
562
|
export const DEFECT_MARK = 'testforge-defect:';
|
|
563
|
+
/** A failure that is real application behaviour against a [baseline]/[observed] rule: kept failing, never a defect. */
|
|
564
|
+
export const FINDING_MARK = 'testforge-finding:';
|
|
525
565
|
/**
|
|
526
566
|
* Repair a generated test from the runner's own failure output.
|
|
527
567
|
*
|
|
@@ -550,11 +590,20 @@ export async function repairWithAIAsync(journey, testPath, current, failure, har
|
|
|
550
590
|
' for the wrong event, a response field the server does not',
|
|
551
591
|
' send, a precondition its own setup got wrong, an assertion',
|
|
552
592
|
' the journey document does not state. FIX the test.',
|
|
593
|
+
' A "strict mode violation" means the locator matched several',
|
|
594
|
+
' elements: narrow it to the one the criterion means — never',
|
|
595
|
+
' .first() on a guess.',
|
|
596
|
+
' When the app sent the test back to an EARLIER step or page,',
|
|
597
|
+
' first suspect a missing precondition the earlier steps',
|
|
598
|
+
' would have set: reach the step with kit.openStep().',
|
|
553
599
|
' APP DEFECT — the test reached the behaviour (the element or response it',
|
|
554
600
|
' checks is there) and the application contradicts a criterion',
|
|
555
601
|
' the journey document states. Change NOTHING in that test.',
|
|
556
602
|
` Put one comment line directly above its declaration:`,
|
|
557
603
|
` // ${DEFECT_MARK} <the criterion, quoted with its tag> — <what the app did instead>`,
|
|
604
|
+
` Only a [spec] criterion can be a defect. When the criterion is`,
|
|
605
|
+
` [baseline] or [observed], write the same line with`,
|
|
606
|
+
` "${FINDING_MARK}" instead: it stays failing, as a finding.`,
|
|
558
607
|
' SKIPPED — a test listed under SKIPPED skipped itself. If its own',
|
|
559
608
|
' setup looked in the wrong place (wrong field, wrong route,',
|
|
560
609
|
' wrong id), fix that so it runs. If what it needs truly does',
|
|
@@ -20,6 +20,8 @@ import { pingAi } from './ai-generator.js';
|
|
|
20
20
|
// every seed retried against a dead CLI. Record the reason, put it in the
|
|
21
21
|
// warning, and once the failure is clearly "AI unavailable" stop starting new
|
|
22
22
|
// journeys: the deferred stubs are re-attempted automatically on the next run.
|
|
23
|
+
/** A limit that resets with the month, not in hours: waiting for it is pointless. */
|
|
24
|
+
const MONTHLY = /monthly (?:spend )?limit|switch to another model/i;
|
|
23
25
|
let lastAiFailure = null;
|
|
24
26
|
let aiUnavailable = null;
|
|
25
27
|
// Process failures in a row with no success between them. The CLI does not
|
|
@@ -85,6 +87,12 @@ export async function waitForAi(config) {
|
|
|
85
87
|
console.log(`\nAI unavailable: ${aiUnavailable}\n` +
|
|
86
88
|
`Waiting for it to come back — checking every ${mins(poll)}, for up to ${mins(max)}. ` +
|
|
87
89
|
`Ctrl-C is safe: re-running later resumes from exactly here.`);
|
|
90
|
+
// A monthly cap does not reset in hours: waiting only burns the day.
|
|
91
|
+
if (aiUnavailable && MONTHLY.test(aiUnavailable)) {
|
|
92
|
+
console.log('This is a MONTHLY limit — it will not come back by waiting. Use another model for TestForge ' +
|
|
93
|
+
'(testforge.config.json "aiCommand": "claude -p --model sonnet"), or ask your admin to raise the limit.');
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
88
96
|
const started = Date.now();
|
|
89
97
|
while (Date.now() - started < max) {
|
|
90
98
|
await new Promise((r) => setTimeout(r, poll));
|