@activate-core/test-forge 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +61 -23
- package/dist/core/docs/excerpts.js +245 -0
- package/dist/core/docs/grounding.js +49 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +202 -104
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +139 -0
- package/dist/core/generate/harness.js +3 -31
- package/dist/core/generate/journey-memory.js +325 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +169 -78
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +115 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +119 -0
- package/dist/core/run/run-tests.js +23 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
|
@@ -18,29 +18,49 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
|
|
20
20
|
import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
|
|
21
|
-
import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored } from '../paths.js';
|
|
21
|
+
import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored, journeyDir } from '../paths.js';
|
|
22
22
|
import { lintGenerated } from '../gates/lint.js';
|
|
23
23
|
import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
|
|
24
24
|
import { writeCoverageManifest } from '../gates/coverage.js';
|
|
25
|
-
import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
|
|
25
|
+
import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlan, readPlanSkips, writePlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
|
|
26
26
|
import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
|
-
import { generateWithAI, generateWithAIAsync,
|
|
30
|
-
import { browserPreflight, missingPackages } from './verify.js';
|
|
31
|
-
import { verifyJourney } from './verify-journey.js';
|
|
29
|
+
import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, } from './ai-generator.js';
|
|
32
30
|
import { normalisePending } from './pending.js';
|
|
33
31
|
import { packageRoot } from '../run/run-tests.js';
|
|
34
32
|
import { groundJourney } from '../docs/grounding.js';
|
|
35
33
|
import { generateSeed } from './seed-generator.js';
|
|
36
34
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
37
35
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
38
|
-
import {
|
|
36
|
+
import { testedTags } from './suite-planner.js';
|
|
37
|
+
import { factsSlice, helperApi, journeySlice, readRules, saveMemory, testedElsewhere } from './journey-memory.js';
|
|
38
|
+
import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
39
|
+
import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
|
|
40
|
+
import { updateJourney } from './update-journey.js';
|
|
41
|
+
import { sourceText } from '../docs/excerpts.js';
|
|
42
|
+
import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
39
43
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
40
44
|
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
41
45
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
42
46
|
import { findMarkdownFiles } from '../find-files.js';
|
|
43
47
|
import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
|
|
48
|
+
/**
|
|
49
|
+
* kit.find is a shared helper the testkit re-exports beside settledClick. A
|
|
50
|
+
* testkit written before it existed gets the one line here, with no AI and no
|
|
51
|
+
* rebuild.
|
|
52
|
+
*/
|
|
53
|
+
export function exportFind(harnessPath) {
|
|
54
|
+
const src = readFileSync(harnessPath, 'utf-8');
|
|
55
|
+
if (/^\s*find\s*[:,]/m.test(src) || !/settledClick:\s*(\w+)\.settledClick,?/.test(src))
|
|
56
|
+
return;
|
|
57
|
+
writeFileSync(harnessPath, src.replace(/^(\s*)settledClick:\s*(\w+)\.settledClick,?/m, (m, pad, mod) => `${m.endsWith(',') ? m : `${m},`}\n${pad}find: ${mod}.find,`), 'utf-8');
|
|
58
|
+
}
|
|
59
|
+
/** Why a file is not valid code, as precisely as the check can say. */
|
|
60
|
+
function syntaxReason(content, ext) {
|
|
61
|
+
const where = /\.(c|m)?js$/i.test(ext) ? null : unbalancedAt(content);
|
|
62
|
+
return `not valid ${ext}${where ? `: ${where}` : ' (syntax)'} — the answer must be the whole file, code only`;
|
|
63
|
+
}
|
|
44
64
|
/** How many times to re-ask the AI when output won't parse. */
|
|
45
65
|
const MAX_GEN_ATTEMPTS = 2;
|
|
46
66
|
/** How many focused files to generate concurrently within a journey. */
|
|
@@ -66,13 +86,17 @@ export async function pool(items, limit, fn) {
|
|
|
66
86
|
return results;
|
|
67
87
|
}
|
|
68
88
|
/** Async version of generateValidated (for concurrent per-file gen). */
|
|
69
|
-
async function generateValidatedAsync(generate, ext, gate
|
|
89
|
+
async function generateValidatedAsync(generate, ext, gate,
|
|
90
|
+
/** Why an earlier draft of the same file (from a batch) was refused. */
|
|
91
|
+
firstRefusal) {
|
|
70
92
|
let content = '';
|
|
93
|
+
let refusal = firstRefusal;
|
|
71
94
|
for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
|
|
72
95
|
if (aiUnavailableReason())
|
|
73
96
|
return null; // retrying a dead CLI only burns time
|
|
74
97
|
try {
|
|
75
|
-
|
|
98
|
+
// A second try is told why the first was refused; asked blind, it made the same file again.
|
|
99
|
+
content = await generate(refusal);
|
|
76
100
|
}
|
|
77
101
|
catch (e) {
|
|
78
102
|
noteAiFailure(e);
|
|
@@ -82,7 +106,8 @@ async function generateValidatedAsync(generate, ext, gate) {
|
|
|
82
106
|
const gateMsg = gate ? gate(content) : null;
|
|
83
107
|
if (isSyntaxValid(content, ext) && !gateMsg)
|
|
84
108
|
return content;
|
|
85
|
-
|
|
109
|
+
refusal = gateMsg || syntaxReason(content, ext);
|
|
110
|
+
noteAiRejected(refusal);
|
|
86
111
|
}
|
|
87
112
|
return null;
|
|
88
113
|
}
|
|
@@ -165,6 +190,7 @@ export async function runHarness(only) {
|
|
|
165
190
|
}
|
|
166
191
|
mkdirSync(dirname(info.path), { recursive: true });
|
|
167
192
|
writeFileSync(info.path, stampHarness(kit), 'utf-8');
|
|
193
|
+
exportFind(info.path);
|
|
168
194
|
writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
|
|
169
195
|
writeHarnessConfigs(framework, areaDir, info.dir, area);
|
|
170
196
|
written.push(info.path);
|
|
@@ -179,27 +205,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
179
205
|
return [];
|
|
180
206
|
}
|
|
181
207
|
const config = loadConfig();
|
|
182
|
-
// Verify is on unless the project or the caller turns it off. verifyFiles
|
|
183
|
-
// itself reports (and skips) the cases where the app cannot be reached, so
|
|
184
|
-
// "on" never means "fail the run because nothing is running".
|
|
185
|
-
const verifyOn = opts.verify ?? config.verify ?? true;
|
|
186
|
-
const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
|
|
187
208
|
const ext = config.testExtension ?? '.test.ts';
|
|
188
209
|
const seedDir = config.seedOutput ?? 'generated-tests/seed';
|
|
189
210
|
const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
|
|
190
|
-
// A browser that cannot start fails every verify, and each failure would be
|
|
191
|
-
// sent for repair. Find out once, before any AI call is spent.
|
|
192
|
-
if (useAI && verifyOn) {
|
|
193
|
-
for (const area of Object.keys(config.testOutput ?? {})) {
|
|
194
|
-
if (resolveFramework(config, area, area) !== 'playwright')
|
|
195
|
-
continue;
|
|
196
|
-
const problem = browserPreflight(packageRoot(testDirForType(config, area)));
|
|
197
|
-
if (problem) {
|
|
198
|
-
throw new Error(`Cannot verify "${area}" tests: ${problem}.\n` +
|
|
199
|
-
'Fix that and re-run, or pass --no-verify to write tests without running them.');
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
211
|
ensureStateDirIgnored();
|
|
204
212
|
const cache = loadCache(CACHE_PATH);
|
|
205
213
|
const results = [];
|
|
@@ -247,9 +255,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
247
255
|
// makes one-by-one (`generate --journey X`) cheap: the first journey
|
|
248
256
|
// in an area builds the testkit, every later journey reuses it — no
|
|
249
257
|
// extra AI call (and no risk of that call eating the token limit).
|
|
250
|
-
//
|
|
251
|
-
|
|
258
|
+
// --force rewrites a journey's tests, not the testkit every journey in the
|
|
259
|
+
// area shares: a current testkit is kept even then. A stale one is rebuilt.
|
|
260
|
+
if (existsSync(info.path)) {
|
|
252
261
|
if (harnessIsCurrent(readFileSync(info.path, 'utf-8'))) {
|
|
262
|
+
exportFind(info.path);
|
|
253
263
|
harnessByArea.set(area, info.path);
|
|
254
264
|
return info.path;
|
|
255
265
|
}
|
|
@@ -265,6 +275,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
265
275
|
}
|
|
266
276
|
mkdirSync(dirname(info.path), { recursive: true });
|
|
267
277
|
writeFileSync(info.path, stampHarness(kit), 'utf-8');
|
|
278
|
+
exportFind(info.path);
|
|
268
279
|
writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
|
|
269
280
|
// Config + globalSetup (playwright) / config + setup (jest) so the specs run
|
|
270
281
|
// AND get their tokens minted once, plus the runner config scaffolding.
|
|
@@ -410,13 +421,21 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
410
421
|
});
|
|
411
422
|
// A freshly written file also may not hold placeholders; a repair is held to
|
|
412
423
|
// the repair guard instead (it may not add a skip), so old files stay repairable.
|
|
424
|
+
// What the app can show: its source and the doc. A fresh test's names and
|
|
425
|
+
// labels are checked against it before the test is run.
|
|
426
|
+
let knownTexts;
|
|
413
427
|
const genGate = (content, filePath) => testGate(content) ??
|
|
414
|
-
checkTestGates(content, ext, {
|
|
428
|
+
checkTestGates(content, ext, {
|
|
429
|
+
framework, forbidPlaceholders: true, filePath,
|
|
430
|
+
maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 2,
|
|
431
|
+
knownTexts: framework === 'playwright'
|
|
432
|
+
? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
433
|
+
: undefined,
|
|
434
|
+
});
|
|
415
435
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
416
436
|
// to retry (a resume pass fills those in without touching real files).
|
|
417
437
|
if (!force &&
|
|
418
438
|
cache[file] === hash &&
|
|
419
|
-
(!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
|
|
420
439
|
!hasDeferredStubs(testDir, ext)) {
|
|
421
440
|
results.push({
|
|
422
441
|
journeyName: journey.name,
|
|
@@ -430,6 +449,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
430
449
|
}
|
|
431
450
|
const warnings = [];
|
|
432
451
|
const usageStart = aiUsage();
|
|
452
|
+
// A feature written up before it is built: its tests come from the doc
|
|
453
|
+
// alone. There is no code to read for facts and no app to run them on.
|
|
454
|
+
const notBuilt = /^status:\s*not built/im.test(markdown);
|
|
455
|
+
if (notBuilt)
|
|
456
|
+
warnings.push('written before the feature exists (Status: not built yet) — run these tests once it ships');
|
|
433
457
|
const collision = collisionWarning(areaDir, journey.id, ext);
|
|
434
458
|
if (collision)
|
|
435
459
|
warnings.push(collision);
|
|
@@ -462,9 +486,70 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
462
486
|
// Capture the PREVIOUS plan's slugs first so we can clean up any
|
|
463
487
|
// files whose slug no longer exists (F6 replace-don't-duplicate).
|
|
464
488
|
const prevSlugs = useAI ? readPlanSlugs(journey) : [];
|
|
489
|
+
// The doc read ONCE: rules, controls and setup, plus the testkit's exports.
|
|
490
|
+
// From here on no AI call gets the whole doc; each gets its own slice.
|
|
491
|
+
// A doc without tagged rules keeps the old whole-doc flow.
|
|
492
|
+
const parsed = useAI ? readRules(markdown) : undefined;
|
|
493
|
+
const tested = testedTags();
|
|
494
|
+
let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
|
|
495
|
+
const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
|
|
496
|
+
// A doc that changed since its suite was written: only what changed is
|
|
497
|
+
// touched. Renames and removals are edits without AI; reworded and added
|
|
498
|
+
// rules get one small call each file; new sections become new files below.
|
|
499
|
+
const prevRecord = memory ? loadMemory(journey) : undefined;
|
|
500
|
+
const prevMemory = !force && !opts.planOnly ? prevRecord : undefined;
|
|
501
|
+
const prevPlan = prevMemory && prevMemory.hash !== hash ? readPlan(journey) : undefined;
|
|
502
|
+
let changes;
|
|
503
|
+
if (memory && prevMemory && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
|
|
504
|
+
const { memory: aligned, diff } = alignRules(prevMemory.rules, memory);
|
|
505
|
+
memory = aligned;
|
|
506
|
+
changes = { renamed: diff.renamed.map((r) => r.id), changed: diff.changed, removed: diff.removed };
|
|
507
|
+
console.log(` ${journey.id}: doc changed — ${describeDiff(diff)}`);
|
|
508
|
+
const facts = notBuilt ? undefined : (await groundJourney(journey, hash, 'src', markdown, true)) ?? undefined;
|
|
509
|
+
const up = await updateJourney({
|
|
510
|
+
journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
|
|
511
|
+
pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
|
|
512
|
+
check: (c, p) => (isSyntaxValid(c, ext) ? genGate(c, p) : syntaxReason(c, ext)),
|
|
513
|
+
});
|
|
514
|
+
for (const n of up.notes)
|
|
515
|
+
console.log(` ${n}`);
|
|
516
|
+
warnings.push(...up.notes);
|
|
517
|
+
writePlan(journey, hash, up.plan);
|
|
518
|
+
}
|
|
465
519
|
const planned = useAI
|
|
466
|
-
? loadOrPlan(journey, hash)
|
|
520
|
+
? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
|
|
467
521
|
: [{ slug: journey.id, title: journey.name, covers: '' }];
|
|
522
|
+
if (memory) {
|
|
523
|
+
const choice = loadSourceChoice();
|
|
524
|
+
saveMemory(journey, hash, memory, helpers, {
|
|
525
|
+
doc: file, reference: choice?.journeys?.[file] ?? choice?.reference,
|
|
526
|
+
plan: planned, skipped: readPlanSkips(journey), prev: prevRecord, changes,
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
if (opts.planOnly) {
|
|
530
|
+
results.push({
|
|
531
|
+
journeyName: journey.name,
|
|
532
|
+
testDir,
|
|
533
|
+
testFiles: [],
|
|
534
|
+
seedPath,
|
|
535
|
+
status: 'skipped',
|
|
536
|
+
seedStatus: 'none',
|
|
537
|
+
warnings: [`plan only: ${planned.length} file(s) planned, nothing written — see ${join(journeyDir(journey), 'plan-check.md')}`],
|
|
538
|
+
});
|
|
539
|
+
continue;
|
|
540
|
+
}
|
|
541
|
+
// What one file is given: its rules with their context, the facts for
|
|
542
|
+
// its controls, the testkit list. A plan made before rules existed
|
|
543
|
+
// gives every file all the rules that get tests.
|
|
544
|
+
const allTested = memory ? memory.rules.filter((r) => r.tag && tested.includes(r.tag)).map((r) => r.id) : [];
|
|
545
|
+
const fileMemory = async (pf) => {
|
|
546
|
+
const facts = await grounding();
|
|
547
|
+
if (!memory)
|
|
548
|
+
return { facts };
|
|
549
|
+
const ids = pf.rules?.length ? pf.rules : allTested;
|
|
550
|
+
const slice = journeySlice(memory, ids);
|
|
551
|
+
return { mem: { journey: slice.text, helpers, rules: ids }, facts: factsSlice(facts, slice.controls) };
|
|
552
|
+
};
|
|
468
553
|
const plannedSlugs = planned.map((p) => p.slug);
|
|
469
554
|
// Read the journey's real code ONCE, before any file is written, and hand
|
|
470
555
|
// the same facts to every file. Without this each file re-opened the same
|
|
@@ -475,7 +560,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
475
560
|
// Only a file that has to be written or repaired needs it, so it is read
|
|
476
561
|
// on first use: re-checking an existing suite costs no grounding at all.
|
|
477
562
|
let groundingP;
|
|
478
|
-
const grounding = () => (groundingP ??= useAI
|
|
563
|
+
const grounding = () => (groundingP ??= useAI && !notBuilt
|
|
479
564
|
? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
|
|
480
565
|
: Promise.resolve(undefined));
|
|
481
566
|
const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
|
|
@@ -488,6 +573,50 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
488
573
|
// parallel agentic AI runs starve each other and time out, so a flaky
|
|
489
574
|
// environment can lower `aiConcurrency` (e.g. 2) for reliability.
|
|
490
575
|
const concurrency = Math.max(1, config.aiConcurrency ?? GEN_CONCURRENCY);
|
|
576
|
+
// Files to write, a few to one AI call: each call carries the shared rules
|
|
577
|
+
// once. A drafted file that fails its checks is written again on its own,
|
|
578
|
+
// told why; a file the batch left out is written on its own.
|
|
579
|
+
const pathOf = (pf) => join(testDir, `${flat ? `${journey.id}-${pf.slug}` : pf.slug}${ext}`);
|
|
580
|
+
const keeps = (p) => {
|
|
581
|
+
if ((force && !redone.has(i)) || !existsSync(p))
|
|
582
|
+
return false;
|
|
583
|
+
try {
|
|
584
|
+
const c = readFileSync(p, 'utf-8');
|
|
585
|
+
return isSyntaxValid(c, ext) && !isDeferredStub(c);
|
|
586
|
+
}
|
|
587
|
+
catch {
|
|
588
|
+
return false;
|
|
589
|
+
}
|
|
590
|
+
};
|
|
591
|
+
const drafts = new Map();
|
|
592
|
+
const perCall = Math.max(1, config.filesPerCall ?? 3);
|
|
593
|
+
if (useAI && memory && perCall > 1) {
|
|
594
|
+
const toWrite = planned.filter((pf) => !keeps(pathOf(pf)));
|
|
595
|
+
const batches = [];
|
|
596
|
+
for (let k = 0; k < toWrite.length; k += perCall)
|
|
597
|
+
batches.push(toWrite.slice(k, k + perCall));
|
|
598
|
+
const writeBatch = async (batch) => {
|
|
599
|
+
const files = await Promise.all(batch.map(async (pf) => {
|
|
600
|
+
const { mem, facts } = await fileMemory(pf);
|
|
601
|
+
return { path: pathOf(pf), prompt: buildPrompt(journey, '', pathOf(pf), { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem) };
|
|
602
|
+
}));
|
|
603
|
+
try {
|
|
604
|
+
const got = await writeFilesWithAI(files);
|
|
605
|
+
noteAiSuccess();
|
|
606
|
+
for (const [p, c] of got)
|
|
607
|
+
drafts.set(p, c);
|
|
608
|
+
}
|
|
609
|
+
catch (e) {
|
|
610
|
+
noteAiFailure(e);
|
|
611
|
+
}
|
|
612
|
+
};
|
|
613
|
+
// The first call alone writes the shared prompt to the cache; the rest
|
|
614
|
+
// then read it instead of each writing its own copy at the same moment.
|
|
615
|
+
const multi = batches.filter((b) => b.length > 1);
|
|
616
|
+
if (multi.length)
|
|
617
|
+
await writeBatch(multi[0]);
|
|
618
|
+
await pool(multi.slice(1), concurrency, writeBatch);
|
|
619
|
+
}
|
|
491
620
|
const generated = await pool(planned, concurrency, async (pf) => {
|
|
492
621
|
// Flat (jest): prefix the file with the journey so many journeys can
|
|
493
622
|
// share one dir without clashing. Folder (playwright): the journey
|
|
@@ -498,7 +627,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
498
627
|
// — don't spend tokens regenerating it. So a journey interrupted
|
|
499
628
|
// midway resumes from exactly the files it hadn't written yet.
|
|
500
629
|
// (--force ignores this and regenerates.)
|
|
501
|
-
|
|
630
|
+
// A redo after stubs fills in only the stubs: --force already rewrote the rest once this run.
|
|
631
|
+
if ((!force || redone.has(i)) && existsSync(filePath)) {
|
|
502
632
|
try {
|
|
503
633
|
const existing = readFileSync(filePath, 'utf-8');
|
|
504
634
|
// Keep a real, parsing file — but a DEFERRED stub is not real
|
|
@@ -519,8 +649,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
519
649
|
let content;
|
|
520
650
|
let status;
|
|
521
651
|
if (useAI) {
|
|
522
|
-
const
|
|
523
|
-
const
|
|
652
|
+
const draft = drafts.get(filePath);
|
|
653
|
+
const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
|
|
654
|
+
const { mem, facts } = await fileMemory(pf);
|
|
655
|
+
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
|
|
524
656
|
if (ai !== null) {
|
|
525
657
|
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
526
658
|
status = 'ai';
|
|
@@ -631,54 +763,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
631
763
|
warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
|
|
632
764
|
lint.remaining.slice(0, 5).join('; '));
|
|
633
765
|
}
|
|
634
|
-
|
|
635
|
-
// done — file by file, then the whole journey together, then an independent
|
|
636
|
-
// review of every claimed defect (see verify-journey.ts). Anything that
|
|
637
|
-
// could not run, or still fails unexplained, leaves the journey UNVERIFIED
|
|
638
|
-
// and uncached, so the next run checks it again.
|
|
639
|
-
let verified;
|
|
640
|
-
let reviewed = [];
|
|
641
|
-
const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
|
|
642
|
-
const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
|
|
643
|
-
if (useAI && verifyOn && targets.length && missing.length) {
|
|
644
|
-
// The suite cannot even start; running it would only feed repair a
|
|
645
|
-
// failure no test edit can fix.
|
|
646
|
-
verified = false;
|
|
647
|
-
warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
|
|
648
|
-
}
|
|
649
|
-
else if (useAI && verifyOn && targets.length) {
|
|
650
|
-
const result = await verifyJourney({
|
|
651
|
-
config,
|
|
652
|
-
journey,
|
|
653
|
-
targets,
|
|
654
|
-
attempts: verifyAttempts,
|
|
655
|
-
repair: async (target, current, failure) => {
|
|
656
|
-
const facts = await grounding();
|
|
657
|
-
const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
|
|
658
|
-
if (out === null)
|
|
659
|
-
return null;
|
|
660
|
-
const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
|
|
661
|
-
return framework === 'playwright' ? normalisePending(stamped) : stamped;
|
|
662
|
-
},
|
|
663
|
-
});
|
|
664
|
-
verified = result.verified;
|
|
665
|
-
reviewed = result.reviewed;
|
|
666
|
-
warnings.push(...result.warnings);
|
|
667
|
-
}
|
|
668
|
-
const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
|
|
766
|
+
const degraded = useAI && (status === 'basic' || seedStatus === 'failed');
|
|
669
767
|
if (!degraded) {
|
|
670
768
|
cache[file] = hash;
|
|
671
|
-
// Remembered apart, so a journey cached before it was ever run is
|
|
672
|
-
// re-checked (not regenerated) once verification is on.
|
|
673
|
-
if (verified)
|
|
674
|
-
cache[`verified:${file}`] = hash;
|
|
675
769
|
}
|
|
676
770
|
else {
|
|
677
771
|
// Actively UN-mark a degraded journey (even under --force, which may
|
|
678
772
|
// be re-processing a stale "done" from an earlier buggy run), so the
|
|
679
773
|
// next `generate` re-does it instead of skipping a thin fallback.
|
|
680
774
|
delete cache[file];
|
|
681
|
-
delete cache[`verified:${file}`];
|
|
682
775
|
}
|
|
683
776
|
saveCache(CACHE_PATH, cache);
|
|
684
777
|
resultRow.set(file, results.length);
|
|
@@ -696,9 +789,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
696
789
|
seedPath,
|
|
697
790
|
status,
|
|
698
791
|
seedStatus,
|
|
699
|
-
verified,
|
|
700
792
|
warnings: warnings.length ? warnings : undefined,
|
|
701
|
-
reviewed: reviewed.length ? reviewed : undefined,
|
|
702
793
|
ai,
|
|
703
794
|
});
|
|
704
795
|
}
|
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { readFileSync, existsSync, readdirSync } from 'node:fs';
|
|
18
18
|
import { join } from 'node:path';
|
|
19
|
-
import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
|
|
20
|
-
import {
|
|
19
|
+
import { loadConfig, splitCommand, withNoTools, withReadOnlyTools } from '../project/config.js';
|
|
20
|
+
import { ruleIndex } from './journey-memory.js';
|
|
21
|
+
import { MEMORY_DIR } from '../paths.js';
|
|
21
22
|
import { runAiSync } from './ai-usage.js';
|
|
22
23
|
const MAP_PATH = 'testforge.md';
|
|
23
24
|
/**
|
|
@@ -30,13 +31,12 @@ const MAP_PATH = 'testforge.md';
|
|
|
30
31
|
export function siblingJourneys(journey, docsDir = 'docs') {
|
|
31
32
|
const lines = [];
|
|
32
33
|
const planned = new Set();
|
|
33
|
-
const dir = join(
|
|
34
|
-
for (const
|
|
35
|
-
|
|
36
|
-
if (!f.endsWith('.json') || id === journey.id)
|
|
34
|
+
const dir = join(MEMORY_DIR, journey.area);
|
|
35
|
+
for (const id of existsSync(dir) ? readdirSync(dir) : []) {
|
|
36
|
+
if (id === journey.id || !existsSync(join(dir, id, 'plan.json')))
|
|
37
37
|
continue;
|
|
38
38
|
try {
|
|
39
|
-
const plan = JSON.parse(readFileSync(join(dir,
|
|
39
|
+
const plan = JSON.parse(readFileSync(join(dir, id, 'plan.json'), 'utf-8'));
|
|
40
40
|
lines.push(`- ${id} (planned):`, ...(plan.files ?? []).map((x) => ` - ${x.title ?? ''}: ${x.covers ?? ''}`));
|
|
41
41
|
planned.add(id);
|
|
42
42
|
}
|
|
@@ -56,7 +56,7 @@ export function siblingJourneys(journey, docsDir = 'docs') {
|
|
|
56
56
|
return lines.join('\n');
|
|
57
57
|
}
|
|
58
58
|
/** Build the planning prompt for one journey. */
|
|
59
|
-
export function buildPlanPrompt(journey, map, siblings = '') {
|
|
59
|
+
export function buildPlanPrompt(journey, map, siblings = '', memory) {
|
|
60
60
|
return [
|
|
61
61
|
`You are a senior test architect. Plan how to split ONE user journey`,
|
|
62
62
|
`into a set of FOCUSED test files — one concern per file — the way a`,
|
|
@@ -64,11 +64,19 @@ export function buildPlanPrompt(journey, map, siblings = '') {
|
|
|
64
64
|
`success, failures, discount, 3D-Secure, zero-amount, price summary,`,
|
|
65
65
|
`and so on). Do NOT plan a single monolithic file.`,
|
|
66
66
|
``,
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
67
|
+
...(memory
|
|
68
|
+
? [
|
|
69
|
+
`You get the journey as its list of RULES (one line each, with an id)`,
|
|
70
|
+
`and its controls, not the whole document, and you have no tools: plan`,
|
|
71
|
+
`from this list. Every file names the rule ids it tests.`,
|
|
72
|
+
]
|
|
73
|
+
: [
|
|
74
|
+
`You are inside the project's repository with read-only tools (Read,`,
|
|
75
|
+
`Grep, Glob, LS). READ the real source for this journey — routes,`,
|
|
76
|
+
`controllers, services, models — and combine it with the journey doc`,
|
|
77
|
+
`to decide the split. Every file must map to a real behaviour that`,
|
|
78
|
+
`exists in the code or is explicitly described in the doc.`,
|
|
79
|
+
]),
|
|
72
80
|
``,
|
|
73
81
|
`Rules:`,
|
|
74
82
|
`- Let the doc's sections and the real endpoints drive the split.`,
|
|
@@ -103,7 +111,18 @@ export function buildPlanPrompt(journey, map, siblings = '') {
|
|
|
103
111
|
: []),
|
|
104
112
|
`Output ONLY a JSON array (no prose, no markdown fences) of objects:`,
|
|
105
113
|
` [{ "slug": "<short-kebab-name>", "title": "<describe title>",`,
|
|
106
|
-
` "covers": "<one or two sentences: exactly what this file tests>" }]`,
|
|
114
|
+
` "covers": "<one or two sentences: exactly what this file tests>"${memory ? ',\n "rules": ["R1", "R4"]' : ''} }]`,
|
|
115
|
+
...(memory
|
|
116
|
+
? [
|
|
117
|
+
`Every rule id below goes in exactly ONE file's "rules". A rule that`,
|
|
118
|
+
`another journey owns, or that cannot be checked from this journey, is`,
|
|
119
|
+
`left out and listed instead as { "skip": "R7", "why": "<short reason>" }`,
|
|
120
|
+
`in the same array. A rule that is neither planned nor skipped is`,
|
|
121
|
+
`reported as missing. Every [spec] rule and every rule marked "(must"`,
|
|
122
|
+
`(security, money and data, concurrency, error boundaries) is planned`,
|
|
123
|
+
`unless another journey owns it.`,
|
|
124
|
+
]
|
|
125
|
+
: []),
|
|
107
126
|
`slug: lowercase kebab-case, no path, no extension — and SHORT: name the`,
|
|
108
127
|
` one concern in 2–4 words (4 max), the way a human names a file. Do NOT`,
|
|
109
128
|
` restate the journey name and do NOT pile on qualifiers.`,
|
|
@@ -113,13 +132,25 @@ export function buildPlanPrompt(journey, map, siblings = '') {
|
|
|
113
132
|
`title: human-readable and specific, e.g. "Payment — Amount validation".`,
|
|
114
133
|
`The very first character of your response must be "[".`,
|
|
115
134
|
``,
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
135
|
+
...(memory
|
|
136
|
+
? [
|
|
137
|
+
`--- JOURNEY: ${journey.name} ---`,
|
|
138
|
+
memory.header,
|
|
139
|
+
``,
|
|
140
|
+
`CONTROLS:`,
|
|
141
|
+
...Object.entries(memory.controls).map(([id, c]) => `${id}: ${c}`),
|
|
142
|
+
``,
|
|
143
|
+
`RULES TO PLAN:`,
|
|
144
|
+
ruleIndex(memory, testedTags()),
|
|
145
|
+
]
|
|
146
|
+
: [`--- PROJECT MAP (data layer) ---`, map, ``, `--- JOURNEY ---`, journey.rawMarkdown]),
|
|
121
147
|
].join('\n');
|
|
122
148
|
}
|
|
149
|
+
/** Which rule tags get tests of their own; the rest only explain the app. */
|
|
150
|
+
export function testedTags() {
|
|
151
|
+
const t = loadConfig().testTags;
|
|
152
|
+
return Array.isArray(t) && t.length ? t : ['spec', 'baseline'];
|
|
153
|
+
}
|
|
123
154
|
/**
|
|
124
155
|
* Pull a JSON array out of the AI's reply, tolerating stray prose or
|
|
125
156
|
* code fences around it. Returns [] if nothing parseable is found.
|
|
@@ -154,10 +185,11 @@ function safeSlug(raw) {
|
|
|
154
185
|
* produces nothing usable, so generation never silently yields zero
|
|
155
186
|
* files.
|
|
156
187
|
*/
|
|
157
|
-
export function planSuite(journey) {
|
|
158
|
-
const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8')
|
|
159
|
-
const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey));
|
|
160
|
-
const
|
|
188
|
+
export function planSuite(journey, memory, skipped = []) {
|
|
189
|
+
const map = memory || !existsSync(MAP_PATH) ? '' : readFileSync(MAP_PATH, 'utf-8');
|
|
190
|
+
const prompt = buildPlanPrompt(journey, map, siblingJourneys(journey), memory);
|
|
191
|
+
const cmd = splitCommand(loadConfig().aiCommand);
|
|
192
|
+
const { program, args } = memory ? withNoTools(cmd) : withReadOnlyTools(cmd);
|
|
161
193
|
let raw = '';
|
|
162
194
|
try {
|
|
163
195
|
raw = runAiSync(program, args, {
|
|
@@ -175,6 +207,10 @@ export function planSuite(journey) {
|
|
|
175
207
|
if (!entry || typeof entry !== 'object')
|
|
176
208
|
continue;
|
|
177
209
|
const e = entry;
|
|
210
|
+
if (typeof e.skip === 'string') {
|
|
211
|
+
skipped.push({ rule: e.skip, why: String(e.why ?? '').trim() });
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
178
214
|
const slug = safeSlug(e.slug ?? e.title);
|
|
179
215
|
if (!slug || seen.has(slug))
|
|
180
216
|
continue;
|
|
@@ -183,6 +219,7 @@ export function planSuite(journey) {
|
|
|
183
219
|
slug,
|
|
184
220
|
title: String(e.title ?? journey.name).trim() || journey.name,
|
|
185
221
|
covers: String(e.covers ?? '').trim(),
|
|
222
|
+
...(memory && Array.isArray(e.rules) ? { rules: e.rules.map(String) } : {}),
|
|
186
223
|
});
|
|
187
224
|
}
|
|
188
225
|
if (files.length === 0) {
|
|
@@ -196,3 +233,71 @@ export function planSuite(journey) {
|
|
|
196
233
|
}
|
|
197
234
|
return files;
|
|
198
235
|
}
|
|
236
|
+
/** Sections larger than this are split by their sub-sections, then in chunks. */
|
|
237
|
+
const MAX_RULES_PER_FILE = 10;
|
|
238
|
+
/**
|
|
239
|
+
* The plan without the AI: the doc is already organised one concern per
|
|
240
|
+
* numbered section, so each section becomes one file and gets that section's
|
|
241
|
+
* rules. A section with more rules than one file should hold is split by its
|
|
242
|
+
* sub-sections, then in chunks. Rules another journey already tests come in
|
|
243
|
+
* as skips. Costs nothing.
|
|
244
|
+
*/
|
|
245
|
+
export function planBySections(journey, memory, skipped = []) {
|
|
246
|
+
const tags = testedTags();
|
|
247
|
+
const skip = new Set(skipped.map((x) => x.rule));
|
|
248
|
+
const rules = memory.rules.filter((r) => r.tag && tags.includes(r.tag) && !skip.has(r.id));
|
|
249
|
+
const byKey = (key, rs) => {
|
|
250
|
+
const m = new Map();
|
|
251
|
+
for (const r of rs)
|
|
252
|
+
m.set(key(r), [...(m.get(key(r)) ?? []), r]);
|
|
253
|
+
return m;
|
|
254
|
+
};
|
|
255
|
+
const files = [];
|
|
256
|
+
const used = new Set();
|
|
257
|
+
const STOP = new Set(['a', 'an', 'the', 'on', 'of', 'and', 'or', 'for', 'to', 'in', 'with', 'from', 'at', 'by', 'is', 'its']);
|
|
258
|
+
const slugFor = (title) => {
|
|
259
|
+
const part = / part (\d+)$/.exec(title)?.[1];
|
|
260
|
+
const words = title.replace(/^§[\d.]+\s*/, '').replace(/ part \d+$/, '').toLowerCase().split(/[^a-z0-9]+/).filter((w) => w && !STOP.has(w));
|
|
261
|
+
const base = part ? `${words.slice(0, 3).join('-')}-${part}` : safeSlug(words.join('-')) || 'rules';
|
|
262
|
+
let slug = base;
|
|
263
|
+
for (let n = 2; used.has(slug); n++)
|
|
264
|
+
slug = `${base.split('-').slice(0, 3).join('-')}-${n}`;
|
|
265
|
+
used.add(slug);
|
|
266
|
+
return slug;
|
|
267
|
+
};
|
|
268
|
+
const push = (title, rs) => {
|
|
269
|
+
const name = title.replace(/^§[\d.]+\s*/, '');
|
|
270
|
+
files.push({ slug: slugFor(title), title: `${journey.name} — ${name}`, covers: `The ${rs.length} rule(s) of ${title}.`, rules: rs.map((r) => r.id) });
|
|
271
|
+
};
|
|
272
|
+
for (const [group, rs] of byKey((r) => r.group || r.section, rules)) {
|
|
273
|
+
if (rs.length <= MAX_RULES_PER_FILE) {
|
|
274
|
+
push(group, rs);
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
// Too many for one file: its sub-sections, small ones kept together, big ones in parts.
|
|
278
|
+
let titles = [];
|
|
279
|
+
let acc = [];
|
|
280
|
+
const flush = () => {
|
|
281
|
+
if (acc.length)
|
|
282
|
+
push(titles.join(' & '), acc);
|
|
283
|
+
titles = [];
|
|
284
|
+
acc = [];
|
|
285
|
+
};
|
|
286
|
+
for (const [section, part] of byKey((r) => r.section, rs)) {
|
|
287
|
+
if (part.length > MAX_RULES_PER_FILE) {
|
|
288
|
+
flush();
|
|
289
|
+
const parts = Math.ceil(part.length / MAX_RULES_PER_FILE);
|
|
290
|
+
const size = Math.ceil(part.length / parts);
|
|
291
|
+
for (let i = 0; i < part.length; i += size)
|
|
292
|
+
push(`${section} part ${i / size + 1}`, part.slice(i, i + size));
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
if (acc.length + part.length > MAX_RULES_PER_FILE)
|
|
296
|
+
flush();
|
|
297
|
+
titles.push(section.replace(/^§[\d.]+\s*/, ''));
|
|
298
|
+
acc.push(...part);
|
|
299
|
+
}
|
|
300
|
+
flush();
|
|
301
|
+
}
|
|
302
|
+
return files;
|
|
303
|
+
}
|