@activate-core/test-forge 1.3.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -28
- package/dist/cli/index.js +39 -33
- package/dist/core/docs/cache.js +2 -1
- package/dist/core/docs/discover.js +49 -2
- package/dist/core/docs/doc-guidance.js +70 -29
- package/dist/core/docs/excerpts.js +294 -0
- package/dist/core/docs/grounding.js +55 -22
- package/dist/core/docs/reference-text.js +78 -0
- package/dist/core/docs/reference.js +60 -3
- package/dist/core/docs/review.js +38 -0
- package/dist/core/docs/source-of-truth.js +1 -1
- package/dist/core/docs/tickets.js +3 -3
- package/dist/core/gates/coverage.js +1 -16
- package/dist/core/gates/test-quality.js +295 -0
- package/dist/core/gates/validate.js +171 -7
- package/dist/core/generate/ai-generator.js +267 -117
- package/dist/core/generate/ai-usage.js +116 -5
- package/dist/core/generate/bind.js +91 -0
- package/dist/core/generate/complete-file.js +77 -0
- package/dist/core/generate/defect-review.js +59 -9
- package/dist/core/generate/doc-diff.js +140 -0
- package/dist/core/generate/harness.js +18 -34
- package/dist/core/generate/journey-memory.js +394 -0
- package/dist/core/generate/plan-cache.js +114 -7
- package/dist/core/generate/review-journey.js +3 -3
- package/dist/core/generate/run-generate.js +240 -82
- package/dist/core/generate/suite-planner.js +128 -23
- package/dist/core/generate/test-blocks.js +84 -0
- package/dist/core/generate/update-journey.js +118 -0
- package/dist/core/paths.js +90 -13
- package/dist/core/project/config.js +25 -2
- package/dist/core/project/env-template.js +2 -1
- package/dist/core/run/ci.js +1 -1
- package/dist/core/run/clean.js +1 -1
- package/dist/core/run/readiness.js +146 -0
- package/dist/core/run/run-tests.js +26 -2
- package/package.json +1 -1
- package/reporters/_shared.cjs +2 -2
- package/reporters/run-data.cjs +1 -1
- package/reporters/ui-helpers.cjs +78 -1
- package/dist/core/generate/verify-journey.js +0 -215
|
@@ -18,29 +18,52 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
|
|
20
20
|
import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
|
|
21
|
-
import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored } from '../paths.js';
|
|
21
|
+
import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored, journeyDir } from '../paths.js';
|
|
22
22
|
import { lintGenerated } from '../gates/lint.js';
|
|
23
23
|
import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
|
|
24
24
|
import { writeCoverageManifest } from '../gates/coverage.js';
|
|
25
|
-
import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
|
|
25
|
+
import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlan, readPlanSkips, writePlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
|
|
26
26
|
import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
|
-
import { generateWithAI, generateWithAIAsync,
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
29
|
+
import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
|
+
import { completeFile, fileIsComplete } from './complete-file.js';
|
|
31
|
+
import { harnessSteps } from '../gates/test-quality.js';
|
|
32
32
|
import { normalisePending } from './pending.js';
|
|
33
33
|
import { packageRoot } from '../run/run-tests.js';
|
|
34
|
+
import { pendingReason, withPendingStamp } from '../run/readiness.js';
|
|
34
35
|
import { groundJourney } from '../docs/grounding.js';
|
|
35
36
|
import { generateSeed } from './seed-generator.js';
|
|
36
37
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
37
38
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
38
|
-
import {
|
|
39
|
+
import { testedTags } from './suite-planner.js';
|
|
40
|
+
import { factsSlice, helperApi, journeySlice, readRules, recordWritten, saveMemory, testedElsewhere } from './journey-memory.js';
|
|
41
|
+
import { loadSourceChoice } from '../docs/source-of-truth.js';
|
|
42
|
+
import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
|
|
43
|
+
import { updateJourney } from './update-journey.js';
|
|
44
|
+
import { sourceText } from '../docs/excerpts.js';
|
|
45
|
+
import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
39
46
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
40
47
|
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
41
48
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
42
49
|
import { findMarkdownFiles } from '../find-files.js';
|
|
43
50
|
import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
|
|
51
|
+
/**
|
|
52
|
+
* kit.find is a shared helper the testkit re-exports beside settledClick. A
|
|
53
|
+
* testkit written before it existed gets the one line here, with no AI and no
|
|
54
|
+
* rebuild.
|
|
55
|
+
*/
|
|
56
|
+
export function exportFind(harnessPath) {
|
|
57
|
+
const src = readFileSync(harnessPath, 'utf-8');
|
|
58
|
+
if (/^\s*find\s*[:,]/m.test(src) || !/settledClick:\s*(\w+)\.settledClick,?/.test(src))
|
|
59
|
+
return;
|
|
60
|
+
writeFileSync(harnessPath, src.replace(/^(\s*)settledClick:\s*(\w+)\.settledClick,?/m, (m, pad, mod) => `${m.endsWith(',') ? m : `${m},`}\n${pad}find: ${mod}.find,`), 'utf-8');
|
|
61
|
+
}
|
|
62
|
+
/** Why a file is not valid code, as precisely as the check can say. */
|
|
63
|
+
function syntaxReason(content, ext) {
|
|
64
|
+
const where = /\.(c|m)?js$/i.test(ext) ? null : unbalancedAt(content);
|
|
65
|
+
return `not valid ${ext}${where ? `: ${where}` : ' (syntax)'} — the answer must be the whole file, code only`;
|
|
66
|
+
}
|
|
44
67
|
/** How many times to re-ask the AI when output won't parse. */
|
|
45
68
|
const MAX_GEN_ATTEMPTS = 2;
|
|
46
69
|
/** How many focused files to generate concurrently within a journey. */
|
|
@@ -66,13 +89,17 @@ export async function pool(items, limit, fn) {
|
|
|
66
89
|
return results;
|
|
67
90
|
}
|
|
68
91
|
/** Async version of generateValidated (for concurrent per-file gen). */
|
|
69
|
-
async function generateValidatedAsync(generate, ext, gate
|
|
92
|
+
async function generateValidatedAsync(generate, ext, gate,
|
|
93
|
+
/** Why an earlier draft of the same file (from a batch) was refused. */
|
|
94
|
+
firstRefusal) {
|
|
70
95
|
let content = '';
|
|
96
|
+
let refusal = firstRefusal;
|
|
71
97
|
for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
|
|
72
98
|
if (aiUnavailableReason())
|
|
73
99
|
return null; // retrying a dead CLI only burns time
|
|
74
100
|
try {
|
|
75
|
-
|
|
101
|
+
// A second try is told why the first was refused; asked blind, it made the same file again.
|
|
102
|
+
content = await generate(refusal);
|
|
76
103
|
}
|
|
77
104
|
catch (e) {
|
|
78
105
|
noteAiFailure(e);
|
|
@@ -82,7 +109,8 @@ async function generateValidatedAsync(generate, ext, gate) {
|
|
|
82
109
|
const gateMsg = gate ? gate(content) : null;
|
|
83
110
|
if (isSyntaxValid(content, ext) && !gateMsg)
|
|
84
111
|
return content;
|
|
85
|
-
|
|
112
|
+
refusal = gateMsg || syntaxReason(content, ext);
|
|
113
|
+
noteAiRejected(refusal);
|
|
86
114
|
}
|
|
87
115
|
return null;
|
|
88
116
|
}
|
|
@@ -165,6 +193,7 @@ export async function runHarness(only) {
|
|
|
165
193
|
}
|
|
166
194
|
mkdirSync(dirname(info.path), { recursive: true });
|
|
167
195
|
writeFileSync(info.path, stampHarness(kit), 'utf-8');
|
|
196
|
+
exportFind(info.path);
|
|
168
197
|
writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
|
|
169
198
|
writeHarnessConfigs(framework, areaDir, info.dir, area);
|
|
170
199
|
written.push(info.path);
|
|
@@ -179,27 +208,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
179
208
|
return [];
|
|
180
209
|
}
|
|
181
210
|
const config = loadConfig();
|
|
182
|
-
// Verify is on unless the project or the caller turns it off. verifyFiles
|
|
183
|
-
// itself reports (and skips) the cases where the app cannot be reached, so
|
|
184
|
-
// "on" never means "fail the run because nothing is running".
|
|
185
|
-
const verifyOn = opts.verify ?? config.verify ?? true;
|
|
186
|
-
const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
|
|
187
211
|
const ext = config.testExtension ?? '.test.ts';
|
|
188
212
|
const seedDir = config.seedOutput ?? 'generated-tests/seed';
|
|
189
213
|
const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
|
|
190
|
-
// A browser that cannot start fails every verify, and each failure would be
|
|
191
|
-
// sent for repair. Find out once, before any AI call is spent.
|
|
192
|
-
if (useAI && verifyOn) {
|
|
193
|
-
for (const area of Object.keys(config.testOutput ?? {})) {
|
|
194
|
-
if (resolveFramework(config, area, area) !== 'playwright')
|
|
195
|
-
continue;
|
|
196
|
-
const problem = browserPreflight(packageRoot(testDirForType(config, area)));
|
|
197
|
-
if (problem) {
|
|
198
|
-
throw new Error(`Cannot verify "${area}" tests: ${problem}.\n` +
|
|
199
|
-
'Fix that and re-run, or pass --no-verify to write tests without running them.');
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
214
|
ensureStateDirIgnored();
|
|
204
215
|
const cache = loadCache(CACHE_PATH);
|
|
205
216
|
const results = [];
|
|
@@ -247,9 +258,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
247
258
|
// makes one-by-one (`generate --journey X`) cheap: the first journey
|
|
248
259
|
// in an area builds the testkit, every later journey reuses it — no
|
|
249
260
|
// extra AI call (and no risk of that call eating the token limit).
|
|
250
|
-
//
|
|
251
|
-
|
|
261
|
+
// --force rewrites a journey's tests, not the testkit every journey in the
|
|
262
|
+
// area shares: a current testkit is kept even then. A stale one is rebuilt.
|
|
263
|
+
if (existsSync(info.path)) {
|
|
252
264
|
if (harnessIsCurrent(readFileSync(info.path, 'utf-8'))) {
|
|
265
|
+
exportFind(info.path);
|
|
253
266
|
harnessByArea.set(area, info.path);
|
|
254
267
|
return info.path;
|
|
255
268
|
}
|
|
@@ -265,6 +278,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
265
278
|
}
|
|
266
279
|
mkdirSync(dirname(info.path), { recursive: true });
|
|
267
280
|
writeFileSync(info.path, stampHarness(kit), 'utf-8');
|
|
281
|
+
exportFind(info.path);
|
|
268
282
|
writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
|
|
269
283
|
// Config + globalSetup (playwright) / config + setup (jest) so the specs run
|
|
270
284
|
// AND get their tokens minted once, plus the runner config scaffolding.
|
|
@@ -410,14 +424,39 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
410
424
|
});
|
|
411
425
|
// A freshly written file also may not hold placeholders; a repair is held to
|
|
412
426
|
// the repair guard instead (it may not add a skip), so old files stay repairable.
|
|
427
|
+
// What the app can show: its source and the doc. A fresh test's names and
|
|
428
|
+
// labels are checked against it before the test is run.
|
|
429
|
+
let knownTexts;
|
|
430
|
+
const appTexts = () => framework === 'playwright'
|
|
431
|
+
? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
432
|
+
: undefined;
|
|
413
433
|
const genGate = (content, filePath) => testGate(content) ??
|
|
414
|
-
checkTestGates(content, ext, {
|
|
434
|
+
checkTestGates(content, ext, {
|
|
435
|
+
framework, forbidPlaceholders: true, filePath,
|
|
436
|
+
maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 2,
|
|
437
|
+
knownTexts: appTexts(),
|
|
438
|
+
});
|
|
415
439
|
// Skip if unchanged — but NOT if the journey still has deferred stubs
|
|
416
440
|
// to retry (a resume pass fills those in without touching real files).
|
|
441
|
+
// A suite whose tests leave rules out or cannot fail is not done either:
|
|
442
|
+
// it goes on, and only those tests are written again.
|
|
443
|
+
const suiteComplete = () => {
|
|
444
|
+
const plan = readPlan(journey);
|
|
445
|
+
if (!plan?.some((f) => f.rules?.length))
|
|
446
|
+
return true;
|
|
447
|
+
const kit = harnessInfo(areaDir, framework, ext).path;
|
|
448
|
+
const steps = existsSync(kit) ? harnessSteps(readFileSync(kit, 'utf-8')) : [];
|
|
449
|
+
return plan.every((f) => {
|
|
450
|
+
const p = join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`);
|
|
451
|
+
if (!f.rules?.length || !existsSync(p))
|
|
452
|
+
return true;
|
|
453
|
+
return fileIsComplete(readFileSync(p, 'utf-8'), f.rules, steps, appTexts());
|
|
454
|
+
});
|
|
455
|
+
};
|
|
417
456
|
if (!force &&
|
|
418
457
|
cache[file] === hash &&
|
|
419
|
-
(
|
|
420
|
-
|
|
458
|
+
!hasDeferredStubs(testDir, ext) &&
|
|
459
|
+
suiteComplete()) {
|
|
421
460
|
results.push({
|
|
422
461
|
journeyName: journey.name,
|
|
423
462
|
testDir,
|
|
@@ -430,6 +469,14 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
430
469
|
}
|
|
431
470
|
const warnings = [];
|
|
432
471
|
const usageStart = aiUsage();
|
|
472
|
+
// A feature written up before it is built: its tests come from the doc
|
|
473
|
+
// alone. There is no code to read for facts and no app to run them on.
|
|
474
|
+
const notBuilt = /^status:\s*not built/im.test(markdown);
|
|
475
|
+
// Not live yet (not built, or behind a flag): every file is stamped, and
|
|
476
|
+
// `testforge run` holds it back until the doc's Status line goes.
|
|
477
|
+
const pending = pendingReason(markdown);
|
|
478
|
+
if (pending)
|
|
479
|
+
warnings.push(`${pending}: its files are held back by \`testforge run\` until the doc's Status line is removed`);
|
|
433
480
|
const collision = collisionWarning(areaDir, journey.id, ext);
|
|
434
481
|
if (collision)
|
|
435
482
|
warnings.push(collision);
|
|
@@ -462,9 +509,97 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
462
509
|
// Capture the PREVIOUS plan's slugs first so we can clean up any
|
|
463
510
|
// files whose slug no longer exists (F6 replace-don't-duplicate).
|
|
464
511
|
const prevSlugs = useAI ? readPlanSlugs(journey) : [];
|
|
512
|
+
// The doc read ONCE: rules, controls and setup, plus the testkit's exports.
|
|
513
|
+
// From here on no AI call gets the whole doc; each gets its own slice.
|
|
514
|
+
// A doc without tagged rules keeps the old whole-doc flow.
|
|
515
|
+
const parsed = useAI ? readRules(markdown) : undefined;
|
|
516
|
+
const tested = testedTags();
|
|
517
|
+
let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
|
|
518
|
+
const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
|
|
519
|
+
const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
|
|
520
|
+
// What each file really tests once written: the memory and the report come from this, not the plan.
|
|
521
|
+
const written = [];
|
|
522
|
+
const finish = async (content, pf, filePath) => {
|
|
523
|
+
if (!memory || !pf.rules?.length)
|
|
524
|
+
return content;
|
|
525
|
+
const { mem, facts } = await fileMemory(pf);
|
|
526
|
+
const done = await completeFile(content, {
|
|
527
|
+
journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
|
|
528
|
+
harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
|
|
529
|
+
check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
|
|
530
|
+
});
|
|
531
|
+
written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
|
|
532
|
+
for (const n of done.notes)
|
|
533
|
+
warnings.push(`${pf.slug}: ${n}`);
|
|
534
|
+
return done.content;
|
|
535
|
+
};
|
|
536
|
+
// A doc that changed since its suite was written: only what changed is
|
|
537
|
+
// touched. Renames and removals are edits without AI; reworded and added
|
|
538
|
+
// rules get one small call each file; new sections become new files below.
|
|
539
|
+
const prevRecord = memory ? loadMemory(journey) : undefined;
|
|
540
|
+
// Rule ids always follow the last written version, even for --force and
|
|
541
|
+
// --plan-only: an id that moved would point the tests' oracle comments at
|
|
542
|
+
// another rule. Only the in-place update of the tests is skipped then.
|
|
543
|
+
let changes;
|
|
544
|
+
let docDiff;
|
|
545
|
+
if (memory && prevRecord && prevRecord.hash !== hash) {
|
|
546
|
+
const { memory: aligned, diff } = alignRules(prevRecord.rules, memory, (prevRecord.removed ?? []).map((r) => r.id));
|
|
547
|
+
memory = aligned;
|
|
548
|
+
docDiff = diff;
|
|
549
|
+
changes = { renamed: diff.renamed.map((r) => r.id), changed: diff.changed, removed: diff.removed };
|
|
550
|
+
}
|
|
551
|
+
const prevPlan = docDiff && !force && !opts.planOnly ? readPlan(journey) : undefined;
|
|
552
|
+
if (memory && docDiff && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
|
|
553
|
+
const aligned = memory;
|
|
554
|
+
const diff = docDiff;
|
|
555
|
+
console.log(` ${journey.id}: doc changed — ${describeDiff(diff)}`);
|
|
556
|
+
const facts = notBuilt ? undefined : (await groundJourney(journey, hash, 'src', markdown, true)) ?? undefined;
|
|
557
|
+
const up = await updateJourney({
|
|
558
|
+
journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
|
|
559
|
+
pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
|
|
560
|
+
check: (c, p) => (isSyntaxValid(c, ext) ? genGate(c, p) : syntaxReason(c, ext)),
|
|
561
|
+
});
|
|
562
|
+
for (const n of up.notes)
|
|
563
|
+
console.log(` ${n}`);
|
|
564
|
+
warnings.push(...up.notes);
|
|
565
|
+
writePlan(journey, hash, up.plan);
|
|
566
|
+
}
|
|
465
567
|
const planned = useAI
|
|
466
|
-
? loadOrPlan(journey, hash)
|
|
568
|
+
? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
|
|
467
569
|
: [{ slug: journey.id, title: journey.name, covers: '' }];
|
|
570
|
+
// A plan-only run leaves the memory as the tests were last written, so the
|
|
571
|
+
// next generate still sees what changed in the doc and updates those tests.
|
|
572
|
+
if (memory && !opts.planOnly) {
|
|
573
|
+
const choice = loadSourceChoice();
|
|
574
|
+
saveMemory(journey, hash, memory, helpers, {
|
|
575
|
+
doc: file, reference: choice?.journeys?.[file] ?? choice?.reference,
|
|
576
|
+
plan: planned, skipped: readPlanSkips(journey), prev: prevRecord, changes,
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
if (opts.planOnly) {
|
|
580
|
+
results.push({
|
|
581
|
+
journeyName: journey.name,
|
|
582
|
+
testDir,
|
|
583
|
+
testFiles: [],
|
|
584
|
+
seedPath,
|
|
585
|
+
status: 'skipped',
|
|
586
|
+
seedStatus: 'none',
|
|
587
|
+
warnings: [`plan only: ${planned.length} file(s) planned, nothing written — see ${join(journeyDir(journey), 'plan-check.md')}`],
|
|
588
|
+
});
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
// What one file is given: its rules with their context, the facts for
|
|
592
|
+
// its controls, the testkit list. A plan made before rules existed
|
|
593
|
+
// gives every file all the rules that get tests.
|
|
594
|
+
const allTested = memory ? memory.rules.filter((r) => r.tag && tested.includes(r.tag)).map((r) => r.id) : [];
|
|
595
|
+
const fileMemory = async (pf) => {
|
|
596
|
+
const facts = await grounding();
|
|
597
|
+
if (!memory)
|
|
598
|
+
return { facts };
|
|
599
|
+
const ids = pf.rules?.length ? pf.rules : allTested;
|
|
600
|
+
const slice = journeySlice(memory, ids);
|
|
601
|
+
return { mem: { journey: slice.text, helpers, rules: ids }, facts: factsSlice(facts, slice.controls) };
|
|
602
|
+
};
|
|
468
603
|
const plannedSlugs = planned.map((p) => p.slug);
|
|
469
604
|
// Read the journey's real code ONCE, before any file is written, and hand
|
|
470
605
|
// the same facts to every file. Without this each file re-opened the same
|
|
@@ -475,7 +610,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
475
610
|
// Only a file that has to be written or repaired needs it, so it is read
|
|
476
611
|
// on first use: re-checking an existing suite costs no grounding at all.
|
|
477
612
|
let groundingP;
|
|
478
|
-
const grounding = () => (groundingP ??= useAI
|
|
613
|
+
const grounding = () => (groundingP ??= useAI && !notBuilt
|
|
479
614
|
? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
|
|
480
615
|
: Promise.resolve(undefined));
|
|
481
616
|
const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
|
|
@@ -488,6 +623,59 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
488
623
|
// parallel agentic AI runs starve each other and time out, so a flaky
|
|
489
624
|
// environment can lower `aiConcurrency` (e.g. 2) for reliability.
|
|
490
625
|
const concurrency = Math.max(1, config.aiConcurrency ?? GEN_CONCURRENCY);
|
|
626
|
+
// Files to write, a few to one AI call: each call carries the shared rules
|
|
627
|
+
// once. A drafted file that fails its checks is written again on its own,
|
|
628
|
+
// told why; a file the batch left out is written on its own.
|
|
629
|
+
const pathOf = (pf) => join(testDir, `${flat ? `${journey.id}-${pf.slug}` : pf.slug}${ext}`);
|
|
630
|
+
const keeps = (p) => {
|
|
631
|
+
if ((force && !redone.has(i)) || !existsSync(p))
|
|
632
|
+
return false;
|
|
633
|
+
try {
|
|
634
|
+
const c = readFileSync(p, 'utf-8');
|
|
635
|
+
return isSyntaxValid(c, ext) && !isDeferredStub(c);
|
|
636
|
+
}
|
|
637
|
+
catch {
|
|
638
|
+
return false;
|
|
639
|
+
}
|
|
640
|
+
};
|
|
641
|
+
const drafts = new Map();
|
|
642
|
+
const perCall = Math.max(1, config.filesPerCall ?? 3);
|
|
643
|
+
if (useAI && memory && perCall > 1) {
|
|
644
|
+
const toWrite = planned.filter((pf) => !keeps(pathOf(pf)));
|
|
645
|
+
const batches = [];
|
|
646
|
+
// A few files to one call, but never so many rules that the writer drops
|
|
647
|
+
// some: 21 rules in one answer left 14 of them without a test.
|
|
648
|
+
const maxRules = 12;
|
|
649
|
+
for (const pf of toWrite) {
|
|
650
|
+
const last = batches[batches.length - 1];
|
|
651
|
+
const rules = (b) => b.reduce((n, f) => n + (f.rules?.length ?? 0), 0);
|
|
652
|
+
if (last && last.length < perCall && rules(last) + (pf.rules?.length ?? 0) <= maxRules)
|
|
653
|
+
last.push(pf);
|
|
654
|
+
else
|
|
655
|
+
batches.push([pf]);
|
|
656
|
+
}
|
|
657
|
+
const writeBatch = async (batch) => {
|
|
658
|
+
const files = await Promise.all(batch.map(async (pf) => {
|
|
659
|
+
const { mem, facts } = await fileMemory(pf);
|
|
660
|
+
return { path: pathOf(pf), prompt: buildPrompt(journey, '', pathOf(pf), { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem) };
|
|
661
|
+
}));
|
|
662
|
+
try {
|
|
663
|
+
const got = await writeFilesWithAI(files);
|
|
664
|
+
noteAiSuccess();
|
|
665
|
+
for (const [p, c] of got)
|
|
666
|
+
drafts.set(p, c);
|
|
667
|
+
}
|
|
668
|
+
catch (e) {
|
|
669
|
+
noteAiFailure(e);
|
|
670
|
+
}
|
|
671
|
+
};
|
|
672
|
+
// The first call alone writes the shared prompt to the cache; the rest
|
|
673
|
+
// then read it instead of each writing its own copy at the same moment.
|
|
674
|
+
const multi = batches.filter((b) => b.length > 1);
|
|
675
|
+
if (multi.length)
|
|
676
|
+
await writeBatch(multi[0]);
|
|
677
|
+
await pool(multi.slice(1), concurrency, writeBatch);
|
|
678
|
+
}
|
|
491
679
|
const generated = await pool(planned, concurrency, async (pf) => {
|
|
492
680
|
// Flat (jest): prefix the file with the journey so many journeys can
|
|
493
681
|
// share one dir without clashing. Folder (playwright): the journey
|
|
@@ -498,7 +686,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
498
686
|
// — don't spend tokens regenerating it. So a journey interrupted
|
|
499
687
|
// midway resumes from exactly the files it hadn't written yet.
|
|
500
688
|
// (--force ignores this and regenerates.)
|
|
501
|
-
|
|
689
|
+
// A redo after stubs fills in only the stubs: --force already rewrote the rest once this run.
|
|
690
|
+
if ((!force || redone.has(i)) && existsSync(filePath)) {
|
|
502
691
|
try {
|
|
503
692
|
const existing = readFileSync(filePath, 'utf-8');
|
|
504
693
|
// Keep a real, parsing file — but a DEFERRED stub is not real
|
|
@@ -507,8 +696,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
507
696
|
// A suite written before pending tests carried their reason gets
|
|
508
697
|
// it now, deterministically — no regeneration, no AI.
|
|
509
698
|
const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
|
|
510
|
-
|
|
511
|
-
|
|
699
|
+
// A kept file that leaves rules out or holds tests that cannot fail:
|
|
700
|
+
// only those tests are written again.
|
|
701
|
+
const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath) : normalised, pending);
|
|
702
|
+
if (finished !== existing)
|
|
703
|
+
writeFileSync(filePath, finished, 'utf-8');
|
|
512
704
|
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
513
705
|
}
|
|
514
706
|
}
|
|
@@ -519,10 +711,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
519
711
|
let content;
|
|
520
712
|
let status;
|
|
521
713
|
if (useAI) {
|
|
522
|
-
const
|
|
523
|
-
const
|
|
714
|
+
const draft = drafts.get(filePath);
|
|
715
|
+
const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
|
|
716
|
+
const { mem, facts } = await fileMemory(pf);
|
|
717
|
+
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
|
|
524
718
|
if (ai !== null) {
|
|
525
|
-
content = framework === 'playwright' ? normalisePending(ai) : ai;
|
|
719
|
+
content = await finish(framework === 'playwright' ? normalisePending(ai) : ai, pf, filePath);
|
|
526
720
|
status = 'ai';
|
|
527
721
|
}
|
|
528
722
|
else {
|
|
@@ -541,6 +735,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
541
735
|
if (!content.includes(GEN_MARKER)) {
|
|
542
736
|
content = `// ${GEN_MARKER}\n${content}`;
|
|
543
737
|
}
|
|
738
|
+
content = withPendingStamp(content, pending);
|
|
544
739
|
// Write IMMEDIATELY (not batched at the end of the journey), so an
|
|
545
740
|
// interruption preserves every file generated so far.
|
|
546
741
|
// keepDeferredStubs:false (config) — do NOT write a DEFERRED stub file;
|
|
@@ -553,6 +748,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
553
748
|
}
|
|
554
749
|
return { slug: pf.slug, path: filePath, status };
|
|
555
750
|
});
|
|
751
|
+
if (memory && written.length) {
|
|
752
|
+
const sum = recordWritten(journey, written);
|
|
753
|
+
console.log(` ${journey.id}: ${sum}`);
|
|
754
|
+
}
|
|
556
755
|
const testFiles = generated.map((g) => ({
|
|
557
756
|
slug: g.slug,
|
|
558
757
|
path: g.path,
|
|
@@ -631,54 +830,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
631
830
|
warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
|
|
632
831
|
lint.remaining.slice(0, 5).join('; '));
|
|
633
832
|
}
|
|
634
|
-
|
|
635
|
-
// done — file by file, then the whole journey together, then an independent
|
|
636
|
-
// review of every claimed defect (see verify-journey.ts). Anything that
|
|
637
|
-
// could not run, or still fails unexplained, leaves the journey UNVERIFIED
|
|
638
|
-
// and uncached, so the next run checks it again.
|
|
639
|
-
let verified;
|
|
640
|
-
let reviewed = [];
|
|
641
|
-
const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
|
|
642
|
-
const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
|
|
643
|
-
if (useAI && verifyOn && targets.length && missing.length) {
|
|
644
|
-
// The suite cannot even start; running it would only feed repair a
|
|
645
|
-
// failure no test edit can fix.
|
|
646
|
-
verified = false;
|
|
647
|
-
warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
|
|
648
|
-
}
|
|
649
|
-
else if (useAI && verifyOn && targets.length) {
|
|
650
|
-
const result = await verifyJourney({
|
|
651
|
-
config,
|
|
652
|
-
journey,
|
|
653
|
-
targets,
|
|
654
|
-
attempts: verifyAttempts,
|
|
655
|
-
repair: async (target, current, failure) => {
|
|
656
|
-
const facts = await grounding();
|
|
657
|
-
const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
|
|
658
|
-
if (out === null)
|
|
659
|
-
return null;
|
|
660
|
-
const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
|
|
661
|
-
return framework === 'playwright' ? normalisePending(stamped) : stamped;
|
|
662
|
-
},
|
|
663
|
-
});
|
|
664
|
-
verified = result.verified;
|
|
665
|
-
reviewed = result.reviewed;
|
|
666
|
-
warnings.push(...result.warnings);
|
|
667
|
-
}
|
|
668
|
-
const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
|
|
833
|
+
const degraded = useAI && (status === 'basic' || seedStatus === 'failed');
|
|
669
834
|
if (!degraded) {
|
|
670
835
|
cache[file] = hash;
|
|
671
|
-
// Remembered apart, so a journey cached before it was ever run is
|
|
672
|
-
// re-checked (not regenerated) once verification is on.
|
|
673
|
-
if (verified)
|
|
674
|
-
cache[`verified:${file}`] = hash;
|
|
675
836
|
}
|
|
676
837
|
else {
|
|
677
838
|
// Actively UN-mark a degraded journey (even under --force, which may
|
|
678
839
|
// be re-processing a stale "done" from an earlier buggy run), so the
|
|
679
840
|
// next `generate` re-does it instead of skipping a thin fallback.
|
|
680
841
|
delete cache[file];
|
|
681
|
-
delete cache[`verified:${file}`];
|
|
682
842
|
}
|
|
683
843
|
saveCache(CACHE_PATH, cache);
|
|
684
844
|
resultRow.set(file, results.length);
|
|
@@ -696,9 +856,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
696
856
|
seedPath,
|
|
697
857
|
status,
|
|
698
858
|
seedStatus,
|
|
699
|
-
verified,
|
|
700
859
|
warnings: warnings.length ? warnings : undefined,
|
|
701
|
-
reviewed: reviewed.length ? reviewed : undefined,
|
|
702
860
|
ai,
|
|
703
861
|
});
|
|
704
862
|
}
|