@activate-core/test-forge 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +21 -0
- package/dist/cli/index.js +14 -4
- package/dist/core/docs/discover.js +15 -0
- package/dist/core/docs/doc-origin.js +42 -0
- package/dist/core/docs/reference.js +18 -1
- package/dist/core/gates/validate.js +8 -0
- package/dist/core/generate/actions.js +200 -0
- package/dist/core/generate/ai-generator.js +41 -0
- package/dist/core/generate/ai-usage.js +1 -0
- package/dist/core/generate/complete-file.js +62 -2
- package/dist/core/generate/harness.js +57 -14
- package/dist/core/generate/run-generate.js +203 -27
- package/dist/core/generate/test-blocks.js +123 -2
- package/dist/core/generate/update-journey.js +23 -3
- package/dist/core/project/preflight.js +69 -0
- package/dist/core/project/update.js +120 -14
- package/dist/core/prompts/blocks.js +17 -0
- package/package.json +1 -1
- package/reporters/_shared.cjs +33 -2
- package/reporters/http.cjs +96 -0
- package/reporters/interface.cjs +25 -2
- package/reporters/jest-reporter.cjs +3 -1
|
@@ -37,7 +37,7 @@ export const HARNESS_CONTRACT = [
|
|
|
37
37
|
`- one resolver per seeded entity that tests read back (H3)`,
|
|
38
38
|
`- unique(label?): string, and created(kind, id, cleanup?, role?): void (H8)`,
|
|
39
39
|
`- distinctRunAccount(): boolean (H8)`,
|
|
40
|
-
`- resetRateLimits(): Promise<
|
|
40
|
+
`- resetRateLimits(): Promise<number>, the entries it cleared (H7)`,
|
|
41
41
|
`- REQUIRED_PACKAGES: string[] (H6)`,
|
|
42
42
|
`- a 'run' role alongside the fixture roles (H8)`,
|
|
43
43
|
`- insert(table, row): Promise<id>, one row of scenario data written straight`,
|
|
@@ -48,14 +48,19 @@ export const HARNESS_CONTRACT = [
|
|
|
48
48
|
`Jest areas only`,
|
|
49
49
|
`- call(method, path, { body?, headers?, role? }):`,
|
|
50
50
|
` Promise<{ status, headers, body }>`,
|
|
51
|
-
` Every API test sends its requests through it
|
|
52
|
-
`
|
|
53
|
-
`
|
|
54
|
-
`
|
|
55
|
-
`
|
|
56
|
-
`
|
|
57
|
-
`
|
|
58
|
-
`
|
|
51
|
+
` Every API test sends its requests through it. It is TestForge's own, made`,
|
|
52
|
+
` with makeCall; never write call(), its token handling or its retries`,
|
|
53
|
+
` yourself:`,
|
|
54
|
+
` const { makeCall, verifyDerivedState } =`,
|
|
55
|
+
` require('@activate-core/test-forge/reporters/http.cjs');`,
|
|
56
|
+
` const call = makeCall({ url, ensureToken, resetRateLimits, refused, send });`,
|
|
57
|
+
` url(path) gives the full URL (the product's prefix rules); ensureToken(role,`,
|
|
58
|
+
` { force }) a token the product accepts; refused(status) is true for the`,
|
|
59
|
+
` statuses that mean "token not accepted" (default 401). Pass send(method,`,
|
|
60
|
+
` url, { headers, body }) only for a Node server this package can require and`,
|
|
61
|
+
` drive in-process with its framework's inject; otherwise leave it out (HTTP to`,
|
|
62
|
+
` the base URL in the env file). Never require '@playwright/test': it is not`,
|
|
63
|
+
` installed in backend packages, and a top-level require breaks every suite.`,
|
|
59
64
|
`Playwright areas only`,
|
|
60
65
|
`- login(page, role?): Promise<void>, a session the app really accepts (H2).`,
|
|
61
66
|
` It calls suppressPopups first.`,
|
|
@@ -143,9 +148,11 @@ export const HARNESS_CONTRACT = [
|
|
|
143
148
|
` - suppressPopups(page) uses an init script to pre-set the flags that stop`,
|
|
144
149
|
` promotions, consent, onboarding and permission prompts, before the app's`,
|
|
145
150
|
` code runs and across navigations.`,
|
|
146
|
-
` - resetRateLimits() clears only the limiter's entries
|
|
147
|
-
`
|
|
148
|
-
`
|
|
151
|
+
` - resetRateLimits() clears only the limiter's entries (read in the code how`,
|
|
152
|
+
` the limiter names them, prefix included), and only when they live in a`,
|
|
153
|
+
` store the testkit can reach. It removes no data, so it needs only a target`,
|
|
154
|
+
` that is clearly not production (H5), not TESTFORGE_ALLOW_RESET. It`,
|
|
155
|
+
` returns how many entries it cleared and never throws; setup() calls it.`,
|
|
149
156
|
` - bypassVerification answers the final verify request at url with the`,
|
|
150
157
|
` app's real success shape, carrying a real token for an env account of`,
|
|
151
158
|
` that role. It answers sendUrl the same way when the environment cannot`,
|
|
@@ -189,6 +196,24 @@ export const HARNESS_CONTRACT = [
|
|
|
189
196
|
` - insert fills every NOT NULL column without a default that row leaves out`,
|
|
190
197
|
` with a value of its type (read the columns from the database), returns`,
|
|
191
198
|
` the new id, and remembers the row.`,
|
|
199
|
+
` - A row the product checks before it will use it (a service needs a price`,
|
|
200
|
+
` above the minimum, a record needs to be enabled): read those checks in the`,
|
|
201
|
+
` code and have insert set each checked column the row leaves out to a value`,
|
|
202
|
+
` that passes it, per table. A row a test writes is then usable, the way a`,
|
|
203
|
+
` real one is.`,
|
|
204
|
+
` - Derived state. Some tables the product fills by itself when a row is`,
|
|
205
|
+
` saved: model events and observers, and the jobs they queue (a busy-time`,
|
|
206
|
+
` calendar built from working hours and orders, counters, a search table).`,
|
|
207
|
+
` insert() and update() write straight to the database and fire none of`,
|
|
208
|
+
` them, so read the observers, events and jobs of every table a scenario`,
|
|
209
|
+
` writes. After such a write, insert and update bring the derived rows to`,
|
|
210
|
+
` what the product would have made, computing them the way that code does,`,
|
|
211
|
+
` and remember them so they are undone with the row. Export`,
|
|
212
|
+
` DERIVED_STATE: { <table written>: '<what is rebuilt, from which code>' },`,
|
|
213
|
+
` keyed by the table name the model really uses (its $table, its`,
|
|
214
|
+
` tableName), and have setup() call verifyDerivedState(DERIVED_STATE, q)`,
|
|
215
|
+
` once connected, where q(sql) runs SQL on the test database.`,
|
|
216
|
+
` A scenario built without this tests nothing: every operator looks free.`,
|
|
192
217
|
` - update remembers the previous values of the rows it changes.`,
|
|
193
218
|
` - Each test starts from a clean slate: the rows a test writes, and what it`,
|
|
194
219
|
` updates, are undone right after that test (last first), so one test's`,
|
|
@@ -314,9 +339,10 @@ export function generateHarness(harnessPath, framework) {
|
|
|
314
339
|
return cleanOutput(output);
|
|
315
340
|
}
|
|
316
341
|
/** Async twin, so a harness can generate alongside other work. */
|
|
317
|
-
export function generateHarnessAsync(harnessPath, framework) {
|
|
342
|
+
export function generateHarnessAsync(harnessPath, framework, refusal) {
|
|
318
343
|
const config = loadConfig();
|
|
319
|
-
const prompt = buildHarnessPrompt(harnessPath, framework)
|
|
344
|
+
const prompt = buildHarnessPrompt(harnessPath, framework) +
|
|
345
|
+
(refusal ? `\n\nYour previous testkit was refused: ${refusal.reason}. Write it again with that fixed.` : '');
|
|
320
346
|
const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
|
|
321
347
|
// Same runner as every other async AI call: one timeout guard, one usage count, one trace.
|
|
322
348
|
return runWithStdin(program, args, prompt, config.aiTimeoutMs ?? 300_000).then(cleanOutput);
|
|
@@ -507,3 +533,20 @@ export function fixturesDoc() {
|
|
|
507
533
|
``,
|
|
508
534
|
].join('\n');
|
|
509
535
|
}
|
|
536
|
+
/**
|
|
537
|
+
* Why a generated testkit is not taken, or null: a Jest testkit must build
|
|
538
|
+
* call() with TestForge's makeCall (the header and 429 rules are code, not
|
|
539
|
+
* something each testkit is trusted to write), and one that derives rows must
|
|
540
|
+
* check its table names at setup.
|
|
541
|
+
*/
|
|
542
|
+
export function harnessGate(source, framework) {
|
|
543
|
+
if (framework !== 'jest')
|
|
544
|
+
return null;
|
|
545
|
+
if (!/reporters\/http\.cjs/.test(source) || !/\bmakeCall\s*\(/.test(source)) {
|
|
546
|
+
return "call() must be made with makeCall from require('@activate-core/test-forge/reporters/http.cjs'), not written by hand";
|
|
547
|
+
}
|
|
548
|
+
if (/\bDERIVED_STATE\b/.test(source) && !/\bverifyDerivedState\s*\(/.test(source)) {
|
|
549
|
+
return 'setup() must call verifyDerivedState(DERIVED_STATE, q) so a wrong table name stops the run instead of deriving nothing';
|
|
550
|
+
}
|
|
551
|
+
return null;
|
|
552
|
+
}
|
|
@@ -26,13 +26,48 @@ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, is
|
|
|
26
26
|
import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
|
-
import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
|
-
import { completeFile } from './complete-file.js';
|
|
31
|
-
import { tuckStrayTests } from './test-blocks.js';
|
|
29
|
+
import { rewriteSetupWithAI, buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
|
+
import { completeFile, withoutBlankNames, withoutNames } from './complete-file.js';
|
|
31
|
+
import { findSetupBlocks, findTestBlocks, tuckStrayTests, useInsertIds, withoutDuplicateTests } from './test-blocks.js';
|
|
32
32
|
import { passedTitles } from './passed.js';
|
|
33
|
+
import { actionsPath, actionsStale, clearActionFailures, exportedNames, generateActionsAsync, journeysWith, stampActions } from './actions.js';
|
|
34
|
+
import { draftedByTestForge } from '../docs/doc-origin.js';
|
|
33
35
|
import { scanNames } from '../run/name-scan.js';
|
|
34
36
|
import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
|
|
35
37
|
/** Names the file asks for that testforge/interface.json still has blank. */
|
|
38
|
+
/**
|
|
39
|
+
* Is a blank name used outside every test (a hook, a helper, the file's
|
|
40
|
+
* setup)? Then the whole file waits on it, and only writing the whole file
|
|
41
|
+
* again — with the code facts — clears it; no test of it can have passed.
|
|
42
|
+
*/
|
|
43
|
+
export function blankUsedOutsideTests(content) {
|
|
44
|
+
const blocks = findTestBlocks(content);
|
|
45
|
+
const outside = blocks.reduceRight((c, b) => c.slice(0, b.start) + c.slice(b.end), content);
|
|
46
|
+
return blankNames(content).some((n) => {
|
|
47
|
+
const esc = n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
48
|
+
const v = new RegExp(`\\b(?:const|let|var)\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*need\\(\\s*(['"\`])${esc}\\2`).exec(content)?.[1];
|
|
49
|
+
if (!v)
|
|
50
|
+
return false;
|
|
51
|
+
const uses = outside.match(new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`, 'g')) ?? [];
|
|
52
|
+
return uses.length > 1; // more than its own declaration
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Where a file's blank names are used outside its tests: 'setup' when every
|
|
57
|
+
* such use sits in a hook or helper TestForge can rewrite on its own, 'file'
|
|
58
|
+
* when one sits somewhere it cannot split out (a parametrised table, a test
|
|
59
|
+
* built in a loop), undefined when none.
|
|
60
|
+
*/
|
|
61
|
+
export function blankOutsideKind(content) {
|
|
62
|
+
if (!blankUsedOutsideTests(content))
|
|
63
|
+
return undefined;
|
|
64
|
+
const vars = [...content.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]+)\2/g)]
|
|
65
|
+
.filter((m) => blankNames(`need('${m[3]}')`).length > 0).map((m) => m[1]);
|
|
66
|
+
const covered = [...findTestBlocks(content), ...findSetupBlocks(content)];
|
|
67
|
+
const rest = covered.sort((a, b) => b.start - a.start).reduce((c, b) => c.slice(0, b.start) + c.slice(b.end), content);
|
|
68
|
+
const used = vars.some((v) => (rest.match(new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`, 'g')) ?? []).length > 1);
|
|
69
|
+
return used ? 'file' : 'setup';
|
|
70
|
+
}
|
|
36
71
|
/** Does this test use a name the product has not given yet (directly, or through the constant it was put in)? */
|
|
37
72
|
function usesBlankName(content, block) {
|
|
38
73
|
const text = content.slice(block.start, block.end);
|
|
@@ -81,7 +116,7 @@ import { updateJourney } from './update-journey.js';
|
|
|
81
116
|
import { sourceRoot, sourceText } from '../docs/excerpts.js';
|
|
82
117
|
import { isSyntaxValid, checkTestGates, trimExtraTests, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
|
|
83
118
|
import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
|
|
84
|
-
import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
119
|
+
import { harnessInfo, generateHarnessAsync, harnessGate, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
|
|
85
120
|
import { hashContent, loadCache, saveCache } from '../docs/cache.js';
|
|
86
121
|
import { findMarkdownFiles } from '../find-files.js';
|
|
87
122
|
import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
|
|
@@ -312,7 +347,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
312
347
|
console.log(` the ${area} test harness predates this TestForge version — rebuilding it`);
|
|
313
348
|
}
|
|
314
349
|
const old = existsSync(info.path) ? readFileSync(info.path, 'utf-8') : '';
|
|
315
|
-
const kit = await generateValidatedAsync(() => generateHarnessAsync(info.path, framework), ext.endsWith('.js') ? '.js' : '.ts');
|
|
350
|
+
const kit = await generateValidatedAsync((refusal) => generateHarnessAsync(info.path, framework, refusal), ext.endsWith('.js') ? '.js' : '.ts', (c) => harnessGate(c, framework));
|
|
316
351
|
// The tests already written call the old testkit: a rebuild that fails, or
|
|
317
352
|
// drops something they use, is not taken — the old one stays.
|
|
318
353
|
const lost = old && kit !== null ? exportNames(old).filter((n) => !exportNames(kit).includes(n)) : [];
|
|
@@ -336,6 +371,28 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
336
371
|
harnessByArea.set(area, info.path);
|
|
337
372
|
return info.path;
|
|
338
373
|
}
|
|
374
|
+
// The actions a journey's people take (book, pay, accept…), built once from the
|
|
375
|
+
// handler and the product's own client, so no test guesses a request's shape.
|
|
376
|
+
// Jest areas only. Built again when its contract, this journey's doc, or a
|
|
377
|
+
// refusal in the last run says so; a build that fails or drops an export the
|
|
378
|
+
// tests use is not taken.
|
|
379
|
+
async function ensureActions(harnessPath, journeyId, docPath, doc) {
|
|
380
|
+
const path = actionsPath(harnessPath);
|
|
381
|
+
const why = actionsStale(path, journeyId, doc);
|
|
382
|
+
if (!why)
|
|
383
|
+
return { path, built: false };
|
|
384
|
+
console.log(` actions: building (${why})`);
|
|
385
|
+
const old = existsSync(path) ? readFileSync(path, 'utf-8') : '';
|
|
386
|
+
const src = await generateValidatedAsync(() => generateActionsAsync(path, harnessPath, [docPath]), ext.endsWith('.ts') ? '.ts' : '.js');
|
|
387
|
+
const lost = old && src !== null ? exportedNames(old).filter((n) => !exportedNames(src).includes(n)) : [];
|
|
388
|
+
if (src === null || lost.length) {
|
|
389
|
+
console.log(` kept the existing actions: the build ${src === null ? 'failed' : `dropped ${lost.join(', ')}`}`);
|
|
390
|
+
return { path: old ? path : undefined, built: false };
|
|
391
|
+
}
|
|
392
|
+
writeFileSync(path, stampActions(src, journeysWith(path, journeyId, doc)), 'utf-8');
|
|
393
|
+
clearActionFailures();
|
|
394
|
+
return { path, built: true };
|
|
395
|
+
}
|
|
339
396
|
const waitForLimit = opts.waitForLimit ?? config.waitForLimit ?? true;
|
|
340
397
|
let notAttempted = 0;
|
|
341
398
|
const redone = new Set();
|
|
@@ -480,7 +537,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
480
537
|
const appTexts = () => framework === 'playwright'
|
|
481
538
|
? (knownTexts ??= `${sourceText([sourceRoot(packageRoot(areaDir)), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
|
|
482
539
|
: undefined;
|
|
483
|
-
const fixTests = (content) => trimExtraTests(content, config.maxTestsPerRule ?? 2);
|
|
540
|
+
const fixTests = (content) => trimExtraTests(withoutDuplicateTests(content), config.maxTestsPerRule ?? 2);
|
|
484
541
|
const genGate = (content, filePath) => testGate(content) ??
|
|
485
542
|
checkTestGates(content, ext, {
|
|
486
543
|
framework, forbidPlaceholders: true, filePath,
|
|
@@ -504,6 +561,46 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
504
561
|
return {};
|
|
505
562
|
}
|
|
506
563
|
};
|
|
564
|
+
const rememberRefused = (slug, ids) => {
|
|
565
|
+
try {
|
|
566
|
+
const all = readRefused();
|
|
567
|
+
all[slug] = [...new Set([...(all[slug] ?? []), ...ids])];
|
|
568
|
+
mkdirSync(dirname(refusedPath), { recursive: true });
|
|
569
|
+
writeFileSync(refusedPath, JSON.stringify({ key: refusedKey(), rules: all }, null, 2), 'utf-8');
|
|
570
|
+
}
|
|
571
|
+
catch { /* best effort */ }
|
|
572
|
+
};
|
|
573
|
+
/** The file with the setup blocks that use a blank name written again, or null when that did not work. */
|
|
574
|
+
const redoSetup = async (content, filePath, facts, kitApi) => {
|
|
575
|
+
const isBlank = (n) => blankNames(`need('${n}')`).length > 0;
|
|
576
|
+
const blankVars = [...content.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]+)\2/g)]
|
|
577
|
+
.filter((m) => isBlank(m[3])).map((m) => m[1]);
|
|
578
|
+
const uses = (t) => blankVars.some((v) => new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(t));
|
|
579
|
+
const blocks = findSetupBlocks(content).filter((b) => uses(b.text));
|
|
580
|
+
if (!blocks.length)
|
|
581
|
+
return null;
|
|
582
|
+
const top = withoutBlankNames(content.slice(0, Math.min(...blocks.map((b) => b.start))), isBlank).slice(-6000);
|
|
583
|
+
let got = null;
|
|
584
|
+
try {
|
|
585
|
+
got = await rewriteSetupWithAI({ filePath, top, blocks, facts, helpers: kitApi });
|
|
586
|
+
noteAiSuccess();
|
|
587
|
+
}
|
|
588
|
+
catch (e) {
|
|
589
|
+
noteAiFailure(e);
|
|
590
|
+
return null;
|
|
591
|
+
}
|
|
592
|
+
if (!got)
|
|
593
|
+
return null;
|
|
594
|
+
// Each block is still what it was (the same hook or helper), and asks for nothing.
|
|
595
|
+
if (got.some((g, i) => !g.includes(blocks[i].name) || /\bneed\(/.test(g) || uses(g)))
|
|
596
|
+
return null;
|
|
597
|
+
let out = content;
|
|
598
|
+
for (let i = blocks.length - 1; i >= 0; i--)
|
|
599
|
+
out = out.slice(0, blocks[i].start) + got[i] + out.slice(blocks[i].end);
|
|
600
|
+
out = withoutNames(out);
|
|
601
|
+
const why = isSyntaxValid(out, ext) ? genGate(out, filePath) : syntaxReason(out, ext);
|
|
602
|
+
return why ? null : out;
|
|
603
|
+
};
|
|
507
604
|
const suiteComplete = () => {
|
|
508
605
|
const refused = readRefused();
|
|
509
606
|
const plan = readPlan(journey);
|
|
@@ -528,7 +625,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
528
625
|
continue;
|
|
529
626
|
const p = join(testDir, f);
|
|
530
627
|
const before = readFileSync(p, 'utf-8');
|
|
531
|
-
|
|
628
|
+
// Both mechanical, no AI: stray tests back inside the describe, and
|
|
629
|
+
// kit.insert's id used as the id it is (op[ID_FIELD] → op).
|
|
630
|
+
const after = useInsertIds(tuckStrayTests(before));
|
|
532
631
|
if (after !== before)
|
|
533
632
|
writeFileSync(p, after, 'utf-8');
|
|
534
633
|
}
|
|
@@ -546,6 +645,28 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
546
645
|
catch {
|
|
547
646
|
return false;
|
|
548
647
|
} }));
|
|
648
|
+
// The doc is the same and every rule has its test: new code changes no
|
|
649
|
+
// test. Its facts are read again and the names the tests wait on are
|
|
650
|
+
// filled from them; the test files are not written again (that is
|
|
651
|
+
// --rewrite-waiting, asked for by name).
|
|
652
|
+
if (!force && !opts.rewriteWaiting && waitsOnCode && cache[file] === hash && !hasDeferredStubs(testDir, ext) && suiteComplete()) {
|
|
653
|
+
const usage0 = aiUsage();
|
|
654
|
+
const facts = /^status:\s*not built/im.test(markdown) ? null : await groundJourney(journey, hash, sourceRoot(), markdown, false, opts.refreshCode);
|
|
655
|
+
const blank = [...new Set(testFiles0.flatMap((p) => { try {
|
|
656
|
+
return blankNames(readFileSync(p, 'utf-8'));
|
|
657
|
+
}
|
|
658
|
+
catch {
|
|
659
|
+
return [];
|
|
660
|
+
} }))];
|
|
661
|
+
const res = facts && blank.length ? scanNames(blank, sourceRoot(), facts) : undefined;
|
|
662
|
+
const notes = [
|
|
663
|
+
`doc unchanged: no test written again (code facts read again${res ? `, ${res.filled.length} of ${blank.length} blank name(s) filled from them` : ''})`,
|
|
664
|
+
...(res?.filled.length ? [`filled: ${res.filled.slice(0, 8).map((f) => `${f.name} = ${f.value}`).join(', ')}${res.filled.length > 8 ? ', …' : ''}`] : []),
|
|
665
|
+
...(res?.stillBlank.length ? [`${res.stillBlank.length} name(s) still blank — fill them in testforge/interface.json, or run with --rewrite-waiting to have the tests written again: ${res.stillBlank.slice(0, 8).join(', ')}${res.stillBlank.length > 8 ? ', …' : ''}`] : []),
|
|
666
|
+
];
|
|
667
|
+
results.push({ journeyName: journey.name, testDir, testFiles: [], seedPath, status: 'skipped', seedStatus: 'none', warnings: notes, usage: usageSince(usage0) });
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
549
670
|
if (!force &&
|
|
550
671
|
!waitsOnCode &&
|
|
551
672
|
cache[file] === hash &&
|
|
@@ -611,7 +732,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
611
732
|
const parsed = useAI ? readRules(markdown) : undefined;
|
|
612
733
|
const tested = testedTags();
|
|
613
734
|
let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
|
|
614
|
-
const
|
|
735
|
+
const actions = useAI && harnessPath && framework === 'jest' ? await ensureActions(harnessPath, journey.id, join(docsDir, file), markdown) : { built: false };
|
|
736
|
+
const actionsFile = actions.path;
|
|
737
|
+
const actionsApi = actionsFile && existsSync(actionsFile)
|
|
738
|
+
? `\n\nACTIONS, the module at ${actionsFile} (import it like the testkit, as act):\n${helperApi(readFileSync(actionsFile, 'utf-8'))}`
|
|
739
|
+
: '';
|
|
740
|
+
const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) + actionsApi : '';
|
|
615
741
|
const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
|
|
616
742
|
// What each file really tests once written: the memory and the report come from this, not the plan.
|
|
617
743
|
const written = [];
|
|
@@ -624,6 +750,20 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
624
750
|
if (!memory || !pf.rules?.length)
|
|
625
751
|
return content;
|
|
626
752
|
const { mem, facts } = await fileMemory(pf);
|
|
753
|
+
// A blank name in a hook or helper makes the whole file wait. Only the
|
|
754
|
+
// setup is written again (one small call), never the tests; once per
|
|
755
|
+
// doc and code if it is refused.
|
|
756
|
+
if (kept && stale.has(filePath) && blankOutsideKind(content) === 'setup' && !(readRefused()[pf.slug] ?? []).includes('#setup')) {
|
|
757
|
+
const redone = await redoSetup(content, filePath, facts, mem?.helpers ?? helpers);
|
|
758
|
+
if (redone) {
|
|
759
|
+
warnings.push(`${pf.slug}: its setup (hooks and helpers) was written again from the code facts; its tests were kept`);
|
|
760
|
+
content = redone;
|
|
761
|
+
}
|
|
762
|
+
else {
|
|
763
|
+
warnings.push(`${pf.slug}: its setup could not be written again — the file keeps waiting on its names`);
|
|
764
|
+
rememberRefused(pf.slug, ['#setup']);
|
|
765
|
+
}
|
|
766
|
+
}
|
|
627
767
|
const done = await completeFile(content, {
|
|
628
768
|
kept, locked: passedTitles(filePath),
|
|
629
769
|
redo: kept && stale.has(filePath) ? (redoAll ? () => true : (b) => usesBlankName(content, b)) : undefined,
|
|
@@ -634,17 +774,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
634
774
|
harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
|
|
635
775
|
check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
|
|
636
776
|
});
|
|
637
|
-
written
|
|
638
|
-
//
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
777
|
+
// A file written again whole is taken only if it still tests every rule the
|
|
778
|
+
// earlier one did: a waiting test beats no test.
|
|
779
|
+
const previous = !kept && stale.has(filePath) && existsSync(filePath) ? readFileSync(filePath, 'utf-8') : undefined;
|
|
780
|
+
if (previous) {
|
|
781
|
+
const rules = pf.rules ?? [];
|
|
782
|
+
const had = rules.filter((r) => !ruleCoverage(previous, rules).missing.includes(r) && findTestBlocks(previous).some((t) => t.rules.includes(r)));
|
|
783
|
+
const lost = had.filter((r) => !done.tested.includes(r));
|
|
784
|
+
if (lost.length) {
|
|
785
|
+
warnings.push(`${pf.slug}: kept the earlier file — written again whole it lost the test for ${lost.join(', ')}`);
|
|
786
|
+
const cov = ruleCoverage(previous, rules);
|
|
787
|
+
written.push({ slug: pf.slug, tested: had, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [] });
|
|
788
|
+
return previous;
|
|
645
789
|
}
|
|
646
|
-
catch { /* best effort */ }
|
|
647
790
|
}
|
|
791
|
+
written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
|
|
792
|
+
// What was asked and still has no test is not asked again for this doc and code.
|
|
793
|
+
if (done.refusedRules.length)
|
|
794
|
+
rememberRefused(pf.slug, done.refusedRules);
|
|
648
795
|
for (const n of neededNames(done.content))
|
|
649
796
|
if (!blankNames(`need('${n}')`).length)
|
|
650
797
|
askedNames.add(n);
|
|
@@ -677,6 +824,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
677
824
|
journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
|
|
678
825
|
pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
|
|
679
826
|
check: (c, p) => (isSyntaxValid(c, ext) ? genGate(c, p) : syntaxReason(c, ext)),
|
|
827
|
+
locked: (p) => passedTitles(p),
|
|
828
|
+
// Only a person's edit to the doc (or --force) may change a test that passed.
|
|
829
|
+
protectPassed: !force && draftedByTestForge(file, markdown),
|
|
680
830
|
});
|
|
681
831
|
for (const n of up.notes)
|
|
682
832
|
console.log(` ${n}`);
|
|
@@ -768,7 +918,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
768
918
|
const stale = new Set();
|
|
769
919
|
// Every test of a journey first written with no code is looked at again once its code is there.
|
|
770
920
|
let redoAll = false;
|
|
771
|
-
if (useAI && !notBuilt && blindBefore && testFiles0.length && (await grounding())) {
|
|
921
|
+
if (useAI && !notBuilt && opts.rewriteWaiting && blindBefore && testFiles0.length && (await grounding())) {
|
|
772
922
|
for (const p of planned.map(pathOf))
|
|
773
923
|
if (existsSync(p))
|
|
774
924
|
stale.add(p);
|
|
@@ -776,25 +926,48 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
776
926
|
if (redoAll)
|
|
777
927
|
warnings.push(`its code is here now: the tests first written without it are written again from the code (tests that passed are kept)`);
|
|
778
928
|
}
|
|
779
|
-
if (useAI && !notBuilt && !redoAll) {
|
|
929
|
+
if (useAI && !notBuilt && !redoAll && opts.rewriteWaiting) {
|
|
780
930
|
const waiting = planned.map(pathOf).filter((p) => { try {
|
|
781
931
|
return existsSync(p) && blankNames(readFileSync(p, 'utf-8')).length > 0;
|
|
782
932
|
}
|
|
783
933
|
catch {
|
|
784
934
|
return false;
|
|
785
935
|
} });
|
|
786
|
-
// Only new code (or --refresh-code)
|
|
787
|
-
// again from the same code may word things
|
|
788
|
-
// must never cost a rewrite.
|
|
789
|
-
if (waiting.length && (opts.refreshCode || codeMoved) && (await grounding())) {
|
|
936
|
+
// Only new code (or --refresh-code), or actions built this run, rewrite a
|
|
937
|
+
// waiting file: facts read again from the same code may word things
|
|
938
|
+
// differently, and that alone must never cost a rewrite.
|
|
939
|
+
if (waiting.length && (opts.refreshCode || codeMoved || actions.built) && (await grounding())) {
|
|
790
940
|
for (const p of waiting)
|
|
791
941
|
stale.add(p);
|
|
792
|
-
if (stale.size)
|
|
793
|
-
|
|
942
|
+
if (stale.size) {
|
|
943
|
+
const kinds = [...stale].map((p) => { try {
|
|
944
|
+
return blankOutsideKind(readFileSync(p, 'utf-8'));
|
|
945
|
+
}
|
|
946
|
+
catch {
|
|
947
|
+
return undefined;
|
|
948
|
+
} });
|
|
949
|
+
const setup = kinds.filter((k) => k === 'setup').length;
|
|
950
|
+
const done = readRefused();
|
|
951
|
+
const whole = [...stale].filter((p, i) => kinds[i] === 'file' && !(done[planned.find((pf) => pathOf(pf) === p)?.slug ?? ''] ?? []).includes('#whole')).length;
|
|
952
|
+
warnings.push(`${stale.size} file(s) waiting on names${actions.built ? ' (the actions module is new or rebuilt)' : ' the code now has'}: only their waiting tests are written again${setup ? `; the setup (hooks, helpers) of ${setup}` : ''}${whole ? `; ${whole} written again whole, once (their names sit in parametrised tables or loops)` : ''}`);
|
|
953
|
+
}
|
|
794
954
|
}
|
|
795
955
|
}
|
|
796
956
|
// A doc without rule ids has no per-test redo: its waiting files are written again whole, as before.
|
|
797
|
-
|
|
957
|
+
// Written again whole: a doc without rule ids (no per-test redo), or a file
|
|
958
|
+
// whose blank names sit where no test or setup block can be split out —
|
|
959
|
+
// once per doc and code, and never a file holding a test that passed.
|
|
960
|
+
const rewriteWhole = (p) => stale.has(p) && (!memory || (() => {
|
|
961
|
+
try {
|
|
962
|
+
const c = readFileSync(p, 'utf-8');
|
|
963
|
+
const slug = planned.find((pf) => pathOf(pf) === p)?.slug ?? '';
|
|
964
|
+
return blankOutsideKind(c) === 'file' && !(readRefused()[slug] ?? []).includes('#whole')
|
|
965
|
+
&& !findTestBlocks(c).some((b) => passedTitles(p).has(b.title));
|
|
966
|
+
}
|
|
967
|
+
catch {
|
|
968
|
+
return false;
|
|
969
|
+
}
|
|
970
|
+
})());
|
|
798
971
|
const keeps = (p) => {
|
|
799
972
|
if (rewriteWhole(p))
|
|
800
973
|
return false;
|
|
@@ -865,7 +1038,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
865
1038
|
if (isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
|
|
866
1039
|
// A suite written before pending tests carried their reason gets
|
|
867
1040
|
// it now, deterministically — no regeneration, no AI.
|
|
868
|
-
|
|
1041
|
+
// The same test appended twice (an earlier version did not see test.each) goes, without AI.
|
|
1042
|
+
const normalised = withoutDuplicateTests(framework === 'playwright' ? normalisePending(existing) : existing);
|
|
869
1043
|
// A kept file that leaves rules out or holds tests that cannot fail:
|
|
870
1044
|
// only those tests are written again.
|
|
871
1045
|
const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath, true) : normalised, pending);
|
|
@@ -880,6 +1054,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
880
1054
|
}
|
|
881
1055
|
let content;
|
|
882
1056
|
let status;
|
|
1057
|
+
if (memory && stale.has(filePath) && existsSync(filePath))
|
|
1058
|
+
rememberRefused(pf.slug, ['#whole']);
|
|
883
1059
|
if (useAI) {
|
|
884
1060
|
let draft = drafts.get(filePath);
|
|
885
1061
|
let draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
|
|
@@ -25,9 +25,27 @@ export function findTestBlocks(content) {
|
|
|
25
25
|
const calls = aliases.length
|
|
26
26
|
? new RegExp(`(^|[^\\w.])(?:test|it|${aliases.join('|')})(?:\\.(?:only|fixme|skip|slow|fail))?\\s*\\(\\s*(['"\`])((?:(?!\\2).)*)\\2`, 'gm')
|
|
27
27
|
: TEST_CALL;
|
|
28
|
+
// A parametrised test, test.each(table)('title %s', fn): one block from the
|
|
29
|
+
// table to the call's close. Missed, its rule read as untested and was asked
|
|
30
|
+
// for (and appended) again on every generate.
|
|
31
|
+
const names = ['test', 'it', ...aliases].join('|');
|
|
32
|
+
const each = new RegExp(`(^|[^\\w.])(?:${names})(?:\\.(?:only|skip|concurrent))?\\.each\\s*\\(`, 'gm');
|
|
33
|
+
const hits = [];
|
|
34
|
+
for (const m of content.matchAll(each)) {
|
|
35
|
+
const tableOpen = m.index + m[0].length - 1;
|
|
36
|
+
const tableClose = matchParen(content, tableOpen);
|
|
37
|
+
if (tableClose === -1)
|
|
38
|
+
continue;
|
|
39
|
+
const t = /^\s*\(\s*(['"`])((?:(?!\1).)*)\1/.exec(content.slice(tableClose + 1));
|
|
40
|
+
if (!t)
|
|
41
|
+
continue;
|
|
42
|
+
hits.push({ callAt: m.index + m[1].length, open: content.indexOf('(', tableClose + 1), title: t[2] });
|
|
43
|
+
}
|
|
28
44
|
for (const m of content.matchAll(calls)) {
|
|
29
45
|
const callAt = m.index + m[1].length;
|
|
30
|
-
|
|
46
|
+
hits.push({ callAt, open: content.indexOf('(', callAt), title: m[3] });
|
|
47
|
+
}
|
|
48
|
+
for (const { callAt, open, title } of hits) {
|
|
31
49
|
const close = matchParen(content, open);
|
|
32
50
|
if (close === -1)
|
|
33
51
|
continue;
|
|
@@ -47,10 +65,26 @@ export function findTestBlocks(content) {
|
|
|
47
65
|
}
|
|
48
66
|
const text = content.slice(start, end);
|
|
49
67
|
const rules = [...text.matchAll(RULE_ID)].flatMap((r) => r[1].match(/R\d+/g) ?? []);
|
|
50
|
-
out.push({ start, end, title
|
|
68
|
+
out.push({ start, end, title, rules: [...new Set(rules)] });
|
|
51
69
|
}
|
|
70
|
+
out.sort((a, b) => a.start - b.start);
|
|
52
71
|
return out.filter((b) => !out.some((o) => o !== b && o.start <= b.start && o.end >= b.end && o.end - o.start > b.end - b.start));
|
|
53
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* The file with one test per title: a later test under a title an earlier one
|
|
75
|
+
* already has goes (the same test appended again). No AI.
|
|
76
|
+
*/
|
|
77
|
+
export function withoutDuplicateTests(content) {
|
|
78
|
+
const seen = new Set();
|
|
79
|
+
const extra = findTestBlocks(content).filter((b) => {
|
|
80
|
+
const key = `${b.title}|${b.rules.join(',')}`;
|
|
81
|
+
if (seen.has(key))
|
|
82
|
+
return true;
|
|
83
|
+
seen.add(key);
|
|
84
|
+
return false;
|
|
85
|
+
});
|
|
86
|
+
return extra.length ? spliceBlocks(content, extra.map((block) => ({ block, text: '' }))) : content;
|
|
87
|
+
}
|
|
54
88
|
/** Replace (or, with '', remove) the given blocks; later ones first so offsets stay valid. */
|
|
55
89
|
export function spliceBlocks(content, edits) {
|
|
56
90
|
let out = content;
|
|
@@ -119,3 +153,90 @@ export function renameInBlocks(content, blocks, pairs) {
|
|
|
119
153
|
});
|
|
120
154
|
return { content: spliceBlocks(content, edits), replaced };
|
|
121
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* Tests written when the writer took kit.insert's answer for a row
|
|
158
|
+
* (`const op = await kit.insert(…); op[ID_FIELD]` with ID_FIELD =
|
|
159
|
+
* need('inserted row id field')). kit.insert returns the id itself, so
|
|
160
|
+
* `op[ID_FIELD]` is `op`: rewritten here, the blank name goes, and no AI is
|
|
161
|
+
* asked to write the file again. Only variables assigned from kit.insert.
|
|
162
|
+
*/
|
|
163
|
+
export function useInsertIds(content) {
|
|
164
|
+
const decl = /^[ \t]*(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])([^'"`\n]*\binsert\w*\b[^'"`\n]*\bid\b[^'"`\n]*)\2\s*\);?[ \t]*\n/gim;
|
|
165
|
+
let out = content;
|
|
166
|
+
for (const m of [...content.matchAll(decl)]) {
|
|
167
|
+
const [line, name] = m;
|
|
168
|
+
const fromInsert = new Set([...out.matchAll(/\b([A-Za-z_$][\w$]*)\s*=\s*await\s+kit\.insert\(/g)].map((x) => x[1]));
|
|
169
|
+
const esc = name.replace(/\$/g, '\\$');
|
|
170
|
+
out = out.replace(new RegExp(`\\b([A-Za-z_$][\\w$]*)\\[${esc}\\]`, 'g'), (all, v) => (fromInsert.has(v) ? v : all));
|
|
171
|
+
// The constant goes only when nothing uses it any more.
|
|
172
|
+
if (!new RegExp(`\\b${esc}\\b`).test(out.replace(line, '')))
|
|
173
|
+
out = out.replace(line, '');
|
|
174
|
+
}
|
|
175
|
+
return out;
|
|
176
|
+
}
|
|
177
|
+
/** Index just past the bracket closing the one at `open` ((, { or [), skipping strings, templates and comments; -1 if none. */
|
|
178
|
+
export function closeBracket(src, open) {
|
|
179
|
+
const pairs = { '(': ')', '{': '}', '[': ']' };
|
|
180
|
+
const stack = [];
|
|
181
|
+
for (let i = open; i < src.length; i++) {
|
|
182
|
+
const c = src[i];
|
|
183
|
+
if (c === '/' && src[i + 1] === '/') {
|
|
184
|
+
const n = src.indexOf('\n', i);
|
|
185
|
+
if (n < 0)
|
|
186
|
+
return -1;
|
|
187
|
+
i = n;
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
if (c === '/' && src[i + 1] === '*') {
|
|
191
|
+
const n = src.indexOf('*/', i + 2);
|
|
192
|
+
if (n < 0)
|
|
193
|
+
return -1;
|
|
194
|
+
i = n + 1;
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
if (c === '"' || c === "'" || c === '`') {
|
|
198
|
+
for (i += 1; i < src.length && src[i] !== c; i++)
|
|
199
|
+
if (src[i] === '\\')
|
|
200
|
+
i += 1;
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
if (pairs[c])
|
|
204
|
+
stack.push(pairs[c]);
|
|
205
|
+
else if (c === ')' || c === '}' || c === ']') {
|
|
206
|
+
if (stack.pop() !== c)
|
|
207
|
+
return -1;
|
|
208
|
+
if (!stack.length)
|
|
209
|
+
return i + 1;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return -1;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* The file's setup outside its tests: hooks (beforeAll/beforeEach/afterAll/
|
|
216
|
+
* afterEach) and helper functions (function x() {…}, const x = async () => {…}).
|
|
217
|
+
*/
|
|
218
|
+
export function findSetupBlocks(content) {
|
|
219
|
+
const tests = findTestBlocks(content);
|
|
220
|
+
const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
|
|
221
|
+
const out = [];
|
|
222
|
+
const START = /^[ \t]*(?:(?:test\.)?(beforeAll|beforeEach|afterAll|afterEach)\s*\(|(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*\(|(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)\n]*\)|[A-Za-z_$][\w$]*)\s*=>)/gm;
|
|
223
|
+
for (const m of content.matchAll(START)) {
|
|
224
|
+
const at = m.index;
|
|
225
|
+
if (inTest(at) || out.some((b) => at >= b.start && at < b.end))
|
|
226
|
+
continue;
|
|
227
|
+
let end = -1;
|
|
228
|
+
if (m[1])
|
|
229
|
+
end = closeBracket(content, content.indexOf('(', at));
|
|
230
|
+
else {
|
|
231
|
+
const arrow = m[3] ? content.indexOf('=>', at) : -1;
|
|
232
|
+
const brace = content.indexOf('{', m[2] ? closeBracket(content, content.indexOf('(', at)) : arrow);
|
|
233
|
+
end = brace < 0 ? -1 : closeBracket(content, brace);
|
|
234
|
+
}
|
|
235
|
+
if (end < 0)
|
|
236
|
+
continue;
|
|
237
|
+
if (content[end] === ';')
|
|
238
|
+
end += 1;
|
|
239
|
+
out.push({ start: at, end, name: m[1] ?? m[2] ?? m[3], text: content.slice(at, end) });
|
|
240
|
+
}
|
|
241
|
+
return out;
|
|
242
|
+
}
|