@activate-core/test-forge 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -3
- package/dist/cli/index.js +2 -1
- package/dist/core/docs/excerpts.js +139 -5
- package/dist/core/docs/grounding.js +78 -16
- package/dist/core/docs/reference-text.js +2 -0
- package/dist/core/gates/coverage.js +9 -0
- package/dist/core/gates/validate.js +1 -0
- package/dist/core/generate/ai-generator.js +9 -3
- package/dist/core/generate/ai-usage.js +5 -2
- package/dist/core/generate/complete-file.js +45 -7
- package/dist/core/generate/defect-review.js +2 -1
- package/dist/core/generate/harness.js +34 -0
- package/dist/core/generate/interface-file.js +12 -0
- package/dist/core/generate/journey-memory.js +7 -1
- package/dist/core/generate/passed.js +24 -0
- package/dist/core/generate/run-generate.js +117 -13
- package/dist/core/generate/test-blocks.js +34 -10
- package/dist/core/project/config.js +5 -0
- package/dist/core/prompts/blocks.js +24 -3
- package/dist/core/run/name-scan.js +120 -14
- package/dist/core/run/report.js +7 -4
- package/dist/core/run/run-tests.js +53 -4
- package/package.json +3 -2
- package/reporters/_shared.cjs +56 -3
- package/reporters/interface.cjs +14 -0
- package/reporters/jest-reporter.cjs +7 -3
- package/reporters/playwright-reporter.cjs +6 -2
- package/reporters/run-data.cjs +6 -1
|
@@ -68,3 +68,15 @@ export function recordInterface(found, path = INTERFACE_PATH) {
|
|
|
68
68
|
}
|
|
69
69
|
return blankNames(Object.keys(found), values);
|
|
70
70
|
}
|
|
71
|
+
/** Blank these names again (a value the code no longer has): their tests wait instead of failing for the wrong reason. */
|
|
72
|
+
export function clearInterface(names, path = INTERFACE_PATH) {
|
|
73
|
+
if (!names.length)
|
|
74
|
+
return;
|
|
75
|
+
const values = readInterface(path);
|
|
76
|
+
for (const n of names)
|
|
77
|
+
if (n in values)
|
|
78
|
+
values[n] = '';
|
|
79
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
80
|
+
const sorted = Object.fromEntries(Object.entries(values).sort(([a], [b]) => a.localeCompare(b)));
|
|
81
|
+
writeFileSync(path, `${JSON.stringify(sorted, null, 2)}\n`, 'utf-8');
|
|
82
|
+
}
|
|
@@ -206,6 +206,10 @@ const RUNTIME_HELPERS = {
|
|
|
206
206
|
fillFrameInputs: 'async fillFrameInputs(page, framePrefix, { name: value }) — fills inputs inside an embedded iframe (card fields).',
|
|
207
207
|
watchDialogs: 'watchDialogs(page) → string[] — accepts browser alerts and records their messages.',
|
|
208
208
|
};
|
|
209
|
+
const KIT_CONTRACT = {
|
|
210
|
+
insert: "resolves to the new row's id itself (a number or key, not a row object): const id = await insert('users', {...})",
|
|
211
|
+
update: 'resolves to the number of rows changed',
|
|
212
|
+
};
|
|
209
213
|
export function helperApi(harnessSource) {
|
|
210
214
|
const names = new Set();
|
|
211
215
|
const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(harnessSource);
|
|
@@ -262,7 +266,9 @@ export function helperApi(harnessSource) {
|
|
|
262
266
|
if (text)
|
|
263
267
|
doc.unshift(text);
|
|
264
268
|
}
|
|
265
|
-
|
|
269
|
+
// The scenario-data writers' contract (H10) holds whatever comment sits above them.
|
|
270
|
+
const kit = KIT_CONTRACT[name] && /^(?:async\s+)?function/.test(sig) ? KIT_CONTRACT[name] : doc.join(' ');
|
|
271
|
+
out.push(`- ${sig.length > 200 ? `${sig.slice(0, 197)}…` : sig}${kit ? ` — ${kit}` : ''}`);
|
|
266
272
|
}
|
|
267
273
|
return out.join('\n');
|
|
268
274
|
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — the tests that have passed, as the run recorded them.
|
|
3
|
+
*
|
|
4
|
+
* A test that passed once checked the product for real: generate never
|
|
5
|
+
* removes or rewrites it again. The reporters add to testforge/passed.json
|
|
6
|
+
* after every run ({ "<file relative to the project>": ["title", …] }); a
|
|
7
|
+
* later failure does not take a test off the list — that is a regression to
|
|
8
|
+
* report, not a test to rewrite. `generate --force` is the only way past it.
|
|
9
|
+
*/
|
|
10
|
+
import { readFileSync } from 'node:fs';
|
|
11
|
+
import { join, relative, resolve } from 'node:path';
|
|
12
|
+
import { STATE_DIR } from '../paths.js';
|
|
13
|
+
export const PASSED_PATH = join(STATE_DIR, 'passed.json');
|
|
14
|
+
/** Titles of the tests in `file` that have passed in a recorded run. */
|
|
15
|
+
export function passedTitles(file, path = PASSED_PATH) {
|
|
16
|
+
try {
|
|
17
|
+
const all = JSON.parse(readFileSync(path, 'utf-8'));
|
|
18
|
+
const rel = relative(process.cwd(), resolve(file)).split('\\').join('/');
|
|
19
|
+
return new Set(Array.isArray(all[rel]) ? all[rel] : []);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return new Set();
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* so tests/seeds land where the project expects them. Defaults to
|
|
17
17
|
* generated-tests/ when not configured.
|
|
18
18
|
*/
|
|
19
|
-
import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
|
|
19
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, } from 'node:fs';
|
|
20
20
|
import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
|
|
21
21
|
import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored, journeyDir } from '../paths.js';
|
|
22
22
|
import { lintGenerated } from '../gates/lint.js';
|
|
@@ -27,13 +27,34 @@ import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasPro
|
|
|
27
27
|
import { generateTestCases } from './generator.js';
|
|
28
28
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
29
29
|
import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
|
|
30
|
-
import { completeFile
|
|
30
|
+
import { completeFile } from './complete-file.js';
|
|
31
|
+
import { tuckStrayTests } from './test-blocks.js';
|
|
32
|
+
import { passedTitles } from './passed.js';
|
|
31
33
|
import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
|
|
32
|
-
|
|
34
|
+
/** Names the file asks for that testforge/interface.json still has blank. */
|
|
35
|
+
/** Does this test use a name the product has not given yet (directly, or through the constant it was put in)? */
|
|
36
|
+
function usesBlankName(content, block) {
|
|
37
|
+
const text = content.slice(block.start, block.end);
|
|
38
|
+
return blankNames(content).some((n) => {
|
|
39
|
+
if (text.includes(`'${n}'`) || text.includes(`"${n}"`) || text.includes(`\`${n}\``))
|
|
40
|
+
return true;
|
|
41
|
+
const esc = n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
42
|
+
const v = new RegExp(`\\b(?:const|let|var)\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*need\\(\\s*(['"\`])${esc}\\2`).exec(content)?.[1];
|
|
43
|
+
return !!v && new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`).test(text);
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
function blankNames(content) {
|
|
47
|
+
const iface = readInterface();
|
|
48
|
+
return [...content.matchAll(/\bneed\(\s*(['"`])([^'"`]+)\1\s*\)/g)].map((m) => m[2]).filter((n) => {
|
|
49
|
+
const v = iface[n];
|
|
50
|
+
return typeof v === 'string' ? !v.trim() : v === undefined || v === null;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
import { harnessSteps, ruleCoverage } from '../gates/test-quality.js';
|
|
33
54
|
import { normalisePending } from './pending.js';
|
|
34
55
|
import { packageRoot } from '../run/run-tests.js';
|
|
35
56
|
import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
|
|
36
|
-
import { groundJourney } from '../docs/grounding.js';
|
|
57
|
+
import { factsCodeMoved, writtenWithoutCode, markWrittenWithoutCode, groundJourney } from '../docs/grounding.js';
|
|
37
58
|
import { generateSeed, seedProblem } from './seed-generator.js';
|
|
38
59
|
import { dataAccess, writeRequiredData } from './required-data.js';
|
|
39
60
|
import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
|
|
@@ -452,10 +473,39 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
452
473
|
const p = join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`);
|
|
453
474
|
if (!f.rules?.length || !existsSync(p))
|
|
454
475
|
return true;
|
|
455
|
-
|
|
476
|
+
// Rules left without a test make a suite unfinished. A check that flags an
|
|
477
|
+
// existing test does not: kept tests are reported, never rewritten.
|
|
478
|
+
return !ruleCoverage(readFileSync(p, 'utf-8'), f.rules).missing.length;
|
|
456
479
|
});
|
|
457
480
|
};
|
|
481
|
+
// Tests an earlier version appended after the outer describe go back inside
|
|
482
|
+
// it, so they get the file's setup. No AI, also for an unchanged journey.
|
|
483
|
+
if (existsSync(testDir)) {
|
|
484
|
+
for (const f of readdirSync(testDir)) {
|
|
485
|
+
if (!f.endsWith(ext))
|
|
486
|
+
continue;
|
|
487
|
+
const p = join(testDir, f);
|
|
488
|
+
const before = readFileSync(p, 'utf-8');
|
|
489
|
+
const after = tuckStrayTests(before);
|
|
490
|
+
if (after !== before)
|
|
491
|
+
writeFileSync(p, after, 'utf-8');
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
// Code that arrived after the tests were written: files still asking for
|
|
495
|
+
// names the product did not have are written again from the new code.
|
|
496
|
+
const codeMoved = useAI && (opts.refreshCode || factsCodeMoved(journey, sourceRoot()));
|
|
497
|
+
const testFiles0 = existsSync(testDir) ? readdirSync(testDir).filter((f) => f.endsWith(ext)).map((f) => join(testDir, f)) : [];
|
|
498
|
+
// Tests written while the feature had no code: their "how" was a guess, so
|
|
499
|
+
// the code arriving is a reason to look again even with every name filled.
|
|
500
|
+
const blindBefore = writtenWithoutCode(journey);
|
|
501
|
+
const waitsOnCode = codeMoved && (blindBefore || testFiles0.some((p) => { try {
|
|
502
|
+
return blankNames(readFileSync(p, 'utf-8')).length > 0;
|
|
503
|
+
}
|
|
504
|
+
catch {
|
|
505
|
+
return false;
|
|
506
|
+
} }));
|
|
458
507
|
if (!force &&
|
|
508
|
+
!waitsOnCode &&
|
|
459
509
|
cache[file] === hash &&
|
|
460
510
|
!hasDeferredStubs(testDir, ext) &&
|
|
461
511
|
suiteComplete()) {
|
|
@@ -474,6 +524,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
474
524
|
// A feature written up before it is built: its tests come from the doc
|
|
475
525
|
// alone. There is no code to read for facts and no app to run them on.
|
|
476
526
|
const notBuilt = /^status:\s*not built/im.test(markdown);
|
|
527
|
+
if (notBuilt && useAI)
|
|
528
|
+
markWrittenWithoutCode(journey, sourceRoot());
|
|
477
529
|
// Not live yet (not built, or behind a flag): every file is stamped, and
|
|
478
530
|
// `testforge run` holds it back until the doc's Status line goes.
|
|
479
531
|
const pending = pendingReason(markdown);
|
|
@@ -523,11 +575,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
523
575
|
const written = [];
|
|
524
576
|
// need() names asked so far (the project's and this run's): handed to every later file.
|
|
525
577
|
const askedNames = new Set(Object.keys(readInterface()).filter(isThingName));
|
|
526
|
-
const finish = async (content, pf, filePath) => {
|
|
578
|
+
const finish = async (content, pf, filePath, kept = false) => {
|
|
527
579
|
if (!memory || !pf.rules?.length)
|
|
528
580
|
return content;
|
|
529
581
|
const { mem, facts } = await fileMemory(pf);
|
|
530
582
|
const done = await completeFile(content, {
|
|
583
|
+
kept, locked: passedTitles(filePath),
|
|
584
|
+
redo: kept && stale.has(filePath) ? (redoAll ? () => true : (b) => usesBlankName(content, b)) : undefined,
|
|
531
585
|
journey, path: filePath, rules: pf.rules, memory, helpers: mem?.helpers ?? helpers, facts,
|
|
532
586
|
harnessPath, transport, steps, knownTexts: appTexts(), write: writeTestsWithAI,
|
|
533
587
|
check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
|
|
@@ -635,7 +689,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
635
689
|
// on first use: re-checking an existing suite costs no grounding at all.
|
|
636
690
|
let groundingP;
|
|
637
691
|
const grounding = () => (groundingP ??= useAI && !notBuilt
|
|
638
|
-
? groundJourney(journey, hash, sourceRoot(), markdown).then((g) => g ?? undefined)
|
|
692
|
+
? groundJourney(journey, hash, sourceRoot(), markdown, false, opts.refreshCode).then((g) => g ?? undefined)
|
|
639
693
|
: Promise.resolve(undefined));
|
|
640
694
|
const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
|
|
641
695
|
if (staleRemoved.length) {
|
|
@@ -651,7 +705,40 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
651
705
|
// once. A drafted file that fails its checks is written again on its own,
|
|
652
706
|
// told why; a file the batch left out is written on its own.
|
|
653
707
|
const pathOf = (pf) => join(testDir, `${flat ? `${journey.id}-${pf.slug}` : pf.slug}${ext}`);
|
|
708
|
+
// A file written from older code facts that still waits on names: written again.
|
|
709
|
+
const stale = new Set();
|
|
710
|
+
// Every test of a journey first written with no code is looked at again once its code is there.
|
|
711
|
+
let redoAll = false;
|
|
712
|
+
if (useAI && !notBuilt && blindBefore && testFiles0.length && (await grounding())) {
|
|
713
|
+
for (const p of planned.map(pathOf))
|
|
714
|
+
if (existsSync(p))
|
|
715
|
+
stale.add(p);
|
|
716
|
+
redoAll = stale.size > 0;
|
|
717
|
+
if (redoAll)
|
|
718
|
+
warnings.push(`its code is here now: the tests first written without it are written again from the code (tests that passed are kept)`);
|
|
719
|
+
}
|
|
720
|
+
if (useAI && !notBuilt && !redoAll) {
|
|
721
|
+
const waiting = planned.map(pathOf).filter((p) => { try {
|
|
722
|
+
return existsSync(p) && blankNames(readFileSync(p, 'utf-8')).length > 0;
|
|
723
|
+
}
|
|
724
|
+
catch {
|
|
725
|
+
return false;
|
|
726
|
+
} });
|
|
727
|
+
// Only new code (or --refresh-code) rewrites a waiting file: facts read
|
|
728
|
+
// again from the same code may word things differently, and that alone
|
|
729
|
+
// must never cost a rewrite.
|
|
730
|
+
if (waiting.length && (opts.refreshCode || codeMoved) && (await grounding())) {
|
|
731
|
+
for (const p of waiting)
|
|
732
|
+
stale.add(p);
|
|
733
|
+
if (stale.size)
|
|
734
|
+
warnings.push(`${stale.size} file(s) waiting on names the code now has: only their waiting tests are written again`);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
// A doc without rule ids has no per-test redo: its waiting files are written again whole, as before.
|
|
738
|
+
const rewriteWhole = (p) => stale.has(p) && !memory;
|
|
654
739
|
const keeps = (p) => {
|
|
740
|
+
if (rewriteWhole(p))
|
|
741
|
+
return false;
|
|
655
742
|
if ((force && !redone.has(i)) || !existsSync(p))
|
|
656
743
|
return false;
|
|
657
744
|
try {
|
|
@@ -711,7 +798,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
711
798
|
// midway resumes from exactly the files it hadn't written yet.
|
|
712
799
|
// (--force ignores this and regenerates.)
|
|
713
800
|
// A redo after stubs fills in only the stubs: --force already rewrote the rest once this run.
|
|
714
|
-
if ((!force || redone.has(i)) && existsSync(filePath)) {
|
|
801
|
+
if ((!force || redone.has(i)) && existsSync(filePath) && !rewriteWhole(filePath)) {
|
|
715
802
|
try {
|
|
716
803
|
const existing = readFileSync(filePath, 'utf-8');
|
|
717
804
|
// Keep a real, parsing file — but a DEFERRED stub is not real
|
|
@@ -722,7 +809,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
722
809
|
const normalised = framework === 'playwright' ? normalisePending(existing) : existing;
|
|
723
810
|
// A kept file that leaves rules out or holds tests that cannot fail:
|
|
724
811
|
// only those tests are written again.
|
|
725
|
-
const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath) : normalised, pending);
|
|
812
|
+
const finished = withPendingStamp(useAI ? await finish(normalised, pf, filePath, true) : normalised, pending);
|
|
726
813
|
if (finished !== existing)
|
|
727
814
|
writeFileSync(filePath, finished, 'utf-8');
|
|
728
815
|
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
@@ -735,8 +822,18 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
735
822
|
let content;
|
|
736
823
|
let status;
|
|
737
824
|
if (useAI) {
|
|
738
|
-
|
|
739
|
-
|
|
825
|
+
let draft = drafts.get(filePath);
|
|
826
|
+
let draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
|
|
827
|
+
// A draft that parses but holds a few refused tests: only those tests are
|
|
828
|
+
// written again (the way a kept file is finished), not the whole file.
|
|
829
|
+
if (draft !== undefined && draftRefusal && isSyntaxValid(draft, ext)) {
|
|
830
|
+
const finished = await finish(framework === 'playwright' ? normalisePending(draft) : draft, pf, filePath);
|
|
831
|
+
const still = isSyntaxValid(finished, ext) ? genGate(finished, filePath) : syntaxReason(finished, ext);
|
|
832
|
+
if (!still) {
|
|
833
|
+
draft = finished;
|
|
834
|
+
draftRefusal = undefined;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
740
837
|
const { mem, facts } = await fileMemory(pf);
|
|
741
838
|
const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal && draft !== undefined ? { reason: draftRefusal, previous: draft } : undefined);
|
|
742
839
|
if (ai !== null) {
|
|
@@ -744,6 +841,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
744
841
|
status = 'ai';
|
|
745
842
|
}
|
|
746
843
|
else {
|
|
844
|
+
// The AI could not write it: a real file already there stays as it is,
|
|
845
|
+
// rather than being replaced by a placeholder.
|
|
846
|
+
const existing = existsSync(filePath) ? readFileSync(filePath, 'utf-8') : '';
|
|
847
|
+
if (existing && isSyntaxValid(existing, ext) && !isDeferredStub(existing)) {
|
|
848
|
+
warnings.push(`${pf.slug}: could not be written again — the earlier file is kept`);
|
|
849
|
+
return { slug: pf.slug, path: filePath, status: 'ai' };
|
|
850
|
+
}
|
|
747
851
|
const built = buildTest(journey, false, ext, filePath, framework);
|
|
748
852
|
content = built.content;
|
|
749
853
|
status = 'basic';
|
|
@@ -809,8 +913,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
809
913
|
if (useAI && dataAccess(config, packageRoot(areaDir)) === 'api') {
|
|
810
914
|
writeRequiredData(seedDir, journey.id, journey.name, markdown);
|
|
811
915
|
}
|
|
812
|
-
else if (useAI && !force && existsSync(seedPath)) {
|
|
813
|
-
seedStatus = 'ok'; // kept, like an existing test file
|
|
916
|
+
else if (useAI && !force && existsSync(seedPath) && !redoAll) {
|
|
917
|
+
seedStatus = 'ok'; // kept, like an existing test file — unless it was written before the code existed
|
|
814
918
|
}
|
|
815
919
|
else if (useAI && pending) {
|
|
816
920
|
// Not live yet: there is nothing to seed for, and its tests make their own data.
|
|
@@ -66,18 +66,42 @@ export function spliceBlocks(content, edits) {
|
|
|
66
66
|
}
|
|
67
67
|
/** Add tests inside the file's outermost describe, before it closes; at the end when there is none. */
|
|
68
68
|
export function appendTests(content, tests) {
|
|
69
|
-
const d =
|
|
70
|
-
if (d)
|
|
71
|
-
|
|
72
|
-
if (close !== -1) {
|
|
73
|
-
// The describe's body closes with "}" just before ")".
|
|
74
|
-
const brace = content.lastIndexOf('}', close);
|
|
75
|
-
if (brace > d.index)
|
|
76
|
-
return `${content.slice(0, brace).trimEnd()}\n\n${indent(tests.trim(), ' ')}\n${content.slice(brace)}`;
|
|
77
|
-
}
|
|
78
|
-
}
|
|
69
|
+
const d = outerDescribe(content);
|
|
70
|
+
if (d)
|
|
71
|
+
return `${content.slice(0, d.brace).trimEnd()}\n\n${indent(tests.trim(), ' ')}\n${content.slice(d.brace)}`;
|
|
79
72
|
return `${content.trimEnd()}\n\n${tests.trim()}\n`;
|
|
80
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* The file's outer describe (Playwright's test.describe or Jest's describe):
|
|
76
|
+
* where it starts, the "}" closing its body and the ")" closing the call.
|
|
77
|
+
*/
|
|
78
|
+
function outerDescribe(content) {
|
|
79
|
+
const d = /^(?:test\.)?describe(?:\.\w+)?\s*\(/m.exec(content) ?? /\b(?:test\.)?describe(?:\.\w+)?\s*\(/.exec(content);
|
|
80
|
+
if (!d)
|
|
81
|
+
return null;
|
|
82
|
+
const close = matchParen(content, content.indexOf('(', d.index));
|
|
83
|
+
if (close === -1)
|
|
84
|
+
return null;
|
|
85
|
+
// The describe's body closes with "}" just before ")".
|
|
86
|
+
const brace = content.lastIndexOf('}', close);
|
|
87
|
+
return brace > d.index ? { start: d.index, brace, close } : null;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Tests that ended up after the outer describe (earlier versions appended
|
|
91
|
+
* them there) are moved inside it, so they get the file's setup and cleanup.
|
|
92
|
+
* No AI; a file with nothing outside is returned as it is.
|
|
93
|
+
*/
|
|
94
|
+
export function tuckStrayTests(content) {
|
|
95
|
+
const d = outerDescribe(content);
|
|
96
|
+
if (!d)
|
|
97
|
+
return content;
|
|
98
|
+
const stray = findTestBlocks(content).filter((b) => b.start > d.close);
|
|
99
|
+
if (!stray.length)
|
|
100
|
+
return content;
|
|
101
|
+
const moved = stray.map((b) => content.slice(b.start, b.end).trim()).join('\n\n');
|
|
102
|
+
const rest = spliceBlocks(content, stray.map((block) => ({ block, text: '' }))).replace(/\n{3,}/g, '\n\n');
|
|
103
|
+
return appendTests(rest, moved);
|
|
104
|
+
}
|
|
81
105
|
const indent = (t, pad) => t.split('\n').map((l) => (l ? pad + l : l)).join('\n');
|
|
82
106
|
/** Literal replacement of each old text with the new one, only inside the given blocks. */
|
|
83
107
|
export function renameInBlocks(content, blocks, pairs) {
|
|
@@ -282,6 +282,11 @@ export function loadEnvFile(path) {
|
|
|
282
282
|
(value.startsWith("'") && value.endsWith("'"))) {
|
|
283
283
|
value = value.slice(1, -1);
|
|
284
284
|
}
|
|
285
|
+
else {
|
|
286
|
+
// An unquoted value ends at a comment: `KEY= # hint` (the template's
|
|
287
|
+
// unfilled line) is empty, not the hint.
|
|
288
|
+
value = value.replace(/(^|\s)#.*$/, '').trim();
|
|
289
|
+
}
|
|
285
290
|
out[key] = value;
|
|
286
291
|
}
|
|
287
292
|
return out;
|
|
@@ -110,7 +110,10 @@ FORMAT
|
|
|
110
110
|
"## Test setup", which lists what the environment must provide (accounts and
|
|
111
111
|
records to seed, flags to turn on, external services to stub with their
|
|
112
112
|
endpoint or SDK, browser dialogs), written as instructions to the test
|
|
113
|
-
generator
|
|
113
|
+
generator. Name every request the tests send by its method and path
|
|
114
|
+
(POST /api/v2/booking-order), even a booking or payment the journey only
|
|
115
|
+
passes through: only a route named here has its handler and validation
|
|
116
|
+
rules read for the tests;
|
|
114
117
|
"## Questions for product (not asserted)", which holds only what neither the
|
|
115
118
|
reference nor the code settles. Keep it short.
|
|
116
119
|
- Do not write a Coverage table. It is generated from the [C<ids>] on each
|
|
@@ -174,8 +177,15 @@ E2. What you touch comes from <code_facts> and <testkit>.
|
|
|
174
177
|
any version prefix, and use the same prefix throughout the file.
|
|
175
178
|
- A response field comes from the serializer for that endpoint in
|
|
176
179
|
<code_facts>, not from the types of another endpoint.
|
|
177
|
-
- A
|
|
178
|
-
|
|
180
|
+
- A request is sent with the fields its handler validates, as SHAPES in
|
|
181
|
+
<code_facts> lists them, nested as written (address.latitude is
|
|
182
|
+
{ address: { latitude } }). A column, table or status value comes from DATA.
|
|
183
|
+
- Look in ROUTES, SHAPES and DATA before need(): a name any of them shows is
|
|
184
|
+
written as it is there. need() is only for a name none of them has (see
|
|
185
|
+
NAMES). Never guess one.
|
|
186
|
+
- Never need() what the testkit already gives: kit.insert returns the new
|
|
187
|
+
row's id; a second person of a role (another operator, another customer)
|
|
188
|
+
is a row made with kit.insert; the roles are the ones <testkit> lists.
|
|
179
189
|
- Call testkit helpers exactly as <testkit> lists them, and never write your
|
|
180
190
|
own copy.
|
|
181
191
|
|
|
@@ -306,6 +316,17 @@ export const API = `API (Jest)
|
|
|
306
316
|
token, the wrong role, and a different owner.
|
|
307
317
|
- Check stored state only through testkit resolvers or helpers. Never open
|
|
308
318
|
your own database connection.
|
|
319
|
+
- Build the data a scenario describes (operators and their locations,
|
|
320
|
+
services, hours and earlier jobs; a setting switched on or off) with
|
|
321
|
+
kit.insert(table, row) and kit.update(table, match, values), using the
|
|
322
|
+
table and column names from <code_facts>. Never ask for a URL or command to
|
|
323
|
+
seed, rank or switch something: data is written, and the outcome is read
|
|
324
|
+
from the product's own endpoints (the operator a booking gets, a status).
|
|
325
|
+
- A value the product works out from other records (a standing, an average
|
|
326
|
+
per day) is not a column: set it with the testkit's _tf_ column from its
|
|
327
|
+
DERIVED export. Where the product picks one record among many (the nearest,
|
|
328
|
+
the best), build the scenario at kit.AREA so nothing already in the
|
|
329
|
+
database takes part.
|
|
309
330
|
- Keep tests hermetic. Every external service listed under EXTERNALS in
|
|
310
331
|
<code_facts> is stood in for:
|
|
311
332
|
- when the server runs inside the test process, with jest.mock on the exact
|
|
@@ -16,8 +16,10 @@ import { dirname, join, relative } from 'node:path';
|
|
|
16
16
|
import { STATE_DIR } from '../paths.js';
|
|
17
17
|
import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
18
18
|
import { runAiSync } from '../generate/ai-usage.js';
|
|
19
|
-
import { readInterface, recordInterface } from '../generate/interface-file.js';
|
|
19
|
+
import { clearInterface, readInterface, recordInterface } from '../generate/interface-file.js';
|
|
20
20
|
const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
|
|
21
|
+
/** Values the scan filled (name → value), so each run can check the code still has them. */
|
|
22
|
+
const FILLED_PATH = join(STATE_DIR, 'interface-filled.json');
|
|
21
23
|
const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
|
|
22
24
|
const CODE = /\.(php|ts|tsx|js|jsx|mjs|cjs|py|rb|go|vue|svelte|prisma|sql)$/;
|
|
23
25
|
const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls', 'item', 'items', 'menu', 'link', 'entry', 'icon', 'text', 'title', 'page']);
|
|
@@ -26,11 +28,14 @@ export function kindOf(name) {
|
|
|
26
28
|
const n = name.trim().toLowerCase();
|
|
27
29
|
if (/\b(?:url|urls|path|prefix|endpoint|route)$/.test(n))
|
|
28
30
|
return 'route';
|
|
31
|
+
// The HTTP method of a request lives on the route line (Route::post, router.post, @Post).
|
|
32
|
+
if (/\bmethod$/.test(n))
|
|
33
|
+
return 'method';
|
|
29
34
|
if (/\bcommand$/.test(n))
|
|
30
35
|
return 'command';
|
|
31
|
-
if (/\b(?:table|column|model)$/.test(n))
|
|
36
|
+
if (/\b(?:table|column|model|class)$/.test(n))
|
|
32
37
|
return 'schema';
|
|
33
|
-
if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|setting)$/.test(n))
|
|
38
|
+
if (/\b(?:field|fields|key|param|parameter|parameters|header|status value|value name|setting|role|permission|event|queue|job)$/.test(n))
|
|
34
39
|
return 'field';
|
|
35
40
|
if (/\blabel$/.test(n))
|
|
36
41
|
return 'label';
|
|
@@ -39,28 +44,32 @@ export function kindOf(name) {
|
|
|
39
44
|
/** Files of a kind: a route file, a console command, a migration or model, a resource or controller, a component. */
|
|
40
45
|
function fileMatches(kind, rel) {
|
|
41
46
|
switch (kind) {
|
|
42
|
-
case 'route':
|
|
47
|
+
case 'route':
|
|
48
|
+
case 'method': return /(^|\/)(routes?|router|urls)(\/|\.[a-z]+$)|\.routes?\.[a-z]+$|\/routes?\.[a-z]+$|(^|\/)app\/.*\/(page|route)\.[jt]sx?$|\.controller\.[jt]s$/i.test(rel);
|
|
43
49
|
case 'command': return /(^|\/)(console|commands?|cli|bin)(\/|$)|command/i.test(rel);
|
|
44
50
|
case 'schema': return /(^|\/)(migrations?|models?|entities|schema)(\/|\.)|\.prisma$|\.sql$/i.test(rel);
|
|
45
|
-
|
|
51
|
+
// A request's fields are where it is validated (FormRequests, rule and
|
|
52
|
+
// logic classes), a status value where its enum or model constant is.
|
|
53
|
+
case 'field': return /(^|\/)(resources?|transformers?|serializers?|controllers?|dto|types?|api|requests?|validators?|validation|rules?|logic|services?|models?|enums?|constants?|schemas?)(\/|\.)|resource|serializer|controller|request|validat|enum/i.test(rel);
|
|
46
54
|
case 'label': return /\.(tsx|jsx|vue|svelte)$/.test(rel);
|
|
47
55
|
}
|
|
48
56
|
}
|
|
49
57
|
/** What a line that DEFINES a name of each kind looks like. */
|
|
50
58
|
const SHAPE = {
|
|
51
59
|
// …or a URL the app keeps as a constant (faq: 'https://…').
|
|
60
|
+
method: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|export\s+(?:async\s+)?function\s+(?:GET|POST|PUT|PATCH|DELETE)\b/,
|
|
52
61
|
route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]|\b\w+\s*:\s*['"`]https?:\/\//,
|
|
53
62
|
command: /\$signature\s*=|\$name\s*=\s*['"]|->command\(\s*['"]|\.command\(\s*['"]|add_parser\(|@click\.command/,
|
|
54
63
|
schema: /Schema::(?:create|table)\(|\$table->\w+\(\s*['"]|create_table|\bclass\s+\w+\s+extends\s+\w*Model\b|protected\s+\$(?:table|fillable)|\bmodel\s+\w+\s*\{|@(?:Entity|Column|Table)\b/,
|
|
55
|
-
field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]
|
|
64
|
+
field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]|^\s*case\s+\w+\s*=|\bconst\s+[A-Z_]+\s*=/,
|
|
56
65
|
// …or a menu entry's label (label: 'Orders').
|
|
57
66
|
// …or a JSX text child on a line of its own (<Button>\n Escalate\n</Button>).
|
|
58
67
|
label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]|\blabel\s*:\s*['"`]|^\s*[A-Z][\w'’ ]{1,40}\s*$|\/>\s*[A-Z][\w'’ ]{1,40}\s*$/,
|
|
59
68
|
};
|
|
60
69
|
/** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
|
|
61
70
|
function candidateLines(name, kind, files, root) {
|
|
62
|
-
const words = [...new Set(name.toLowerCase().match(/[a-z]{
|
|
63
|
-
.filter((w) => !GENERIC.has(w))
|
|
71
|
+
const words = [...new Set(name.toLowerCase().match(/[a-z]{2,}/g) ?? [])]
|
|
72
|
+
.filter((w) => !GENERIC.has(w) && (w.length >= 3 || w === 'id'))
|
|
64
73
|
.map((w) => (w.length > 5 ? w.replace(/(?:ings?|ed|es|s)$/, '') : w));
|
|
65
74
|
if (!words.length)
|
|
66
75
|
return [];
|
|
@@ -83,19 +92,46 @@ function candidateLines(name, kind, files, root) {
|
|
|
83
92
|
lines.forEach((l, i) => {
|
|
84
93
|
if (/^\s*(?:\/\/|#|\*)/.test(l) || l.length > 300)
|
|
85
94
|
return;
|
|
95
|
+
// Log and console lines name things in prose, not as the product's keys.
|
|
96
|
+
if (/\b(?:Log::|logger\.|console\.|->(?:info|error|warning|debug)\()/.test(l))
|
|
97
|
+
return;
|
|
86
98
|
const low = l.toLowerCase();
|
|
87
|
-
|
|
99
|
+
// Whole identifier parts, not substrings: "order" is not in "orderBy",
|
|
100
|
+
// "out" is not in "Route"; address_id and addressId both give "address" and "id".
|
|
101
|
+
const parts = new Set(l.replace(/([a-z0-9])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
|
|
102
|
+
const has = (w) => parts.has(w) || [...parts].some((p) => p.length > 3 && (p.startsWith(w) || w.startsWith(p)) && Math.abs(p.length - w.length) <= 3);
|
|
103
|
+
if ((kind === 'route' || kind === 'method') && /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>/.test(l))
|
|
88
104
|
prefixes.push(`${rel}:${i + 1}: ${l.trim()}`);
|
|
89
105
|
if (!SHAPE[kind].test(l))
|
|
90
106
|
return;
|
|
91
|
-
const score = words.filter((w) =>
|
|
107
|
+
const score = words.filter((w) => has(w)).length;
|
|
92
108
|
if (score >= need)
|
|
93
|
-
hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score });
|
|
109
|
+
hits.push({ line: `${rel}:${i + 1}: ${l.trim()}`, score, file: rel, at: i, lines });
|
|
94
110
|
});
|
|
95
111
|
}
|
|
96
112
|
if (!hits.length)
|
|
97
113
|
return [];
|
|
98
|
-
const
|
|
114
|
+
const top = hits.sort((a, b) => b.score - a.score).slice(0, 8);
|
|
115
|
+
const best = top.map((h) => h.line);
|
|
116
|
+
if (kind === 'route' || kind === 'method') {
|
|
117
|
+
// The prefixes of the groups each route sits in (by indentation), not
|
|
118
|
+
// whichever prefixes come first in its file.
|
|
119
|
+
const PREFIX = /->prefix\(|::prefix\(|\bprefix\s*[:=]|['"]prefix['"]\s*=>|Route::group\(/;
|
|
120
|
+
const indent = (l) => l.length - l.trimStart().length;
|
|
121
|
+
const enclosing = [];
|
|
122
|
+
for (const h of top) {
|
|
123
|
+
let cur = indent(h.lines[h.at]);
|
|
124
|
+
for (let j = h.at - 1; j >= 0 && cur > 0; j--) {
|
|
125
|
+
const l = h.lines[j];
|
|
126
|
+
if (!l.trim() || indent(l) >= cur)
|
|
127
|
+
continue;
|
|
128
|
+
cur = indent(l);
|
|
129
|
+
if (PREFIX.test(l))
|
|
130
|
+
enclosing.push(`${h.file}:${j + 1}: ${l.trim()}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return [...best, ...new Set(enclosing)].slice(0, 16);
|
|
134
|
+
}
|
|
99
135
|
// A route's full path needs the prefix of the group it sits in: those of its own files ride along.
|
|
100
136
|
const files_ = new Set(best.map((l) => l.split(':')[0]));
|
|
101
137
|
return [...best, ...prefixes.filter((p) => files_.has(p.split(':')[0])).slice(0, 6)];
|
|
@@ -140,6 +176,17 @@ export function foundInCode(value, lines) {
|
|
|
140
176
|
const parts = v.split('/').filter((p) => p && p !== 'api' && !/^[{:<]/.test(p));
|
|
141
177
|
return parts.length > 0 && parts.every((p) => text.includes(p));
|
|
142
178
|
}
|
|
179
|
+
// An HTTP method: Route::post / router.post / @Post / export function POST.
|
|
180
|
+
if (/^(?:get|post|put|patch|delete)$/i.test(v))
|
|
181
|
+
return new RegExp(`\\b${v}\\b`, 'i').test(text);
|
|
182
|
+
// A nested response field (error.message): each part is a key in the code.
|
|
183
|
+
if (/^[\w-]+(?:\.[\w-]+)+$/.test(v) && !/^\d/.test(v))
|
|
184
|
+
return v.split('.').every((p) => text.includes(p));
|
|
185
|
+
// A short or numeric value ("80", "on") must stand as a value, not inside another word.
|
|
186
|
+
if (v.length < 4 || /^\d+(?:\.\d+)?$/.test(v)) {
|
|
187
|
+
const esc = v.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
188
|
+
return new RegExp(`(['"\`])${esc}\\1|(?:=>|=|:)\\s*${esc}\\b`).test(text);
|
|
189
|
+
}
|
|
143
190
|
return text.includes(v);
|
|
144
191
|
}
|
|
145
192
|
/**
|
|
@@ -202,7 +249,9 @@ export function scanNames(blank, root = '.') {
|
|
|
202
249
|
'- A URL is the path a client calls: it starts with "/" and includes every',
|
|
203
250
|
' route prefix the lines show (a route inside a prefix group gets the prefix).',
|
|
204
251
|
'- A command is the command name a person types.',
|
|
205
|
-
'- A field, key or column is the exact key.',
|
|
252
|
+
'- A field, key or column is the exact key. A field inside an object is its path with dots (error.message).',
|
|
253
|
+
'- A request method is the HTTP verb in capitals (POST).',
|
|
254
|
+
'- A status or enum value is the stored value, not the constant\'s name.',
|
|
206
255
|
'If no line clearly is that thing, the value is null.',
|
|
207
256
|
...(known.length ? ['', '<filled_examples>', ...known.map(([k, v]) => `- ${k}: ${v}`), '</filled_examples>', 'These values are already filled; follow their format.'] : []),
|
|
208
257
|
'',
|
|
@@ -227,8 +276,22 @@ export function scanNames(blank, root = '.') {
|
|
|
227
276
|
keep[a.name] = v.trim();
|
|
228
277
|
}
|
|
229
278
|
recordInterface(keep);
|
|
230
|
-
for (const [name, value] of Object.entries(keep))
|
|
279
|
+
for (const [name, value] of Object.entries(keep)) {
|
|
231
280
|
filled.push({ name, value });
|
|
281
|
+
// Asked again if someone blanks it later, even with the same lines.
|
|
282
|
+
memo[name] = `filled:${memo[name]}`;
|
|
283
|
+
}
|
|
284
|
+
try {
|
|
285
|
+
const prev = JSON.parse(readFileSync(FILLED_PATH, 'utf-8'));
|
|
286
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify({ ...prev, ...keep }, null, 2)}\n`, 'utf-8');
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
try {
|
|
290
|
+
mkdirSync(dirname(FILLED_PATH), { recursive: true });
|
|
291
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify(keep, null, 2)}\n`, 'utf-8');
|
|
292
|
+
}
|
|
293
|
+
catch { /* best effort */ }
|
|
294
|
+
}
|
|
232
295
|
}
|
|
233
296
|
catch (e) {
|
|
234
297
|
console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
|
|
@@ -246,3 +309,46 @@ export function scanNames(blank, root = '.') {
|
|
|
246
309
|
const done = new Set(filled.map((f) => f.name));
|
|
247
310
|
return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
|
|
248
311
|
}
|
|
312
|
+
/**
|
|
313
|
+
* Values the scan filled earlier that the code no longer has (a route or field
|
|
314
|
+
* renamed): blanked again, so their tests report "waiting" rather than failing
|
|
315
|
+
* as if the product were wrong, and the next scan looks for the new name. No AI.
|
|
316
|
+
*/
|
|
317
|
+
export function recheckFilled(root = '.') {
|
|
318
|
+
let filled;
|
|
319
|
+
try {
|
|
320
|
+
filled = JSON.parse(readFileSync(FILLED_PATH, 'utf-8'));
|
|
321
|
+
}
|
|
322
|
+
catch {
|
|
323
|
+
return [];
|
|
324
|
+
}
|
|
325
|
+
const current = readInterface();
|
|
326
|
+
const files = codeFiles(root);
|
|
327
|
+
const gone = [];
|
|
328
|
+
for (const [name, value] of Object.entries(filled)) {
|
|
329
|
+
if (String(current[name] ?? '').trim() !== value)
|
|
330
|
+
continue; // changed or set by hand: not ours to judge
|
|
331
|
+
const kind = kindOf(name);
|
|
332
|
+
if (!kind)
|
|
333
|
+
continue;
|
|
334
|
+
const lines = files.filter((f) => fileMatches(kind, relative(root, f))).flatMap((f) => {
|
|
335
|
+
try {
|
|
336
|
+
return readFileSync(f, 'utf-8').split('\n');
|
|
337
|
+
}
|
|
338
|
+
catch {
|
|
339
|
+
return [];
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
if (!foundInCode(value, lines))
|
|
343
|
+
gone.push(name);
|
|
344
|
+
}
|
|
345
|
+
if (gone.length) {
|
|
346
|
+
clearInterface(gone);
|
|
347
|
+
const left = Object.fromEntries(Object.entries(filled).filter(([n]) => !gone.includes(n)));
|
|
348
|
+
try {
|
|
349
|
+
writeFileSync(FILLED_PATH, `${JSON.stringify(left, null, 2)}\n`, 'utf-8');
|
|
350
|
+
}
|
|
351
|
+
catch { /* best effort */ }
|
|
352
|
+
}
|
|
353
|
+
return gone;
|
|
354
|
+
}
|