@activate-core/test-forge 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +24 -3
- package/dist/cli/index.js +2 -1
- package/dist/core/docs/excerpts.js +186 -6
- package/dist/core/docs/grounding.js +82 -16
- package/dist/core/docs/reference-text.js +2 -0
- package/dist/core/gates/coverage.js +9 -0
- package/dist/core/gates/validate.js +50 -23
- package/dist/core/generate/ai-generator.js +9 -3
- package/dist/core/generate/ai-usage.js +5 -2
- package/dist/core/generate/complete-file.js +70 -10
- package/dist/core/generate/defect-review.js +2 -1
- package/dist/core/generate/harness.js +34 -0
- package/dist/core/generate/interface-file.js +12 -0
- package/dist/core/generate/journey-memory.js +7 -1
- package/dist/core/generate/passed.js +24 -0
- package/dist/core/generate/run-generate.js +201 -20
- package/dist/core/generate/test-blocks.js +34 -10
- package/dist/core/project/config.js +5 -0
- package/dist/core/project/env-diff.js +3 -1
- package/dist/core/project/setup.js +10 -2
- package/dist/core/prompts/blocks.js +37 -8
- package/dist/core/run/name-scan.js +178 -17
- package/dist/core/run/report.js +9 -4
- package/dist/core/run/run-tests.js +58 -4
- package/package.json +5 -3
- package/reporters/_shared.cjs +56 -3
- package/reporters/interface.cjs +40 -3
- package/reporters/jest-reporter.cjs +7 -3
- package/reporters/playwright-reporter.cjs +6 -2
- package/reporters/run-data.cjs +6 -1
|
@@ -51,10 +51,11 @@ export const DEFAULT_EXTERNAL_MODULES = [
|
|
|
51
51
|
* clean. Runs AFTER syntax validation.
|
|
52
52
|
*/
|
|
53
53
|
export function checkTestGates(content, _ext, ctx) {
|
|
54
|
+
const reasons = [];
|
|
54
55
|
// F3 — runner correctness: a Jest/backend area must never pull in
|
|
55
56
|
// Playwright (not installed there; a top-level import breaks the suite).
|
|
56
57
|
if (ctx.framework === 'jest' && /@playwright\/test/.test(content)) {
|
|
57
|
-
|
|
58
|
+
reasons.push("jest area must not import '@playwright/test'");
|
|
58
59
|
}
|
|
59
60
|
// F4 — goto guard: `page.goto(...)` must take a real route/URL, not a
|
|
60
61
|
// pasted description and not an unfilled route-file placeholder. Literal
|
|
@@ -67,10 +68,10 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
67
68
|
while ((m = gotoRe.exec(content)) !== null) {
|
|
68
69
|
const arg = m[2];
|
|
69
70
|
if (/\s/.test(arg)) {
|
|
70
|
-
|
|
71
|
+
reasons.push(`goto() argument looks like prose, not a URL/path: "${arg.slice(0, 40)}…"`);
|
|
71
72
|
}
|
|
72
73
|
if (/\[[A-Za-z_]|\/:[A-Za-z_]|\{[A-Za-z_]|<[A-Za-z_]/.test(arg)) {
|
|
73
|
-
|
|
74
|
+
reasons.push(`goto() has an unresolved route placeholder — fill it with a real value: "${arg.slice(0, 40)}"`);
|
|
74
75
|
}
|
|
75
76
|
}
|
|
76
77
|
// F13 — a swallowed action must be bounded. `.click({ force: true })`
|
|
@@ -85,7 +86,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
85
86
|
let sw;
|
|
86
87
|
while ((sw = swallowed.exec(content)) !== null) {
|
|
87
88
|
if (!/timeout\s*:/.test(sw[0])) {
|
|
88
|
-
|
|
89
|
+
reasons.push('an action whose failure is swallowed with .catch() has no timeout — ' +
|
|
89
90
|
'it will wait until the whole test times out and blame a later line');
|
|
90
91
|
}
|
|
91
92
|
}
|
|
@@ -96,7 +97,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
96
97
|
// that never comes. Hold the promise first, then act, then await it.
|
|
97
98
|
const lateWait = /\b(?:await\s+)?page\.goto\([^\n]*\);?\s*\n(?:\s*\/\/[^\n]*\n)*\s*await\s+page\.waitFor(?:Response|Request|Event)\s*\(/;
|
|
98
99
|
if (lateWait.test(content)) {
|
|
99
|
-
|
|
100
|
+
reasons.push('page.waitForResponse/Request is armed AFTER the navigation that triggers ' +
|
|
100
101
|
'it — hold the promise first, then navigate, then await it');
|
|
101
102
|
}
|
|
102
103
|
// F2 — no silent false-green: a file with RUNNABLE tests must assert
|
|
@@ -108,7 +109,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
108
109
|
const assertions = (content.match(/\bexpect\s*\(/g) || []).length +
|
|
109
110
|
(content.match(/\bassert\b/g) || []).length;
|
|
110
111
|
if (activeTests > 0 && assertions === 0) {
|
|
111
|
-
|
|
112
|
+
reasons.push('has runnable test(s) but no assertions — would pass without testing anything');
|
|
112
113
|
}
|
|
113
114
|
// F1 — dependency awareness: reject imports/requires of a module the
|
|
114
115
|
// project can't resolve (bare specifier not in allowedModules).
|
|
@@ -116,7 +117,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
116
117
|
for (const mod of importedModules(content)) {
|
|
117
118
|
const top = topLevelModule(mod);
|
|
118
119
|
if (!ctx.allowedModules.has(top)) {
|
|
119
|
-
|
|
120
|
+
reasons.push(`imports '${mod}' which the project does not have installed`);
|
|
120
121
|
}
|
|
121
122
|
}
|
|
122
123
|
}
|
|
@@ -139,7 +140,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
139
140
|
content.includes(`jest.mock('${top}'`) ||
|
|
140
141
|
content.includes(`jest.mock("${top}"`);
|
|
141
142
|
if (!mocked) {
|
|
142
|
-
|
|
143
|
+
reasons.push(`imports external service '${top}' without a jest.mock — the test would hit the live service; mock it at the seam`);
|
|
143
144
|
}
|
|
144
145
|
}
|
|
145
146
|
}
|
|
@@ -160,7 +161,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
160
161
|
const showedWork = /\b(?:port|origin|path\s*prefix|prefix|resource\s*type|header|predicate)\b/i;
|
|
161
162
|
const m = content.match(sameHost);
|
|
162
163
|
if (m && !showedWork.test(m[0])) {
|
|
163
|
-
|
|
164
|
+
reasons.push('a test is skipped because the dependency "shares a host" with the app, ' +
|
|
164
165
|
'without saying what else was checked — a different port is a different ' +
|
|
165
166
|
'origin, and a path prefix or a request predicate usually separates them ' +
|
|
166
167
|
'too; scope the stub to one of those, or name the ones you ruled out');
|
|
@@ -169,20 +170,20 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
169
170
|
// F16 — no placeholders in a freshly written file: a placeholder never runs,
|
|
170
171
|
// and a suite full of them reads as coverage it does not have.
|
|
171
172
|
if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
|
|
172
|
-
|
|
173
|
+
reasons.push('placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file');
|
|
173
174
|
}
|
|
174
175
|
// F23 — need() asks for the name of a thing, anywhere in the file (a constant at
|
|
175
176
|
// the top too), never for data the test should make itself.
|
|
176
177
|
if (ctx.forbidPlaceholders) {
|
|
177
178
|
const data = [...content.matchAll(/\bneed\(\s*(['"`])([^'"`\n]+)\1/g)].map((m) => m[2]).filter((n) => !isThingName(n));
|
|
178
179
|
if (data.length) {
|
|
179
|
-
|
|
180
|
+
reasons.push(`asks need() for ${[...new Set(data)].slice(0, 3).map((n) => `"${n}"`).join(', ')}, which is data, not the name of a thing: the test makes its own ids, payloads and records; need() is only for "… URL", "… command", "… table", "… field", "… label"`);
|
|
180
181
|
}
|
|
181
182
|
}
|
|
182
183
|
// F22 — a fresh file exists to test its rules: one with no test at all says
|
|
183
184
|
// nothing, and passes. It goes back once, told to stub what it cannot create.
|
|
184
185
|
if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
|
|
185
|
-
|
|
186
|
+
reasons.push('the file has no test: every rule it was given gets one — a state the test cannot create is stubbed with page.route(), never left out');
|
|
186
187
|
}
|
|
187
188
|
// F24 — one comment policy: the header, an oracle line above each test,
|
|
188
189
|
// cannot-test lines, a timeout reason, and a short note on a parametrised
|
|
@@ -190,7 +191,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
190
191
|
if (ctx.forbidPlaceholders) {
|
|
191
192
|
const stray = strayComment(content);
|
|
192
193
|
if (stray) {
|
|
193
|
-
|
|
194
|
+
reasons.push(`comment not allowed (line ${stray.line}): "${stray.text.slice(0, 60)}" — the only comments are the // @generated header, one // oracle: line above each test, // cannot-test: lines, a // timeout: reason, and a one-line note on a parametrised row`);
|
|
194
195
|
}
|
|
195
196
|
}
|
|
196
197
|
// F21 — a run-time skip is for what the ENVIRONMENT lacks (an account, a
|
|
@@ -200,27 +201,27 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
200
201
|
if (ctx.forbidPlaceholders) {
|
|
201
202
|
const why = elementSkips(content);
|
|
202
203
|
if (why.length) {
|
|
203
|
-
|
|
204
|
+
reasons.push(`a test skips itself because an element is not on the page (${why[0]}): a missing element the doc names is a failure — assert it instead. Skip only for an account, a variable or data the environment lacks`);
|
|
204
205
|
}
|
|
205
206
|
}
|
|
206
207
|
// F17 — relative imports resolve from where the file is written.
|
|
207
208
|
if (ctx.filePath) {
|
|
208
209
|
const missing = unresolvedImports(content, ctx.filePath);
|
|
209
210
|
if (missing.length) {
|
|
210
|
-
|
|
211
|
+
reasons.push(`relative import(s) reach no file from ${ctx.filePath}: ${missing.join(', ')}`);
|
|
211
212
|
}
|
|
212
213
|
}
|
|
213
214
|
// F18 — no fixed sleeps. A wait for a set time either wastes it or is too
|
|
214
215
|
// short on a slower machine; wait for the event or the state instead.
|
|
215
216
|
if (ctx.framework === 'playwright' && ctx.forbidPlaceholders && /\bwaitForTimeout\s*\(/.test(content)) {
|
|
216
|
-
|
|
217
|
+
reasons.push('page.waitForTimeout() is not used: wait for the element, response or URL the test needs (expect(...).toBeVisible(), page.waitForResponse(), expect(page).toHaveURL())');
|
|
217
218
|
}
|
|
218
219
|
// F19 — a long test timeout needs a reason. A 240 s budget hides a test that
|
|
219
220
|
// is stuck; the cap makes it fail early with its real error.
|
|
220
221
|
if (ctx.maxTestTimeoutMs) {
|
|
221
222
|
const long = longTimeoutsWithoutReason(content, ctx.maxTestTimeoutMs);
|
|
222
223
|
if (long.length) {
|
|
223
|
-
|
|
224
|
+
reasons.push(`test.setTimeout(${long[0]}) is above ${ctx.maxTestTimeoutMs} ms with no "// timeout: <why>" comment: wait for what the test needs instead, or state why it takes that long`);
|
|
224
225
|
}
|
|
225
226
|
}
|
|
226
227
|
// F20 — one criterion, a few tests. Thirty inputs against a rule that only
|
|
@@ -229,7 +230,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
229
230
|
if (ctx.maxTestsPerRule && ctx.forbidPlaceholders) {
|
|
230
231
|
const over = testsPerCriterion(content).find((c) => c.tests > ctx.maxTestsPerRule);
|
|
231
232
|
if (over) {
|
|
232
|
-
|
|
233
|
+
reasons.push(`${over.tests} tests check the same criterion (${over.criterion.slice(0, 80)}…): keep at most ${ctx.maxTestsPerRule}, one per branch, and put further inputs in one parametrised test`);
|
|
233
234
|
}
|
|
234
235
|
}
|
|
235
236
|
// F15 — a pending test says why. A reasonless fixme/skip reaches the report
|
|
@@ -238,7 +239,7 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
238
239
|
if (ctx.framework === 'playwright') {
|
|
239
240
|
const bare = reasonlessPending(content);
|
|
240
241
|
if (bare.length) {
|
|
241
|
-
|
|
242
|
+
reasons.push(`pending test(s) with no reason: "${bare[0]}"… — declare them as ` +
|
|
242
243
|
`test.fixme(title, { annotation: { type: 'fixme', description: '<why>' } }, async () => {})`);
|
|
243
244
|
}
|
|
244
245
|
}
|
|
@@ -257,14 +258,14 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
257
258
|
const sel = String.raw `(?:getByTestId\s*\(|\[\s*data-(?:testid|cy|test)\s*=)`;
|
|
258
259
|
const selNearMarker = new RegExp(`(?:${sel}[^\\n]*?\\b${marker}\\b)|(?:\\b${marker}\\b[^\\n]*?${sel})`, 'i');
|
|
259
260
|
if (selNearMarker.test(active)) {
|
|
260
|
-
|
|
261
|
+
reasons.push('a runnable test uses a guessed/placeholder selector — ground it in a real component attribute, or defer the test (test.fixme/skip)');
|
|
261
262
|
}
|
|
262
263
|
// (b) When the project's real test-hooks are known, any test-id a
|
|
263
264
|
// runnable test uses that isn't one of them is an invented selector.
|
|
264
265
|
if (ctx.knownSelectors && ctx.knownSelectors.size > 0) {
|
|
265
266
|
for (const id of usedTestIds(active)) {
|
|
266
267
|
if (!ctx.knownSelectors.has(id)) {
|
|
267
|
-
|
|
268
|
+
reasons.push(`references selector '${id}' not found in the project's components — ground it or defer the test`);
|
|
268
269
|
}
|
|
269
270
|
}
|
|
270
271
|
}
|
|
@@ -272,11 +273,13 @@ export function checkTestGates(content, _ext, ctx) {
|
|
|
272
273
|
if (ctx.knownTexts) {
|
|
273
274
|
const missing = usedLabels(active).filter((l) => !ctx.knownTexts.includes(l.toLowerCase()));
|
|
274
275
|
if (missing.length) {
|
|
275
|
-
|
|
276
|
+
reasons.push(`uses ${missing.map((m) => `"${m}"`).slice(0, 3).join(', ')} as an accessible name, label or placeholder, but that text appears nowhere in the app's source or the journey doc — take every name from the code facts or the rules`);
|
|
276
277
|
}
|
|
277
278
|
}
|
|
278
279
|
}
|
|
279
|
-
|
|
280
|
+
// Every reason at once: a retry told one problem fixes it and trips on the next.
|
|
281
|
+
const unique = [...new Set(reasons)];
|
|
282
|
+
return unique.length ? (unique.length === 1 ? unique[0] : `${unique[0]}\nAlso:\n${unique.slice(1, 5).map((r) => `- ${r}`).join('\n')}`) : null;
|
|
280
283
|
}
|
|
281
284
|
/**
|
|
282
285
|
* The run-time skips whose condition looks at the page: test.skip(true, …),
|
|
@@ -490,6 +493,7 @@ function looksStructurallyClean(content) {
|
|
|
490
493
|
t.startsWith('function ') ||
|
|
491
494
|
t.startsWith('async ') ||
|
|
492
495
|
t.startsWith('require(') ||
|
|
496
|
+
/^(?:describe|test|it|beforeAll|beforeEach)\s*\(/.test(t) ||
|
|
493
497
|
t.startsWith("'use strict'") ||
|
|
494
498
|
t.startsWith('"use strict"');
|
|
495
499
|
if (!startsWithCode)
|
|
@@ -675,6 +679,29 @@ function testCriteria(content) {
|
|
|
675
679
|
return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
|
|
676
680
|
});
|
|
677
681
|
}
|
|
682
|
+
/**
|
|
683
|
+
* The file with at most `max` tests per criterion: the first ones are kept,
|
|
684
|
+
* the extras removed. A mechanical fix — no AI retry for "too many tests".
|
|
685
|
+
*/
|
|
686
|
+
export function trimExtraTests(content, max) {
|
|
687
|
+
const blocks = findTestBlocks(content);
|
|
688
|
+
const crit = testCriteria(content);
|
|
689
|
+
const seen = new Map();
|
|
690
|
+
const extra = blocks.filter((_, i) => {
|
|
691
|
+
const c = crit[i]?.criterion;
|
|
692
|
+
if (!c)
|
|
693
|
+
return false;
|
|
694
|
+
const n = (seen.get(c) ?? 0) + 1;
|
|
695
|
+
seen.set(c, n);
|
|
696
|
+
return n > max;
|
|
697
|
+
});
|
|
698
|
+
if (!extra.length)
|
|
699
|
+
return content;
|
|
700
|
+
let out = content;
|
|
701
|
+
for (const b of [...extra].sort((x, y) => y.start - x.start))
|
|
702
|
+
out = out.slice(0, b.start) + out.slice(b.end).replace(/^;?[ \t]*\n?/, '');
|
|
703
|
+
return out;
|
|
704
|
+
}
|
|
678
705
|
/** How many tests of a file check each criterion (tests with no oracle are not counted). */
|
|
679
706
|
export function testsPerCriterion(content) {
|
|
680
707
|
const counts = new Map();
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { spawn } from 'node:child_process';
|
|
14
14
|
import { readFileSync, existsSync } from 'node:fs';
|
|
15
|
-
import { loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
|
|
15
|
+
import { isClaude, loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
|
|
16
16
|
import { runAiSync, noteAiCall, traceArgs, traceCall } from './ai-usage.js';
|
|
17
17
|
import { API, WEB, codeOutput, evidence } from '../prompts/blocks.js';
|
|
18
18
|
import { stripStrayComments } from '../gates/validate.js';
|
|
@@ -339,7 +339,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
|
|
|
339
339
|
``,
|
|
340
340
|
`NAMES THE PRODUCT GIVES. The doc says WHAT happens; the product names HOW to`,
|
|
341
341
|
`reach it — a route, a button's label, a command, a table or a field.`,
|
|
342
|
-
` - When the CODE FACTS show that name,
|
|
342
|
+
` - When the CODE FACTS show that name (in ROUTES, SHAPES or DATA), use it`,
|
|
343
|
+
` as written there: the request fields a handler validates, a table's`,
|
|
344
|
+
` columns, a status value. need() is never for a name the facts show.`,
|
|
343
345
|
` - When they do not (the code is not built yet, or the facts missed it),`,
|
|
344
346
|
` write the test anyway and take the name from need('<plain name>'):`,
|
|
345
347
|
` const { need } = require('@activate-core/test-forge/reporters/interface.cjs');`,
|
|
@@ -753,6 +755,9 @@ function writerCommand(prompt, memory) {
|
|
|
753
755
|
if (!memory)
|
|
754
756
|
return { ...withReadOnlyTools(cmd), input: prompt };
|
|
755
757
|
const { system, message } = splitShared(prompt);
|
|
758
|
+
// Only the Claude CLI takes a system prompt; any other AI gets the rules at the top of its input.
|
|
759
|
+
if (system && !isClaude(cmd.program))
|
|
760
|
+
return { ...withNoTools(cmd), input: `${system}\n\n${message}` };
|
|
756
761
|
return system ? { ...withNoTools(cmd, system), input: message } : { ...withNoTools(cmd), input: prompt };
|
|
757
762
|
}
|
|
758
763
|
/**
|
|
@@ -843,7 +848,8 @@ export async function writeFilesWithAI(files) {
|
|
|
843
848
|
const config = loadConfig();
|
|
844
849
|
const cmd = splitCommand(config.aiCommand);
|
|
845
850
|
const { program, args } = system ? withNoTools(cmd, system) : withNoTools(cmd);
|
|
846
|
-
|
|
851
|
+
const input = system && !isClaude(cmd.program) ? `${system}\n\n${message}` : message;
|
|
852
|
+
return splitBatchAnswer(await runWithStdin(program, args, input, config.aiTimeoutMs), files.map((f) => f.path));
|
|
847
853
|
}
|
|
848
854
|
/** Each file's code from a batch answer, by its marker line. */
|
|
849
855
|
export function splitBatchAnswer(out, paths) {
|
|
@@ -81,6 +81,7 @@ let traceSeq = 0;
|
|
|
81
81
|
const STAGES = [
|
|
82
82
|
[/^=== REPAIR ===$/m, 'repair'],
|
|
83
83
|
[/^You are a senior QA engineer\. Generate a complete/, 'write'],
|
|
84
|
+
[/^You are a senior QA engineer writing one .* test file/, 'write'],
|
|
84
85
|
[/^You are a senior test architect/, 'plan'],
|
|
85
86
|
[/^You are a senior engineer establishing the MECHANICS/, 'grounding'],
|
|
86
87
|
[/^You review ONE claimed product defect/, 'defect-review'],
|
|
@@ -128,7 +129,9 @@ export function traceCall(prompt, raw, started, error) {
|
|
|
128
129
|
}
|
|
129
130
|
});
|
|
130
131
|
const result = events.find((e) => e.type === 'result');
|
|
131
|
-
|
|
132
|
+
// Only a Claude event stream has typed events; another AI's JSON answer is the answer itself.
|
|
133
|
+
const isStream = events.some((e) => typeof e.type === 'string');
|
|
134
|
+
const text = isStream ? String(result?.result ?? '') : raw;
|
|
132
135
|
const ru = result?.usage ?? {};
|
|
133
136
|
const tokens = ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'].reduce((n, k) => n + (Number(ru[k]) || 0), 0);
|
|
134
137
|
const cost = Number(result?.total_cost_usd) || 0;
|
|
@@ -154,7 +157,7 @@ export function traceCall(prompt, raw, started, error) {
|
|
|
154
157
|
try {
|
|
155
158
|
mkdirSync(TRACE_DIR, { recursive: true });
|
|
156
159
|
writeFileSync(join(TRACE_DIR, `${name}.prompt.md`), prompt);
|
|
157
|
-
writeFileSync(join(TRACE_DIR, `${name}.${
|
|
160
|
+
writeFileSync(join(TRACE_DIR, `${name}.${isStream ? 'stream.jsonl' : 'out.txt'}`), raw);
|
|
158
161
|
const u = result?.usage ?? {};
|
|
159
162
|
appendFileSync(join(TRACE_DIR, 'index.jsonl'), JSON.stringify({
|
|
160
163
|
seq, stage, file: file ? basename(file) : undefined,
|
|
@@ -27,11 +27,31 @@ export function shortReason(why) {
|
|
|
27
27
|
export function withoutNames(content) {
|
|
28
28
|
return content.replace(/^[ \t]*(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])[^'"`\n]*\2\s*\);?[ \t]*\n/gm, (line, name) => new RegExp(`\\b${name}\\b`).test(content.replace(line, '')) ? line : '');
|
|
29
29
|
}
|
|
30
|
+
/** A file top without the `const X = need('…')` lines whose name is still blank. */
|
|
31
|
+
export function withoutBlankNames(head, isBlank) {
|
|
32
|
+
if (!isBlank)
|
|
33
|
+
return head;
|
|
34
|
+
return head.replace(/^[ \t]*(?:const|let)\s+[A-Za-z_$][\w$]*\s*=\s*need\(\s*(['"`])([^'"`\n]*)\1\s*\);?[ \t]*\n/gm, (line, _q, name) => (isBlank(name) ? '' : line));
|
|
35
|
+
}
|
|
30
36
|
/** The file without the tests that have a problem. */
|
|
31
37
|
function withoutBlocks(content, problems) {
|
|
32
38
|
const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
|
|
33
39
|
return blocks.length ? spliceBlocks(content, blocks.map((block) => ({ block, text: '' }))) : content;
|
|
34
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* What an answer adds to the file: its test blocks for the rules asked for.
|
|
43
|
+
* An answer that came back as a whole file (imports, an outer describe) or
|
|
44
|
+
* repeats a test the file already has would otherwise land nested or twice.
|
|
45
|
+
*/
|
|
46
|
+
export function onlyNewTests(answer, file, rules) {
|
|
47
|
+
if (!/^\s*(?:import\s|const\s+\w+\s*=\s*require\(|(?:test\.)?describe\s*\()/m.test(answer))
|
|
48
|
+
return answer;
|
|
49
|
+
const have = new Set(findTestBlocks(file).map((b) => b.title));
|
|
50
|
+
return findTestBlocks(answer)
|
|
51
|
+
.filter((b) => !have.has(b.title) && (!b.rules.length || b.rules.some((r) => rules.includes(r))))
|
|
52
|
+
.map((b) => answer.slice(b.start, b.end).replace(/^\s*\n/, ''))
|
|
53
|
+
.join('\n\n');
|
|
54
|
+
}
|
|
35
55
|
/** Is the file complete as it is? */
|
|
36
56
|
export function fileIsComplete(content, rules, steps, knownTexts) {
|
|
37
57
|
const { cov, problems } = review(content, rules, steps, knownTexts);
|
|
@@ -46,38 +66,78 @@ export async function completeFile(content, ctx) {
|
|
|
46
66
|
const drop = (ps) => { for (const p of ps)
|
|
47
67
|
if (!dropped.has(p.title))
|
|
48
68
|
dropped.set(p.title, p.reason); };
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
69
|
+
const locked = ctx.locked ?? new Set();
|
|
70
|
+
const before = new Set(findTestBlocks(content).map((b) => b.title));
|
|
71
|
+
// A test that passed is never touched. A kept file's other tests are only
|
|
72
|
+
// reported: a check added later must not quietly shrink a working suite.
|
|
73
|
+
// Only the tests asked for again (redo) go, and so do new tests just written.
|
|
74
|
+
const removable = (p) => !locked.has(p.title) && (!ctx.kept || !before.has(p.title) || !!ctx.redo?.(p.block));
|
|
75
|
+
const redoBlocks = ctx.kept && ctx.redo
|
|
76
|
+
? findTestBlocks(content).filter((b) => !locked.has(b.title) && ctx.redo(b))
|
|
77
|
+
: [];
|
|
78
|
+
const redo = redoBlocks
|
|
79
|
+
.filter((b) => !first.problems.some((p) => p.block.start === b.start))
|
|
80
|
+
.map((b) => ({ title: b.title, rules: b.rules, reason: `it waits on a name the code now has: write it again from the code with the same title, "${b.title}"`, block: b }));
|
|
81
|
+
const firstDrop = [...first.problems.filter(removable), ...redo];
|
|
82
|
+
let failedRedo = false;
|
|
83
|
+
const refusedRules = [];
|
|
84
|
+
const reported = first.problems.filter((p) => !removable(p));
|
|
85
|
+
if (reported.length) {
|
|
86
|
+
notes.push(`left ${reported.length} earlier test(s) in place that a check flags (${[...new Set(reported.map((p) => shortReason(p.reason)))].slice(0, 3).join('; ')}) — not removed, they were already in the file`);
|
|
87
|
+
}
|
|
88
|
+
if (first.cov.missing.length || firstDrop.length) {
|
|
89
|
+
out = withoutNames(withoutBlocks(content, firstDrop));
|
|
90
|
+
drop(firstDrop);
|
|
52
91
|
// Only rules the doc has are written again: an id with no text makes the writer invent one.
|
|
53
92
|
const known = new Set(ctx.memory.rules.filter((r) => r.tag).map((r) => r.id));
|
|
54
|
-
const left = ruleCoverage(out, ctx.rules).missing.
|
|
93
|
+
const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])]
|
|
94
|
+
.filter((id) => known.has(id) && ctx.rules.includes(id) && !(ctx.skip?.has(id) && !redo.some((p) => p.rules.includes(id))));
|
|
55
95
|
if (left.length) {
|
|
56
96
|
const blocks = findTestBlocks(out);
|
|
57
|
-
|
|
97
|
+
// The file's top as the writer sees it: a constant that still waits on a
|
|
98
|
+
// blank name is left out, or the new tests reuse it and wait again.
|
|
99
|
+
const head = withoutBlankNames(out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000), ctx.isBlank);
|
|
58
100
|
const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(out)?.[2] ?? ctx.journey.name;
|
|
59
101
|
const slice = journeySlice(ctx.memory, left);
|
|
60
102
|
calls = 1;
|
|
61
103
|
try {
|
|
62
|
-
const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(
|
|
63
|
-
const candidate = appendTests(out, tests);
|
|
104
|
+
const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(firstDrop, first.cov));
|
|
105
|
+
const candidate = (ctx.fix ?? ((c) => c))(appendTests(out, onlyNewTests(tests, out, left)));
|
|
64
106
|
const refused = ctx.check(candidate);
|
|
65
|
-
if (refused)
|
|
107
|
+
if (refused) {
|
|
66
108
|
notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
|
|
109
|
+
failedRedo = true;
|
|
110
|
+
refusedRules.push(...left);
|
|
111
|
+
}
|
|
67
112
|
else
|
|
68
113
|
out = candidate;
|
|
69
114
|
}
|
|
70
115
|
catch (e) {
|
|
71
116
|
notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
|
|
117
|
+
failedRedo = true;
|
|
72
118
|
}
|
|
73
119
|
// What the second answer still gets wrong is removed, not sent again.
|
|
74
|
-
const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts });
|
|
120
|
+
const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts }).filter(removable);
|
|
75
121
|
if (second.length) {
|
|
76
122
|
out = withoutNames(withoutBlocks(out, second));
|
|
77
123
|
drop(second);
|
|
78
124
|
}
|
|
79
125
|
}
|
|
80
126
|
}
|
|
127
|
+
// A kept file whose tests were taken out to be written again, and the new
|
|
128
|
+
// ones did not come: it stays as it was. A waiting test beats no test.
|
|
129
|
+
if (ctx.kept && failedRedo && redo.length + firstDrop.length > 0) {
|
|
130
|
+
notes.push('kept the file as it was: the tests to write again did not come back');
|
|
131
|
+
out = content;
|
|
132
|
+
dropped.clear();
|
|
133
|
+
}
|
|
134
|
+
// The last word, checked mechanically: every test that passed is still here.
|
|
135
|
+
const lost = [...locked].filter((t) => before.has(t) && !findTestBlocks(out).some((b) => b.title === t));
|
|
136
|
+
if (lost.length) {
|
|
137
|
+
notes.push(`kept the file as it was: the change would have lost ${lost.length} test(s) that passed, e.g. "${lost[0].slice(0, 60)}"`);
|
|
138
|
+
out = content;
|
|
139
|
+
dropped.clear();
|
|
140
|
+
}
|
|
81
141
|
// A cannot-test line that is no longer true (its rule has a test now) or never
|
|
82
142
|
// was a reason ("the locator is unknown") does not stay in the file.
|
|
83
143
|
const has = new Set(findTestBlocks(out).flatMap((b) => b.rules));
|
|
@@ -95,5 +155,5 @@ export async function completeFile(content, ctx) {
|
|
|
95
155
|
}
|
|
96
156
|
if (cov.missing.length)
|
|
97
157
|
notes.push(`no test for ${cov.missing.join(', ')}`);
|
|
98
|
-
return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes };
|
|
158
|
+
return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes, refusedRules: [...new Set([...refusedRules, ...(calls ? ruleCoverage(out, ctx.rules).missing.filter((id) => !refusedRules.includes(id) && !failedRedo) : [])])] };
|
|
99
159
|
}
|
|
@@ -246,7 +246,8 @@ export function oracleClaim(t, content) {
|
|
|
246
246
|
return `${t.title} ${NOT_CLAIMED}`;
|
|
247
247
|
}
|
|
248
248
|
/** A [spec] line that carries the client's words, quoted or pointing at the quote above it ("source as above"). */
|
|
249
|
-
|
|
249
|
+
// A quoted source, a ticket key on its own ("source: SHOP-1") — the ticket is the source — or "same as".
|
|
250
|
+
export const HAS_SOURCE = /\((?:source(?::\s*(?:[A-Z][A-Z0-9]+-\d+:?\s*)?["“]|:\s*[A-Z][A-Z0-9]+-\d+\b|:?\s*(?:as|same)\b)|same source\b)/i;
|
|
250
251
|
/** Playwright's report of a locator that matched more than one element. */
|
|
251
252
|
const AMBIGUOUS = /strict mode violation|resolved to \d+ elements/i;
|
|
252
253
|
/** An application file:line, the only evidence a defect may rest on. */
|
|
@@ -40,6 +40,11 @@ export const HARNESS_CONTRACT = [
|
|
|
40
40
|
`- resetRateLimits(): Promise<void> (H7)`,
|
|
41
41
|
`- REQUIRED_PACKAGES: string[] (H6)`,
|
|
42
42
|
`- a 'run' role alongside the fixture roles (H8)`,
|
|
43
|
+
`- insert(table, row): Promise<id>, one row of scenario data written straight`,
|
|
44
|
+
` into the test database (H10)`,
|
|
45
|
+
`- DERIVED and AREA (H10)`,
|
|
46
|
+
`- update(table, match, values): Promise<number>, columns set on the rows`,
|
|
47
|
+
` matching every key of match (H10)`,
|
|
43
48
|
`Jest areas only`,
|
|
44
49
|
`- call(method, path, { body?, headers?, role? }):`,
|
|
45
50
|
` Promise<{ status, headers, body }>`,
|
|
@@ -177,6 +182,35 @@ export const HARNESS_CONTRACT = [
|
|
|
177
182
|
` 'unknown step "<name>" (steps: …)'. If there is no such flow, export`,
|
|
178
183
|
` STEPS = {} and have openStep resolve`,
|
|
179
184
|
` 'this product has no multi-step flow'.`,
|
|
185
|
+
``,
|
|
186
|
+
`H10. Scenario data. A scenario describes people and their state (an operator`,
|
|
187
|
+
` with a location, services, working hours and earlier jobs; a setting`,
|
|
188
|
+
` switched on). insert() and update() let a test build exactly that:`,
|
|
189
|
+
` - insert fills every NOT NULL column without a default that row leaves out`,
|
|
190
|
+
` with a value of its type (read the columns from the database), returns`,
|
|
191
|
+
` the new id, and remembers the row.`,
|
|
192
|
+
` - update remembers the previous values of the rows it changes.`,
|
|
193
|
+
` - Each test starts from a clean slate: the rows a test writes, and what it`,
|
|
194
|
+
` updates, are undone right after that test (last first), so one test's`,
|
|
195
|
+
` data never takes part in the next. The testkit does this itself when it`,
|
|
196
|
+
` is loaded: it notes where the record stands before each test and undoes`,
|
|
197
|
+
` back to that point after it (Jest: the global beforeEach/afterEach;`,
|
|
198
|
+
` Playwright: test.beforeEach/afterEach). Tests do nothing for it.`,
|
|
199
|
+
` - What a beforeAll writes stays for the whole file; disconnect() undoes it`,
|
|
200
|
+
` and anything left, so every run leaves the database as it found it.`,
|
|
201
|
+
` - Both refuse a database that is not clearly a test one, like reset() (H5).`,
|
|
202
|
+
` - Some values a scenario sets are not columns: the product works them out`,
|
|
203
|
+
` from other records (a standing from past jobs, an average per day, a`,
|
|
204
|
+
` count). Read in the code how each is worked out. insert() accepts each as`,
|
|
205
|
+
` a virtual column named _tf_<name> (lower case, underscores) and, instead`,
|
|
206
|
+
` of writing it, writes the records that give exactly that value (and are`,
|
|
207
|
+
` undone with the row). Export DERIVED: { _tf_<name>: 'how it is made' }`,
|
|
208
|
+
` for each, so tests can use them.`,
|
|
209
|
+
` - Export AREA: { latitude, longitude } (or the project's own key, when`,
|
|
210
|
+
` the product picks among records by something else): a place no record`,
|
|
211
|
+
` already in the database reaches, found by querying it. Scenarios where`,
|
|
212
|
+
` the product picks one record among many are built there, so data that`,
|
|
213
|
+
` was already in the database never takes part.`,
|
|
180
214
|
].join('\n');
|
|
181
215
|
/**
|
|
182
216
|
* Stamp written into every generated testkit: the contract it was built
|
|
@@ -68,3 +68,15 @@ export function recordInterface(found, path = INTERFACE_PATH) {
|
|
|
68
68
|
}
|
|
69
69
|
return blankNames(Object.keys(found), values);
|
|
70
70
|
}
|
|
71
|
+
/** Blank these names again (a value the code no longer has): their tests wait instead of failing for the wrong reason. */
|
|
72
|
+
export function clearInterface(names, path = INTERFACE_PATH) {
|
|
73
|
+
if (!names.length)
|
|
74
|
+
return;
|
|
75
|
+
const values = readInterface(path);
|
|
76
|
+
for (const n of names)
|
|
77
|
+
if (n in values)
|
|
78
|
+
values[n] = '';
|
|
79
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
80
|
+
const sorted = Object.fromEntries(Object.entries(values).sort(([a], [b]) => a.localeCompare(b)));
|
|
81
|
+
writeFileSync(path, `${JSON.stringify(sorted, null, 2)}\n`, 'utf-8');
|
|
82
|
+
}
|
|
@@ -206,6 +206,10 @@ const RUNTIME_HELPERS = {
|
|
|
206
206
|
fillFrameInputs: 'async fillFrameInputs(page, framePrefix, { name: value }) — fills inputs inside an embedded iframe (card fields).',
|
|
207
207
|
watchDialogs: 'watchDialogs(page) → string[] — accepts browser alerts and records their messages.',
|
|
208
208
|
};
|
|
209
|
+
const KIT_CONTRACT = {
|
|
210
|
+
insert: "resolves to the new row's id itself (a number or key, not a row object): const id = await insert('users', {...})",
|
|
211
|
+
update: 'resolves to the number of rows changed',
|
|
212
|
+
};
|
|
209
213
|
export function helperApi(harnessSource) {
|
|
210
214
|
const names = new Set();
|
|
211
215
|
const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(harnessSource);
|
|
@@ -262,7 +266,9 @@ export function helperApi(harnessSource) {
|
|
|
262
266
|
if (text)
|
|
263
267
|
doc.unshift(text);
|
|
264
268
|
}
|
|
265
|
-
|
|
269
|
+
// The scenario-data writers' contract (H10) holds whatever comment sits above them.
|
|
270
|
+
const kit = KIT_CONTRACT[name] && /^(?:async\s+)?function/.test(sig) ? KIT_CONTRACT[name] : doc.join(' ');
|
|
271
|
+
out.push(`- ${sig.length > 200 ? `${sig.slice(0, 197)}…` : sig}${kit ? ` — ${kit}` : ''}`);
|
|
266
272
|
}
|
|
267
273
|
return out.join('\n');
|
|
268
274
|
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — the tests that have passed, as the run recorded them.
|
|
3
|
+
*
|
|
4
|
+
* A test that passed once checked the product for real: generate never
|
|
5
|
+
* removes or rewrites it again. The reporters add to testforge/passed.json
|
|
6
|
+
* after every run ({ "<file relative to the project>": ["title", …] }); a
|
|
7
|
+
* later failure does not take a test off the list — that is a regression to
|
|
8
|
+
* report, not a test to rewrite. `generate --force` is the only way past it.
|
|
9
|
+
*/
|
|
10
|
+
import { readFileSync } from 'node:fs';
|
|
11
|
+
import { join, relative, resolve } from 'node:path';
|
|
12
|
+
import { STATE_DIR } from '../paths.js';
|
|
13
|
+
export const PASSED_PATH = join(STATE_DIR, 'passed.json');
|
|
14
|
+
/** Titles of the tests in `file` that have passed in a recorded run. */
|
|
15
|
+
export function passedTitles(file, path = PASSED_PATH) {
|
|
16
|
+
try {
|
|
17
|
+
const all = JSON.parse(readFileSync(path, 'utf-8'));
|
|
18
|
+
const rel = relative(process.cwd(), resolve(file)).split('\\').join('/');
|
|
19
|
+
return new Set(Array.isArray(all[rel]) ? all[rel] : []);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return new Set();
|
|
23
|
+
}
|
|
24
|
+
}
|