@activate-core/test-forge 2.6.0 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/cli/index.js +4 -1
- package/dist/core/docs/excerpts.js +49 -3
- package/dist/core/docs/grounding.js +4 -0
- package/dist/core/gates/validate.js +49 -23
- package/dist/core/generate/ai-generator.js +41 -0
- package/dist/core/generate/ai-usage.js +1 -0
- package/dist/core/generate/complete-file.js +46 -7
- package/dist/core/generate/run-generate.js +224 -18
- package/dist/core/generate/test-blocks.js +87 -0
- package/dist/core/project/env-diff.js +3 -1
- package/dist/core/project/setup.js +10 -2
- package/dist/core/prompts/blocks.js +13 -5
- package/dist/core/run/name-scan.js +58 -3
- package/dist/core/run/report.js +2 -0
- package/dist/core/run/run-tests.js +5 -0
- package/package.json +3 -2
- package/reporters/interface.cjs +26 -3
|
@@ -18,6 +18,8 @@ import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
|
|
|
18
18
|
import { runAiSync } from '../generate/ai-usage.js';
|
|
19
19
|
import { clearInterface, readInterface, recordInterface } from '../generate/interface-file.js';
|
|
20
20
|
const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
|
|
21
|
+
/** Why each still-blank name is blank, for the waiting message (name → reason). */
|
|
22
|
+
const STATUS_PATH = join(STATE_DIR, 'interface-status.json');
|
|
21
23
|
/** Values the scan filled (name → value), so each run can check the code still has them. */
|
|
22
24
|
const FILLED_PATH = join(STATE_DIR, 'interface-filled.json');
|
|
23
25
|
const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
|
|
@@ -215,7 +217,27 @@ export function namesToAsk(blank, root, memo) {
|
|
|
215
217
|
* Look for the blank names in the code and fill what is found. `blank` are
|
|
216
218
|
* the names the tests to run still wait for. Returns what was filled.
|
|
217
219
|
*/
|
|
218
|
-
|
|
220
|
+
/** Is `value` in the project's code, as a key, string, route path or verb (the facts are read from it, so this checks them)? */
|
|
221
|
+
function inProjectCode(value, root) {
|
|
222
|
+
const files = codeFiles(root);
|
|
223
|
+
const route = files.filter((f) => fileMatches('route', relative(root, f)));
|
|
224
|
+
const read = (fs_) => fs_.flatMap((f) => { try {
|
|
225
|
+
return readFileSync(f, 'utf-8').split('\n');
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
return [];
|
|
229
|
+
} });
|
|
230
|
+
if (/^\/|^[\w-]+\//.test(value.trim()))
|
|
231
|
+
return foundInCode(value, read(route));
|
|
232
|
+
return foundInCode(value, read(files));
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Look for the blank names in the code and fill what is found. `blank` are
|
|
236
|
+
* the names the tests to run still wait for. With `facts` (at generate time)
|
|
237
|
+
* every blank name is asked once against the journey's code facts too; a
|
|
238
|
+
* value is kept only if the code itself has it. Returns what was filled.
|
|
239
|
+
*/
|
|
240
|
+
export function scanNames(blank, root = '.', facts) {
|
|
219
241
|
const memo = (() => {
|
|
220
242
|
try {
|
|
221
243
|
return JSON.parse(readFileSync(SCAN_PATH, 'utf-8'));
|
|
@@ -225,6 +247,10 @@ export function scanNames(blank, root = '.') {
|
|
|
225
247
|
}
|
|
226
248
|
})();
|
|
227
249
|
const all = namesToAsk(blank, root, memo);
|
|
250
|
+
if (facts)
|
|
251
|
+
for (const n of blank)
|
|
252
|
+
if (!all.some((a) => a.name === n))
|
|
253
|
+
all.push({ name: n, lines: [] });
|
|
228
254
|
// One call stays small: names past the budget are asked on the next run.
|
|
229
255
|
const ask = [];
|
|
230
256
|
let size = 0;
|
|
@@ -239,11 +265,14 @@ export function scanNames(blank, root = '.') {
|
|
|
239
265
|
}
|
|
240
266
|
const filled = [];
|
|
241
267
|
let calls = 0;
|
|
268
|
+
let callFailed = false;
|
|
242
269
|
if (ask.length) {
|
|
243
270
|
const known = Object.entries(readInterface()).filter(([, v]) => String(v).trim()).slice(0, 8);
|
|
244
271
|
const prompt = [
|
|
245
272
|
'You match names a test needs to values in code. For each name in <names>,',
|
|
246
|
-
|
|
273
|
+
facts
|
|
274
|
+
? "pick its value from that name's own lines or from <code_facts> (the routes, request fields, response shapes and tables of this feature), or null."
|
|
275
|
+
: "pick its value from that name's own lines only, or null.",
|
|
247
276
|
'',
|
|
248
277
|
'A value is copied from the lines, never invented.',
|
|
249
278
|
'- A URL is the path a client calls: it starts with "/" and includes every',
|
|
@@ -255,6 +284,7 @@ export function scanNames(blank, root = '.') {
|
|
|
255
284
|
'If no line clearly is that thing, the value is null.',
|
|
256
285
|
...(known.length ? ['', '<filled_examples>', ...known.map(([k, v]) => `- ${k}: ${v}`), '</filled_examples>', 'These values are already filled; follow their format.'] : []),
|
|
257
286
|
'',
|
|
287
|
+
...(facts ? ['<code_facts>', facts.slice(0, 20_000), '</code_facts>', ''] : []),
|
|
258
288
|
'<names>',
|
|
259
289
|
...ask.flatMap((a) => [`NAME: ${a.name}`, ...a.lines.map((l) => ` ${l}`)]),
|
|
260
290
|
'</names>',
|
|
@@ -272,7 +302,7 @@ export function scanNames(blank, root = '.') {
|
|
|
272
302
|
const keep = {};
|
|
273
303
|
for (const a of ask) {
|
|
274
304
|
const v = json[a.name];
|
|
275
|
-
if (typeof v === 'string' && foundInCode(v, a.lines))
|
|
305
|
+
if (typeof v === 'string' && v.trim() && (foundInCode(v, a.lines) || (facts && inProjectCode(v, root))))
|
|
276
306
|
keep[a.name] = v.trim();
|
|
277
307
|
}
|
|
278
308
|
recordInterface(keep);
|
|
@@ -297,6 +327,7 @@ export function scanNames(blank, root = '.') {
|
|
|
297
327
|
console.warn(` name scan: the lookup call failed — ${e.message.split('\n')[0].slice(0, 120)}; the names stay blank`);
|
|
298
328
|
for (const a of ask)
|
|
299
329
|
delete memo[a.name]; // look again next time
|
|
330
|
+
callFailed = true;
|
|
300
331
|
}
|
|
301
332
|
}
|
|
302
333
|
try {
|
|
@@ -307,6 +338,30 @@ export function scanNames(blank, root = '.') {
|
|
|
307
338
|
// best-effort memo
|
|
308
339
|
}
|
|
309
340
|
const done = new Set(filled.map((f) => f.name));
|
|
341
|
+
// Why each name is still blank, so a waiting test says whether to look at
|
|
342
|
+
// the product or at TestForge.
|
|
343
|
+
try {
|
|
344
|
+
const status = (() => { try {
|
|
345
|
+
return JSON.parse(readFileSync(STATUS_PATH, 'utf-8'));
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
return {};
|
|
349
|
+
} })();
|
|
350
|
+
for (const n of blank) {
|
|
351
|
+
if (done.has(n)) {
|
|
352
|
+
delete status[n];
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
const asked = ask.find((a) => a.name === n);
|
|
356
|
+
status[n] = !kindOf(n) && !facts ? 'TestForge does not look this kind of name up in the code; fill it by hand'
|
|
357
|
+
: asked && callFailed ? 'the lookup could not run this time; it is tried again on the next run'
|
|
358
|
+
: asked ? 'looked for in the code and not found'
|
|
359
|
+
: 'nothing in the code looks like it yet';
|
|
360
|
+
}
|
|
361
|
+
mkdirSync(dirname(STATUS_PATH), { recursive: true });
|
|
362
|
+
writeFileSync(STATUS_PATH, `${JSON.stringify(status, null, 2)}\n`, 'utf-8');
|
|
363
|
+
}
|
|
364
|
+
catch { /* the message just has no reason */ }
|
|
310
365
|
return { filled, stillBlank: blank.filter((n) => !done.has(n)), calls };
|
|
311
366
|
}
|
|
312
367
|
/**
|
package/dist/core/run/report.js
CHANGED
|
@@ -43,6 +43,8 @@ export function loadDotenv(cwd = process.cwd()) {
|
|
|
43
43
|
let v = s.slice(i + 1).trim();
|
|
44
44
|
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))
|
|
45
45
|
v = v.slice(1, -1);
|
|
46
|
+
else
|
|
47
|
+
v = v.replace(/(^|\s)#.*$/, '').trim(); // `KEY= # hint` is empty, not the hint
|
|
46
48
|
if (k && process.env[k] === undefined)
|
|
47
49
|
process.env[k] = v;
|
|
48
50
|
}
|
|
@@ -110,6 +110,11 @@ export function runnerCommand(config, area, framework, root, rel, journey, targe
|
|
|
110
110
|
}
|
|
111
111
|
export function runTests(opts = {}) {
|
|
112
112
|
const config = loadConfig();
|
|
113
|
+
// The journey goes into a shell command: only a journey id's characters.
|
|
114
|
+
if (opts.journey && !/^[\w.-]+$/.test(opts.journey)) {
|
|
115
|
+
console.log(`"${opts.journey}" is not a journey id (letters, digits, - _ . only) — nothing was run.`);
|
|
116
|
+
return 1;
|
|
117
|
+
}
|
|
113
118
|
const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
|
|
114
119
|
// An env file named on the command line that is not there is a mistake, not a default.
|
|
115
120
|
if (opts.envFile && !existsSync(opts.envFile)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.1",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -10,7 +10,8 @@
|
|
|
10
10
|
"files": [
|
|
11
11
|
"dist",
|
|
12
12
|
"reporters",
|
|
13
|
-
"README.md"
|
|
13
|
+
"README.md",
|
|
14
|
+
"CHANGELOG.md"
|
|
14
15
|
],
|
|
15
16
|
"scripts": {
|
|
16
17
|
"build": "rm -rf dist && tsc",
|
package/reporters/interface.cjs
CHANGED
|
@@ -103,7 +103,9 @@ function usersOf(file, name) {
|
|
|
103
103
|
const end = close < 0 ? lines.length : lineOf(close) + 1;
|
|
104
104
|
return { start: i, end, text: lines.slice(i, end).join('\n'), title: (/\(\s*(['"`])((?:(?!\1).)*)\1/.exec(lines[i]) || [])[2] };
|
|
105
105
|
});
|
|
106
|
-
const
|
|
106
|
+
const word = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
|
|
107
|
+
// A name inside a string ('redeem points URL') is not a use of the constant URL.
|
|
108
|
+
const uses = { test: (t) => word.test(String(t).replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, '""')) };
|
|
107
109
|
const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
|
|
108
110
|
if (lines.some((l, i) => i !== declLine && !inTest(i) && uses.test(l))) return undefined;
|
|
109
111
|
return tests.filter((t) => uses.test(t.text));
|
|
@@ -159,14 +161,35 @@ function quietHooks() {
|
|
|
159
161
|
}
|
|
160
162
|
}
|
|
161
163
|
|
|
164
|
+
/** Why the last name scan left `name` blank, if it recorded one. */
|
|
165
|
+
function scanReason(name) {
|
|
166
|
+
try {
|
|
167
|
+
const p = path.join(path.dirname(interfacePath()), 'interface-status.json');
|
|
168
|
+
return JSON.parse(fs.readFileSync(p, 'utf-8'))[name];
|
|
169
|
+
} catch {
|
|
170
|
+
return undefined;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
162
174
|
/** The product's value for `name`, or an error naming what to fill. */
|
|
163
175
|
function need(name) {
|
|
164
176
|
const v = values()[name];
|
|
165
177
|
if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
|
|
166
178
|
// The prefix lets the reporters file this as "waiting for the product", not as a defect.
|
|
167
|
-
const
|
|
179
|
+
const why = scanReason(name);
|
|
180
|
+
const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json)${why ? ` — ${why}` : ''}: this test runs for real once it is there.`;
|
|
168
181
|
if (blockTestsThatUse(message, name)) return `<waiting: ${name}>`;
|
|
169
182
|
throw new Error(message);
|
|
170
183
|
}
|
|
171
184
|
|
|
172
|
-
|
|
185
|
+
/**
|
|
186
|
+
* A path from need() with its parameters filled in order, whatever the
|
|
187
|
+
* product calls them: /bookings/{booking}/cancel, /orders/:id or {id}.
|
|
188
|
+
* fillPath(need('booking cancel URL'), bookingId)
|
|
189
|
+
*/
|
|
190
|
+
function fillPath(template, ...values) {
|
|
191
|
+
let i = 0;
|
|
192
|
+
return String(template).replace(/\{[^}/]+\}|(?<=\/):[A-Za-z_]\w*/g, (m) => (i < values.length ? encodeURIComponent(String(values[i++])) : m));
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
module.exports = { need, fillPath, _usersOf: usersOf };
|