@activate-core/test-forge 2.1.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -387,9 +387,12 @@ history, the plan, and the code facts. No later step sends the whole doc again.
387
387
  with the reason, if it sleeps for a fixed time, skips because an element is
388
388
  missing, has no test or no assertion, spends more than `maxTestsPerRule`
389
389
  tests on one rule, or names a label found neither in the code nor the doc.
390
- 6. **Every rule, a test that can fail.** Each test is checked on its own:
391
- every planned rule has a test, or a `// cannot-test: R7 — <the state this
392
- area cannot set up>` line ("the locator is unknown" is not such a state);
390
+ 6. **Every rule, a test that can fail.** Every scenario comes from the doc
391
+ (plus the edge, security and user cases TestForge adds); what the code has
392
+ or lacks never removes one. Each test is checked on its own: every planned
393
+ rule has a test, or — only when it needs an outside service the test
394
+ cannot drive (a payment provider's inside, an email inbox, another app) —
395
+ a `// cannot-test: R7 — <the service>` line;
393
396
  no assertion accepts the good and the bad outcome alike, compares the page's
394
397
  whole text, sits only inside an `if`, or repeats what its own stub returned;
395
398
  `kit.find` is awaited and finds elements by the names the screen shows;
@@ -402,10 +405,21 @@ history, the plan, and the code facts. No later step sends the whole doc again.
402
405
  in its tests, a removed rule's tests are deleted (both without AI); a
403
406
  reworded or new rule gets its tests written by one small call.
404
407
 
408
+ **Names the product gives.** A test reaches the product through names: a
409
+ route, a button's label, a command, a table or field. When the code already
410
+ has one, the test uses it. When it does not (yet), the test asks for it by a
411
+ plain name — `need('order summary URL')` from
412
+ `@activate-core/test-forge/reporters/interface.cjs` — and the name is listed
413
+ in `testforge/interface.json` with an empty value. The developer fills it once
414
+ it exists; `testforge run` and `bind` hold a file back, and list what it waits
415
+ for, while one of its names is empty. TestForge never overwrites a value.
416
+
417
+ Every run ends with its AI cost per step (write, grounding, seed, harness …).
418
+
405
419
  **A feature that is not live yet.** Put `Status: not built` or
406
420
  `Status: behind flag <name>` in the journey doc's header. Every file of the
407
421
  journey is stamped `// testforge-pending: …`, and `testforge run` (and `bind`)
408
- hold it back until the line is removed; `--all` runs it anyway.
422
+ hold it back until the line is removed; `--all` runs it anyway. No seed is written for it: its tests make their own data.
409
423
 
410
424
  `generate` runs nothing. `testforge run` runs the suite; `testforge review`
411
425
  judges failures when you ask for it.
package/dist/cli/index.js CHANGED
@@ -24,7 +24,7 @@ import { runReview } from '../core/generate/review-journey.js';
24
24
  import { migrateState } from '../core/paths.js';
25
25
  import { acceptNames, bindJourneys } from '../core/generate/bind.js';
26
26
  import { runClean } from '../core/run/clean.js';
27
- import { aiUsage, describeUsage } from '../core/generate/ai-usage.js';
27
+ import { aiUsage, describeUsage, usageByStage } from '../core/generate/ai-usage.js';
28
28
  import { envDiff, describeEnvDiff } from '../core/project/env-diff.js';
29
29
  import { bundleTickets } from '../core/docs/tickets.js';
30
30
  import { writeGithubWorkflow } from '../core/run/ci.js';
@@ -279,8 +279,13 @@ async function main() {
279
279
  console.log(`\nDone. ${generated} generated, ${skipped} skipped (unchanged)` +
280
280
  (notices ? `, ${notices} notice(s) above.` : '.'));
281
281
  const spent = aiUsage();
282
- if (spent.calls)
282
+ if (spent.calls) {
283
283
  console.log(`AI this run: ${describeUsage(spent)}.`);
284
+ const steps = usageByStage().filter((x) => x.calls);
285
+ if (steps.length > 1 || steps[0]?.tokens) {
286
+ console.log(` by step: ${steps.map((x) => `${x.stage} ${x.calls} call(s) ${x.tokens.toLocaleString('en-GB')} tokens $${x.costUsd.toFixed(2)}`).join(' · ')}`);
287
+ }
288
+ }
284
289
  // What the journeys just written still need from the env file.
285
290
  const touched = results.filter((r) => r.testDir && r.status !== 'skipped').map((r) => basename(r.testDir));
286
291
  if (touched.length) {
@@ -68,6 +68,14 @@ const SKIP_DIRS = new Set([
68
68
  LEGACY_STATE_DIR,
69
69
  OLD_STATE_DIR,
70
70
  'generated-tests',
71
+ // The suite TestForge writes is not the product: hashing it would re-read
72
+ // the code facts every time a test changed.
73
+ 'tests',
74
+ 'test',
75
+ '__tests__',
76
+ 'vendor',
77
+ 'storage',
78
+ 'public',
71
79
  ]);
72
80
  export function hashProjectSource(rootDir) {
73
81
  const parts = [];
@@ -202,6 +202,53 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
202
202
  order.push(h.file);
203
203
  }
204
204
  }
205
+ // Where the endpoints are registered (Laravel routes/*.php, an express or
206
+ // Nest router, Django urls.py): the lines whose path or handler shares a
207
+ // word with the doc, first in the excerpts. A doc rarely names a route; a
208
+ // test cannot call one without its real path.
209
+ const words = [...new Set((markdown.toLowerCase().match(/[a-z]{5,}/g) ?? []).filter((w) => !GENERIC.has(w)))];
210
+ const stems = [...new Set(words.map((w) => w.replace(/(?:ings?|ed|es|s)$/, '')).filter((w) => w.length >= 5))];
211
+ const ROUTE_FILE = /(^|[\\/])(routes?|router|urls)([\\/]|\.[a-z]+$)|\.routes?\.[a-z]+$|[\\/]routes?\.[a-z]+$/i;
212
+ const ROUTE_LINE = /Route::\w+\(|\brouter\.(?:get|post|put|patch|delete|all|use)\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(/;
213
+ // Only this package's own routes: a related repo's are another area's.
214
+ const routeFiles = files.filter((f) => !relative(process.cwd(), f).startsWith('..') && ROUTE_FILE.test(relative(process.cwd(), f)));
215
+ // Every route that shares a word with the doc, the most shared first, at
216
+ // most 120 across the project; the prefix and group lines that complete
217
+ // their paths come with them. Commented-out routes are not routes.
218
+ const candidates = [];
219
+ for (const f of routeFiles.slice(0, 8)) {
220
+ let lines;
221
+ try {
222
+ lines = readFileSync(f, 'utf-8').split('\n');
223
+ }
224
+ catch {
225
+ continue;
226
+ }
227
+ lines.forEach((l, i) => {
228
+ if (/^\s*(?:\/\/|#|\*)/.test(l))
229
+ return;
230
+ const low = l.toLowerCase();
231
+ if (/->prefix\(|::prefix\(|\bprefix\s*[:=]|->group\(|::group\(/i.test(l)) {
232
+ add(f, i + 1, i + 1);
233
+ return;
234
+ }
235
+ if (!ROUTE_LINE.test(l))
236
+ return;
237
+ const score = stems.filter((w) => low.includes(w)).length;
238
+ if (score)
239
+ candidates.push({ file: f, line: i + 1, score });
240
+ });
241
+ }
242
+ for (const c of candidates.sort((a, b) => b.score - a.score).slice(0, 120))
243
+ add(c.file, c.line, c.line);
244
+ for (const f of routeFiles.slice(0, 8)) {
245
+ if (candidates.some((c) => c.file === f) && windows.has(f)) {
246
+ const at = order.indexOf(f);
247
+ if (at !== -1)
248
+ order.splice(at, 1);
249
+ order.unshift(f);
250
+ }
251
+ }
205
252
  // A page's own parts (its total, its operator card, its pickers) live in the
206
253
  // components it imports from its own folder. Their visible texts and labels
207
254
  // come right after it, ahead of files that only share a word with the doc.
@@ -237,7 +284,7 @@ export function sourceExcerpts(markdown, roots, budget = 50_000) {
237
284
  // A hook or API module says WHERE the app sends each request, often far below
238
285
  // its first lines: every line that makes a call or names a path is kept.
239
286
  for (const f of order) {
240
- if (!/(^|[\\/])(use[A-Z]\w*|api|services?|dataProvider|queries|requests?)([\\/.]|$)/.test(f))
287
+ if (!/(^|[\\/])(use[A-Z]\w*|api|services?|dataProvider|queries|requests?)([\\/.]|$)/.test(f) || ROUTE_FILE.test(relative(process.cwd(), f)))
241
288
  continue;
242
289
  let lines;
243
290
  try {
@@ -37,7 +37,7 @@ const MAP_PATH = 'testforge.md';
37
37
  * that upgrades keeps serving the facts it cached under the old version —
38
38
  * so the new facts never reach a single prompt and the upgrade does nothing.
39
39
  */
40
- const FACTS_VERSION = 5;
40
+ const FACTS_VERSION = 6;
41
41
  /** Where a journey's grounded facts live, mirroring the plan cache layout. */
42
42
  function groundingPath(journey) {
43
43
  return join(journeyDir(journey), 'facts.json');
@@ -22,6 +22,7 @@
22
22
  */
23
23
  import { matchParen } from './validate.js';
24
24
  import { findTestBlocks } from '../generate/test-blocks.js';
25
+ import { isThingName } from '../generate/interface-file.js';
25
26
  const CANNOT = /\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)$/gm;
26
27
  /** Which planned rules have a test, which are explained, which are simply missing. */
27
28
  /**
@@ -30,12 +31,16 @@ const CANNOT = /\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)
30
31
  * kit.distinctRunAccount(). A rule given one of these is still missing.
31
32
  */
32
33
  const NOT_A_REASON = [
34
+ /\b(?:routes?|endpoints?|urls?|paths?|modules?|seams?|tables?|schemas?|columns?|models?|commands?|settings?|keys?|helpers?|fixtures?|request shapes?|response shapes?|payloads?)\b[^.;]*\b(?:unknown|not known|not shown|not in the (?:code facts|excerpts)|not listed|not documented|not named|named nowhere|not exposed|not identified|not available|does not exist|do not exist|not built|not implemented|missing)\b/i,
35
+ /\b(?:cannot|can't|could not) be seeded\b|\b(?:no|not a) (?:seed|helper|route|endpoint|command|table)\b|\bharness (?:has no|cannot|does not)\b|\btestkit (?:has no|cannot|does not|documents only)\b/i,
33
36
  /\b(?:locators?|selectors?|test ?ids?|elements?|controls?|handles?|buttons?|fields?)\b[^.;]*\b(?:unknown|not known|not in the code facts|not listed|not documented|not named|named nowhere|not exposed|not identified)\b/i,
34
37
  /\bno (?:known |documented )?(?:locator|selector|element|control|handle)\b/i,
35
38
  /\bknown (?:[\w-]+ )?(?:locator|selector|element|control|handle)\b/i,
36
39
  /\b(?:second|two|distinct|another|different) (?:\w+ )?(?:customer|account|user|people|person)s?\b/i,
37
40
  ];
38
- export const acceptedReason = (why) => !NOT_A_REASON.some((re) => re.test(why));
41
+ /** The only reasons a rule cannot be tested: an outside service the test cannot drive. */
42
+ const OUTSIDE = /\b(?:stripe|payment (?:provider|processor|gateway)|card (?:network|issuer|authori[sz]ation)|3-?d ?secure|third[- ]party|external (?:service|system|provider|site)|e-?mail|inbox|sms|text message|push notification|google|routing provider|maps? (?:api|provider)|mapbox|tomtom|another app|mobile app|washer app|operator app|native app|another tab|other tab|second tab|two tabs|iframe|captcha|clock|real time|time passing|wait(?:ing)? (?:\d+|hours|days))\b/i;
43
+ export const acceptedReason = (why) => OUTSIDE.test(why) && !NOT_A_REASON.some((re) => re.test(why));
39
44
  export function ruleCoverage(content, planned) {
40
45
  const tested = new Set(findTestBlocks(content).flatMap((b) => b.rules));
41
46
  const cannotTest = [];
@@ -184,6 +189,9 @@ export function testProblems(content, opts = {}) {
184
189
  // Q10 — the name an element is found by comes from the screen's source or the doc.
185
190
  for (const m of code.matchAll(/\bkit\.find\(\s*\w+\s*,\s*['"`][\w-]+['"`]\s*,\s*/g)) {
186
191
  const rest = code.slice(m.index + m[0].length);
192
+ // A name the product gives later: need('…') is the place for it.
193
+ if (/^need\(\s*['"`]/.test(rest))
194
+ continue;
187
195
  const v = /^([A-Za-z_$][\w$]*)\s*[,)]/.exec(rest)?.[1];
188
196
  // A name from a list written out in the test (a parametrised loop, a const) is fine.
189
197
  const writtenOut = v && (new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
@@ -197,6 +205,11 @@ export function testProblems(content, opts = {}) {
197
205
  add(`finds an element by /${re[1]}/, which matches no text in the app's source or the doc: use the name the screen really shows (the CODE FACTS give the component's text)`);
198
206
  }
199
207
  }
208
+ // Q13 — need() asks for the name of a thing, never for data the test should make.
209
+ const data = [...code.matchAll(/\bneed\(\s*(['"`])([^'"`\n]+)\1/g)].map((m) => m[2]).filter((n) => !isThingName(n));
210
+ if (data.length) {
211
+ add(`asks need() for ${data.slice(0, 3).map((n) => `"${n}"`).join(', ')}, which is data, not the name of a thing: the test makes its own ids, payloads and records; need() is only for "… URL", "… command", "… table", "… field", "… label"`);
212
+ }
200
213
  // Q11 — a logged-out state reached by wiping storage: the testkit's login seed puts the session back.
201
214
  if (/kit\.login\(/.test(code) && /\b(?:clearCookies\s*\(|(?:localStorage|sessionStorage)\.clear\s*\(|storageState\s*\(\s*\{\s*\}|removeItem\s*\()/.test(code)) {
202
215
  add("logs out by clearing storage or cookies after kit.login: the testkit's login seed restores the session — log out through the app's own Log Out control");
@@ -287,7 +300,7 @@ export function harnessSteps(harnessSource) {
287
300
  export function describeProblems(problems, coverage) {
288
301
  const lines = [];
289
302
  if (coverage?.missing.length) {
290
- lines.push(`rules with no test: ${coverage.missing.join(', ')} — write a test for each, or a "// cannot-test: <id> — <the state this area cannot set up>" line. "The locator is unknown" is not such a state (find it with kit.find by the doc's name), and neither is "needs two customers" (skip unless kit.distinctRunAccount())`);
303
+ lines.push(`rules with no test: ${coverage.missing.join(', ')} — write a test for each from the doc. A route, label, command, table or field the code facts do not show comes from need('<plain name>'); data the test needs is made by the test; two customers is a skip unless kit.distinctRunAccount(). "// cannot-test: <id> — <service>" is only for an outside service the test cannot drive (a payment provider's inside, an email inbox, another app, the server's clock)`);
291
304
  }
292
305
  for (const p of problems)
293
306
  lines.push(`"${p.title}"${p.rules.length ? ` (${p.rules.join(', ')})` : ''} ${p.reason}`);
@@ -16,6 +16,8 @@
16
16
  * the common failure — prose leaking into the file — without
17
17
  * false-rejecting valid TypeScript.
18
18
  */
19
+ import { isThingName } from '../generate/interface-file.js';
20
+ import { findTestBlocks } from '../generate/test-blocks.js';
19
21
  import { execFileSync } from 'node:child_process';
20
22
  import { writeFileSync, mkdtempSync, rmSync, existsSync } from 'node:fs';
21
23
  import { join, dirname, resolve } from 'node:path';
@@ -169,6 +171,14 @@ export function checkTestGates(content, _ext, ctx) {
169
171
  if (ctx.forbidPlaceholders && pendingDeclarations(content) > 0) {
170
172
  return 'placeholder tests (test.fixme / test.skip / test.todo declarations) are not written: leave a case with no outcome to check out of the file';
171
173
  }
174
+ // F23 — need() asks for the name of a thing, anywhere in the file (a constant at
175
+ // the top too), never for data the test should make itself.
176
+ if (ctx.forbidPlaceholders) {
177
+ const data = [...content.matchAll(/\bneed\(\s*(['"`])([^'"`\n]+)\1/g)].map((m) => m[2]).filter((n) => !isThingName(n));
178
+ if (data.length) {
179
+ return `asks need() for ${[...new Set(data)].slice(0, 3).map((n) => `"${n}"`).join(', ')}, which is data, not the name of a thing: the test makes its own ids, payloads and records; need() is only for "… URL", "… command", "… table", "… field", "… label"`;
180
+ }
181
+ }
172
182
  // F22 — a fresh file exists to test its rules: one with no test at all says
173
183
  // nothing, and passes. It goes back once, told to stub what it cannot create.
174
184
  if (ctx.forbidPlaceholders && ctx.framework === 'playwright' && !/\b(?:test|it)(?:\.only)?\s*\(\s*['"`]/.test(stripPendingBlocks(content))) {
@@ -642,29 +652,18 @@ export function longTimeoutsWithoutReason(content, capMs) {
642
652
  });
643
653
  return out;
644
654
  }
645
- /** Each test's criterion: the first "// oracle:" line inside it, with its tier. */
655
+ /** Each test's criterion: the first "// oracle:" line above or inside it, with its tier. */
646
656
  function testCriteria(content) {
647
- const lines = content.split('\n');
648
- const starts = [];
649
- lines.forEach((l, i) => { if (/^\s*(?:test|it)\s*\(/.test(l))
650
- starts.push(i); });
651
- return starts.map((start, n) => {
652
- const end = starts[n + 1] ?? lines.length;
653
- // The oracle comment sits right above the test or inside it.
654
- const above = [];
655
- for (let i = start - 1; i > (starts[n - 1] ?? -1) && /^\s*\/\//.test(lines[i] ?? ''); i--)
656
- above.push(i);
657
- for (const i of [...above, ...Array.from({ length: end - start }, (_, k) => start + k)]) {
658
- const m = /\/\/\s*oracle:\s*(.+)$/.exec(lines[i]);
659
- if (m) {
660
- const tier = (/\[(spec|baseline|observed)\]/i.exec(m[1])?.[1]?.toLowerCase() ?? 'untagged');
661
- // The criterion is the rule id when the oracle names one, else the quoted rule, else the oracle text.
662
- const id = /^\s*(R\d+)\b/.exec(m[1])?.[1];
663
- const quoted = /["“]([^"”]{12,})["”]/.exec(m[1]);
664
- return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
665
- }
666
- }
667
- return { criterion: '', tier: 'untagged' };
657
+ // findTestBlocks also knows a file's own names for test (const maybe = … ? test : test.skip).
658
+ return findTestBlocks(content).map((b) => {
659
+ const m = /\/\/\s*oracle:\s*(.+)$/m.exec(content.slice(b.start, b.end));
660
+ if (!m)
661
+ return { criterion: '', tier: 'untagged' };
662
+ const tier = (/\[(spec|baseline|observed)\]/i.exec(m[1])?.[1]?.toLowerCase() ?? 'untagged');
663
+ // The criterion is the rule id when the oracle names one, else the quoted rule, else the oracle text.
664
+ const id = /^\s*(R\d+)\b/.exec(m[1])?.[1];
665
+ const quoted = /["“]([^"”]{12,})["”]/.exec(m[1]);
666
+ return { criterion: (id ?? (quoted ? quoted[1] : m[1])).trim().toLowerCase(), tier };
668
667
  });
669
668
  }
670
669
  /** How many tests of a file check each criterion (tests with no oracle are not counted). */
@@ -241,36 +241,39 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
241
241
  ` (change decoded bytes, not an encoded character), and the valid control`,
242
242
  ` is asserted too where you can.`,
243
243
  ``,
244
- `USE THE TESTKIT'S HELPERS, NEVER YOUR OWN COPY. A helper written into several`,
245
- `files diverges, and one mistake in it fails every file at once.`,
246
- ` - kit.settledClick(locator) for a button inside a pop-up or a form that`,
247
- ` grows while you fill it (payment, address, any modal with a scrollbar);`,
248
- ` kit.fillFrameInputs(page, framePrefix, { name: value }) for inputs inside`,
249
- ` an embedded iframe; kit.watchDialogs(page) to accept and record alerts.`,
250
- ` - Anything else two files of this journey would both need belongs in the`,
251
- ` testkit, not copied into each file. Small one-line locators are fine.`,
252
- ``,
253
- `A LATER STEP OF A FLOW IS OPENED THROUGH THE TESTKIT. Never page.goto() a`,
254
- `later step of a multi-step flow by its URL alone: call`,
255
- `kit.openStep(page, step, input) with a step name the TESTKIT lists for it`,
256
- `(no other name exists), and skip with its reason when it does not resolve`,
257
- `true. A screen after the last listed step is reached from that step through`,
258
- `the UI. Only a test that checks the guard itself (what a visitor who`,
259
- `skipped the earlier steps sees) opens the bare URL.`,
260
- ``,
261
- `kit.find RETURNS A PROMISE. Always \`const pay = await kit.find(page, 'button',`,
262
- `'Pay')\` first, then use pay. Never chain on kit.find(...) (.first(), .click()),`,
263
- `never keep it in a variable without await, and never pass it to expect()`,
264
- `without await.`,
265
- ``,
266
- `kit.login PREPARES ONE PAGE. It sets the session for the page it is given and`,
267
- `restores it when that page loads again. So:`,
268
- ` - log out through the app's own Log Out control, never by clearing storage or`,
269
- ` cookies (the restored session comes straight back);`,
270
- ` - a second tab or page is not logged in by it: call kit.login on that page`,
271
- ` too. "Logged in in another tab" — one tab's login reaching another — is not`,
272
- ` something the testkit can set up: such a rule gets a cannot-test line.`,
273
- ``,
244
+ // Browser-only rules: an API journey never reads them.
245
+ ...(isApi ? [] : [
246
+ `USE THE TESTKIT'S HELPERS, NEVER YOUR OWN COPY. A helper written into several`,
247
+ `files diverges, and one mistake in it fails every file at once.`,
248
+ ` - kit.settledClick(locator) for a button inside a pop-up or a form that`,
249
+ ` grows while you fill it (payment, address, any modal with a scrollbar);`,
250
+ ` kit.fillFrameInputs(page, framePrefix, { name: value }) for inputs inside`,
251
+ ` an embedded iframe; kit.watchDialogs(page) to accept and record alerts.`,
252
+ ` - Anything else two files of this journey would both need belongs in the`,
253
+ ` testkit, not copied into each file. Small one-line locators are fine.`,
254
+ ``,
255
+ `A LATER STEP OF A FLOW IS OPENED THROUGH THE TESTKIT. Never page.goto() a`,
256
+ `later step of a multi-step flow by its URL alone: call`,
257
+ `kit.openStep(page, step, input) with a step name the TESTKIT lists for it`,
258
+ `(no other name exists), and skip with its reason when it does not resolve`,
259
+ `true. A screen after the last listed step is reached from that step through`,
260
+ `the UI. Only a test that checks the guard itself (what a visitor who`,
261
+ `skipped the earlier steps sees) opens the bare URL.`,
262
+ ``,
263
+ `kit.find RETURNS A PROMISE. Always \`const pay = await kit.find(page, 'button',`,
264
+ `'Pay')\` first, then use pay. Never chain on kit.find(...) (.first(), .click()),`,
265
+ `never keep it in a variable without await, and never pass it to expect()`,
266
+ `without await.`,
267
+ ``,
268
+ `kit.login PREPARES ONE PAGE. It sets the session for the page it is given and`,
269
+ `restores it when that page loads again. So:`,
270
+ ` - log out through the app's own Log Out control, never by clearing storage or`,
271
+ ` cookies (the restored session comes straight back);`,
272
+ ` - a second tab or page is not logged in by it: call kit.login on that page`,
273
+ ` too. "Logged in in another tab" — one tab's login reaching another — is not`,
274
+ ` something the testkit can set up: such a rule gets a cannot-test line.`,
275
+ ``,
276
+ ]),
274
277
  `DATA A TEST CREATES. Other runs use the same accounts, so what they left`,
275
278
  `behind is already there when this test starts.`,
276
279
  ` - A test that creates a record (a booking, an order, a vehicle, an`,
@@ -315,14 +318,35 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
315
318
  ``,
316
319
  `EVERY RULE GETS A TEST THAT CAN FAIL. Each rule id this file is given gets`,
317
320
  `at least one test, and that test fails when the product breaks the rule.`,
318
- ` - A rule whose STATE this area cannot set up — data that lives in another`,
319
- ` system, a backend-only effect the screen never shows, an external service`,
320
- ` it cannot drive — gets ONE line instead of a test:`,
321
- ` // cannot-test: R7 — <the state this area cannot set up>`,
322
- ` An element, locator or route you were not shown is NEVER that reason:`,
323
- ` find the element with kit.find(page, kind, <the name the doc gives it>),`,
324
- ` which needs nothing from the CODE FACTS. Two different people at once`,
325
- ` is not that reason either: see kit.distinctRunAccount() below.`,
321
+ ` - Every scenario comes from the doc (and the edge, security and user`,
322
+ ` cases you add). What the code has or lacks never removes one: a`,
323
+ ` scenario the code does not support yet is a test that fails until it does.`,
324
+ ` - ONLY a scenario that needs an outside service the test cannot drive (the`,
325
+ ` inside of a payment provider, an email or SMS inbox, a push notification,`,
326
+ ` another app, the server's own clock) gets ONE line instead of a test:`,
327
+ ` // cannot-test: R7 — <the outside service it needs>`,
328
+ ` A route, label, command, table, field or helper you were not shown is`,
329
+ ` NEVER that reason — see NAMES THE PRODUCT GIVES. Two different people at`,
330
+ ` once is not that reason either: see kit.distinctRunAccount() below.`,
331
+ ``,
332
+ `NAMES THE PRODUCT GIVES. The doc says WHAT happens; the product names HOW to`,
333
+ `reach it — a route, a button's label, a command, a table or a field.`,
334
+ ` - When the CODE FACTS show that name, use it as written there.`,
335
+ ` - When they do not (the code is not built yet, or the facts missed it),`,
336
+ ` write the test anyway and take the name from need('<plain name>'):`,
337
+ ` const { need } = require('@activate-core/test-forge/reporters/interface.cjs');`,
338
+ ` const ORDER_SUMMARY = need('order summary URL');`,
339
+ ` Ask only for the NAME of a thing, ending in what it is: "… URL",`,
340
+ ` "… command", "… table", "… field", "… button label", "… status value",`,
341
+ ` "… query parameter". Never for data — ids, payloads, amounts, durations,`,
342
+ ` example records: the test makes those itself. A route the CODE FACTS`,
343
+ ` list is used as written, never asked for. Reuse a name from NAMES`,
344
+ ` ALREADY ASKED FOR exactly, so one thing has one name. The developer`,
345
+ ` fills each value later in testforge/interface.json. Never guess a value.`,
346
+ ` - Data a scenario needs (operators with hours, a disputed booking) is made`,
347
+ ` by the test itself — through the product's API, or the database the`,
348
+ ` TESTKIT connects to — with table and field names from the CODE FACTS or`,
349
+ ` need('<plain name>').`,
326
350
  ` Never leave a rule out silently, and never write a test that passes`,
327
351
  ` whatever the product does in its place.`,
328
352
  ` - Assert the ONE outcome the rule expects. Never a regex or a condition`,
@@ -352,8 +376,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
352
376
  `exports, and the JOURNEY block holds exactly the rules this file tests`,
353
377
  `with their context. Those are the MECHANICS: HOW to reach and drive`,
354
378
  `the behaviour. Never invent a test id, route, request field or helper`,
355
- `that appears nowhere below. An element is found by the name the doc`,
356
- `gives it, with kit.find. But the EXPECTED OUTCOME you`,
379
+ `that appears nowhere below: what is not here comes from need('<plain`,
380
+ `name>'). An element is found with kit.find. But the EXPECTED OUTCOME you`,
357
381
  ]
358
382
  : grounding
359
383
  ? [
@@ -549,7 +573,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
549
573
  ` establish is left out, never a note to finish later.`,
550
574
  ``,
551
575
  // A web file's slice already names its screens; the data-layer map is for API files.
552
- ...(memory && !isApi ? [] : [`--- PROJECT MAP (data layer) ---`, map, ``]),
576
+ // With the journey's code facts the map only repeats them; a web file's slice
577
+ // already names its screens. The map goes only where there is nothing else.
578
+ ...(memory && (!isApi || grounding) ? [] : [`--- PROJECT MAP (data layer) ---`, map, ``]),
553
579
  // The same for every file of the journey, so it sits in the cached shared part.
554
580
  ...(memory ? [`--- TESTKIT (the shared harness's exports) ---`, memory.helpers, ``] : []),
555
581
  // Read from the project's real source once per journey, before any file is
@@ -569,6 +595,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
569
595
  ``,
570
596
  ]
571
597
  : []),
598
+ // The names other files of this run already asked for: one thing, one name.
599
+ ...(memory?.names?.length ? [`--- NAMES ALREADY ASKED FOR (reuse exactly) ---`, memory.names.join('\n'), ``] : []),
572
600
  ...(memory
573
601
  ? [
574
602
  `--- JOURNEY (only the part this file tests; other files test the rest) ---`,
@@ -598,8 +626,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
598
626
  `Use this exact describe() title: "${focus.title}".`,
599
627
  ...(memory?.rules.length
600
628
  ? [
601
- `Test exactly these rules: ${memory.rules.join(', ')}. EVERY one gets a test or a`,
602
- `"// cannot-test: <id> — <why>" line. Name the rule id AND its`,
629
+ `Test exactly these rules: ${memory.rules.join(', ')}. EVERY one gets a test (a name`,
630
+ `the product has not given yet comes from need()), or, only for an outside service,`,
631
+ `a "// cannot-test: <id> — <the service>" line. Name the rule id AND its`,
603
632
  `tag in each test's oracle comment, e.g. "// oracle: ${memory.rules[0]} [spec] …":`,
604
633
  `the tag is how a failure is later graded as a defect or a finding.`,
605
634
  ]
@@ -625,7 +654,8 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
625
654
  ` condition that counts or looks for an element) — that is a failure to assert;`,
626
655
  `- has a test with no expect(), an expect() only inside an if/else/catch, or more`,
627
656
  ` than ${maxPerRule()} tests for one rule;`,
628
- `- leaves a rule with neither a test nor a "// cannot-test: <id> — <why>" line;`,
657
+ `- leaves a rule with neither a test nor a "// cannot-test: <id> — <outside service>" line,`,
658
+ ` or gives a missing route, label, table or command as the reason (use need());`,
629
659
  `- asserts a regex that accepts the good and the bad outcome, compares the page's`,
630
660
  ` whole text, or asserts the words its own stub returns;`,
631
661
  `- uses kit.find(...) without await, or a kit.openStep step the TESTKIT does not list;`,
@@ -648,7 +678,7 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
648
678
  * repair, and the file's own part (its facts, its slice, its task) on stdin.
649
679
  */
650
680
  export function splitShared(prompt) {
651
- const at = ['\n--- CODE FACTS', '\n--- JOURNEY (only'].map((m) => prompt.indexOf(m)).filter((i) => i > 0);
681
+ const at = ['\n--- CODE FACTS', '\n--- NAMES ALREADY ASKED FOR', '\n--- JOURNEY (only'].map((m) => prompt.indexOf(m)).filter((i) => i > 0);
652
682
  if (!at.length)
653
683
  return { system: '', message: prompt };
654
684
  const cut = Math.min(...at);
@@ -16,7 +16,13 @@ import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs';
16
16
  import { basename, join } from 'node:path';
17
17
  import { isClaude } from '../project/config.js';
18
18
  import { RUNS_DIR } from '../paths.js';
19
- const total = { calls: 0, ms: 0, inChars: 0, outChars: 0, tokens: 0, costUsd: 0 };
19
+ const total = { calls: 0, ms: 0, inChars: 0, outChars: 0, tokens: 0, cacheRead: 0, costUsd: 0 };
20
+ /** The same, per step (write, grounding, seed, harness …), so a run says where its cost went. */
21
+ const byStage = new Map();
22
+ /** Each step's calls, tokens and cost, the most expensive first. */
23
+ export function usageByStage() {
24
+ return [...byStage].map(([stage, u]) => ({ stage, ...u })).sort((a, b) => b.costUsd - a.costUsd || b.tokens - a.tokens);
25
+ }
20
26
  export function noteAiCall(inChars, outChars, ms) {
21
27
  total.calls += 1;
22
28
  total.ms += ms;
@@ -31,7 +37,7 @@ export function aiUsage() {
31
37
  export function usageSince(start, now = aiUsage()) {
32
38
  return {
33
39
  calls: now.calls - start.calls, ms: now.ms - start.ms, inChars: now.inChars - start.inChars, outChars: now.outChars - start.outChars,
34
- tokens: now.tokens - start.tokens, costUsd: now.costUsd - start.costUsd,
40
+ tokens: now.tokens - start.tokens, cacheRead: now.cacheRead - start.cacheRead, costUsd: now.costUsd - start.costUsd,
35
41
  };
36
42
  }
37
43
  export const approxTokens = (u) => Math.round((u.inChars + u.outChars) / 4);
@@ -41,7 +47,7 @@ export function describeUsage(u) {
41
47
  // The CLI's own count includes what the AI read with its tools and every
42
48
  // turn's re-sent context; characters ÷ 4 sees only our prompt and its answer.
43
49
  const tokens = u.tokens
44
- ? `${u.tokens.toLocaleString('en-GB')} tokens, $${u.costUsd.toFixed(2)}`
50
+ ? `${u.tokens.toLocaleString('en-GB')} tokens (${(u.tokens - u.cacheRead).toLocaleString('en-GB')} new, ${u.cacheRead.toLocaleString('en-GB')} read from cache), $${u.costUsd.toFixed(2)}`
45
51
  : `~${approxTokens(u).toLocaleString('en-GB')} tokens (estimate)`;
46
52
  return `${u.calls} AI call(s), ${time}, ${tokens}`;
47
53
  }
@@ -112,8 +118,14 @@ export function traceCall(prompt, raw, started, error) {
112
118
  const result = events.find((e) => e.type === 'result');
113
119
  const text = events.length ? String(result?.result ?? '') : raw;
114
120
  const ru = result?.usage ?? {};
115
- total.tokens += ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'].reduce((n, k) => n + (Number(ru[k]) || 0), 0);
116
- total.costUsd += Number(result?.total_cost_usd) || 0;
121
+ const tokens = ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'].reduce((n, k) => n + (Number(ru[k]) || 0), 0);
122
+ const cost = Number(result?.total_cost_usd) || 0;
123
+ total.tokens += tokens;
124
+ total.cacheRead += Number(ru.cache_read_input_tokens) || 0;
125
+ total.costUsd += cost;
126
+ const stage = STAGES.find(([re]) => re.test(prompt))?.[1] ?? (/^Write \d+ test files\./.test(prompt) ? 'write' : 'other');
127
+ const s = byStage.get(stage) ?? { calls: 0, tokens: 0, costUsd: 0, ms: 0 };
128
+ byStage.set(stage, { calls: s.calls + 1, tokens: s.tokens + tokens, costUsd: s.costUsd + cost, ms: s.ms + (Date.now() - started) });
117
129
  if (!TRACE_DIR)
118
130
  return text;
119
131
  const tools = events
@@ -125,7 +137,6 @@ export function traceCall(prompt, raw, started, error) {
125
137
  return `${c.name} ${String(i.file_path ?? i.pattern ?? i.path ?? i.command ?? JSON.stringify(i)).slice(0, 200)}`;
126
138
  });
127
139
  const seq = String(++traceSeq).padStart(3, '0');
128
- const stage = STAGES.find(([re]) => re.test(prompt))?.[1] ?? 'ai';
129
140
  const file = /Save this file at EXACTLY: (\S+)/.exec(prompt)?.[1];
130
141
  const name = `${seq}-${stage}${file ? `-${basename(file).replace(/\.[^.]+$/, '').replace(/\.test$/, '')}` : ''}`;
131
142
  try {
@@ -0,0 +1,70 @@
1
+ /**
2
+ * testforge — testforge/interface.json, the names tests need from the product.
3
+ *
4
+ * Tests ask for a route, a label, a command or a table by a plain name:
5
+ * need('order summary URL'). This file maps each name to the product's value.
6
+ * TestForge fills what the code already has; what it does not have stays
7
+ * blank for the developer. A value somebody wrote is never overwritten.
8
+ */
9
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
10
+ import { dirname, join } from 'node:path';
11
+ import { STATE_DIR } from '../paths.js';
12
+ export const INTERFACE_PATH = join(STATE_DIR, 'interface.json');
13
+ const NEED = /\bneed\(\s*(['"`])([^'"`\n]+)\1\s*\)/g;
14
+ /** A name of a thing (ends in what it is), not a piece of data. */
15
+ export const NAME_KIND = /\b(?:url|urls|path|prefix|endpoint|route|command|table|column|field|fields|label|key|setting|param|parameter|parameters|header|status value|value name|event|queue|job|model|class|method|role|permission)$/i;
16
+ export const isThingName = (n) => NAME_KIND.test(n.trim());
17
+ /** The names a test file asks for. */
18
+ export function neededNames(content) {
19
+ return [...new Set([...content.matchAll(NEED)].map((m) => m[2].trim()))];
20
+ }
21
+ export function readInterface(path = INTERFACE_PATH) {
22
+ try {
23
+ const v = JSON.parse(readFileSync(path, 'utf-8'));
24
+ return v && typeof v === 'object' && !Array.isArray(v) ? v : {};
25
+ }
26
+ catch {
27
+ return {};
28
+ }
29
+ }
30
+ /** Names whose value is still blank. */
31
+ export function blankNames(names, values = readInterface()) {
32
+ return names.filter((n) => !String(values[n] ?? '').trim());
33
+ }
34
+ /**
35
+ * Drop blank names no test asks for any more (a value somebody wrote stays).
36
+ * Returns how many were dropped.
37
+ */
38
+ export function pruneInterface(stillAsked, path = INTERFACE_PATH) {
39
+ if (!existsSync(path))
40
+ return 0;
41
+ const values = readInterface(path);
42
+ const keep = Object.entries(values).filter(([n, v]) => String(v ?? '').trim() || stillAsked.has(n));
43
+ const dropped = Object.keys(values).length - keep.length;
44
+ if (dropped)
45
+ writeFileSync(path, `${JSON.stringify(Object.fromEntries(keep), null, 2)}\n`, 'utf-8');
46
+ return dropped;
47
+ }
48
+ /**
49
+ * Add the names tests ask for, with the value the code gave (or blank).
50
+ * Existing values are kept. Returns the names still blank.
51
+ */
52
+ export function recordInterface(found, path = INTERFACE_PATH) {
53
+ const values = readInterface(path);
54
+ let changed = false;
55
+ for (const [name, value] of Object.entries(found)) {
56
+ if (String(values[name] ?? '').trim())
57
+ continue;
58
+ const next = (value ?? '').trim();
59
+ if (values[name] === next)
60
+ continue;
61
+ values[name] = next;
62
+ changed = true;
63
+ }
64
+ if (changed || !existsSync(path)) {
65
+ mkdirSync(dirname(path), { recursive: true });
66
+ const sorted = Object.fromEntries(Object.entries(values).sort(([a], [b]) => a.localeCompare(b)));
67
+ writeFileSync(path, `${JSON.stringify(sorted, null, 2)}\n`, 'utf-8');
68
+ }
69
+ return blankNames(Object.keys(found), values);
70
+ }
@@ -28,10 +28,11 @@ import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
29
  import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, writeTestsWithAI, } from './ai-generator.js';
30
30
  import { completeFile, fileIsComplete } from './complete-file.js';
31
+ import { INTERFACE_PATH, isThingName, neededNames, pruneInterface, readInterface, recordInterface } from './interface-file.js';
31
32
  import { harnessSteps } from '../gates/test-quality.js';
32
33
  import { normalisePending } from './pending.js';
33
34
  import { packageRoot } from '../run/run-tests.js';
34
- import { pendingReason, withPendingStamp } from '../run/readiness.js';
35
+ import { pendingReason, testFilesIn, withPendingStamp } from '../run/readiness.js';
35
36
  import { groundJourney } from '../docs/grounding.js';
36
37
  import { generateSeed } from './seed-generator.js';
37
38
  import { dataAccess, writeRequiredData } from './required-data.js';
@@ -519,6 +520,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
519
520
  const steps = harnessPath && existsSync(harnessPath) ? harnessSteps(readFileSync(harnessPath, 'utf-8')) : [];
520
521
  // What each file really tests once written: the memory and the report come from this, not the plan.
521
522
  const written = [];
523
+ // need() names asked so far (the project's and this run's): handed to every later file.
524
+ const askedNames = new Set(Object.keys(readInterface()).filter(isThingName));
522
525
  const finish = async (content, pf, filePath) => {
523
526
  if (!memory || !pf.rules?.length)
524
527
  return content;
@@ -529,6 +532,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
529
532
  check: (c) => (isSyntaxValid(c, ext) ? genGate(c, filePath) : syntaxReason(c, ext)),
530
533
  });
531
534
  written.push({ slug: pf.slug, tested: done.tested, cannotTest: done.cannotTest, missing: done.missing, dropped: done.dropped });
535
+ for (const n of neededNames(done.content))
536
+ askedNames.add(n);
532
537
  for (const n of done.notes)
533
538
  warnings.push(`${pf.slug}: ${n}`);
534
539
  return done.content;
@@ -598,7 +603,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
598
603
  return { facts };
599
604
  const ids = pf.rules?.length ? pf.rules : allTested;
600
605
  const slice = journeySlice(memory, ids);
601
- return { mem: { journey: slice.text, helpers, rules: ids }, facts: factsSlice(facts, slice.controls) };
606
+ return { mem: { journey: slice.text, helpers, rules: ids, names: [...askedNames].sort() }, facts: factsSlice(facts, slice.controls) };
602
607
  };
603
608
  const plannedSlugs = planned.map((p) => p.slug);
604
609
  // Read the journey's real code ONCE, before any file is written, and hand
@@ -752,6 +757,14 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
752
757
  const sum = recordWritten(journey, written);
753
758
  console.log(` ${journey.id}: ${sum}`);
754
759
  }
760
+ // The names the tests ask the product for: listed for the developer, kept blank until filled.
761
+ const asked = [...new Set(generated.flatMap((g) => (existsSync(g.path) ? neededNames(readFileSync(g.path, 'utf-8')) : [])))];
762
+ if (asked.length) {
763
+ const blank = recordInterface(Object.fromEntries(asked.map((n) => [n, undefined])));
764
+ if (blank.length) {
765
+ warnings.push(`${blank.length} name(s) the product has not given yet — fill them in ${INTERFACE_PATH}; until then \`testforge run\` holds those files back: ${blank.slice(0, 8).join(', ')}${blank.length > 8 ? ', …' : ''}`);
766
+ }
767
+ }
755
768
  const testFiles = generated.map((g) => ({
756
769
  slug: g.slug,
757
770
  path: g.path,
@@ -780,11 +793,16 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
780
793
  else if (useAI && !force && existsSync(seedPath)) {
781
794
  seedStatus = 'ok'; // kept, like an existing test file
782
795
  }
796
+ else if (useAI && pending) {
797
+ // Not live yet: there is nothing to seed for, and its tests make their own data.
798
+ seedStatus = 'none';
799
+ }
783
800
  else if (useAI) {
784
- let seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
801
+ const seedFacts = await grounding();
802
+ let seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt);
785
803
  // A usage limit is not a failed seed: wait for the AI like every other step, then try again.
786
804
  while (seed === null && aiUnavailableReason() && waitForLimit && (await waitForAi(config))) {
787
- seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
805
+ seed = generateValidated(() => generateSeed(markdown, seedPath, seedFacts), seedExt);
788
806
  }
789
807
  if (seed !== null) {
790
808
  mkdirSync(dirname(seedPath), { recursive: true });
@@ -893,6 +911,13 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
893
911
  }
894
912
  saveCache(CACHE_PATH, cache);
895
913
  writeCoverageManifest(results);
914
+ // Blank names no test asks for any more (a rewritten file stopped using them) go.
915
+ if (useAI && existsSync(INTERFACE_PATH)) {
916
+ const stillAsked = new Set(Object.values(config.testOutput ?? {}).flatMap((d) => testFilesIn(d)).flatMap((f) => neededNames(readFileSync(f, 'utf-8'))));
917
+ const dropped = pruneInterface(stillAsked);
918
+ if (dropped)
919
+ console.log(` ${INTERFACE_PATH}: removed ${dropped} blank name(s) no test asks for any more`);
920
+ }
896
921
  return results;
897
922
  }
898
923
  /**
@@ -15,7 +15,7 @@
15
15
  * named in the map (e.g. process.env.DB_URL), not a literal string.
16
16
  */
17
17
  import { readFileSync, existsSync } from 'node:fs';
18
- import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
18
+ import { loadConfig, splitCommand, withNoTools, withReadOnlyTools } from '../project/config.js';
19
19
  import { cleanOutput } from './ai-generator.js';
20
20
  import { runAiSync } from './ai-usage.js';
21
21
  const MAP_PATH = 'testforge.md';
@@ -91,11 +91,14 @@ export function buildSeedPrompt(journeyDoc, seedPath) {
91
91
  ].join('\n');
92
92
  }
93
93
  /** Generate a seed file's content for one journey doc, via the AI. */
94
- export function generateSeed(journeyDoc, seedPath) {
95
- const prompt = buildSeedPrompt(journeyDoc, seedPath);
96
- // Allow read-only exploration so the seed matches the REAL schema
97
- // (models/collections/enums) read from the code; Write stays denied.
98
- const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
94
+ export function generateSeed(journeyDoc, seedPath, facts) {
95
+ // With the journey's code facts in hand the seed is one call with no tools;
96
+ // without them it may read the schema itself (read-only).
97
+ const prompt = facts
98
+ ? `${buildSeedPrompt(journeyDoc, seedPath)}\n\n--- CODE FACTS (the schema, models and routes read from this project) ---\n${facts}\nYou have no tools: use only these facts and the map above. A table or column they do not show is left out of the seed.`
99
+ : buildSeedPrompt(journeyDoc, seedPath);
100
+ const cmd = splitCommand(loadConfig().aiCommand);
101
+ const { program, args } = facts ? withNoTools(cmd) : withReadOnlyTools(cmd);
99
102
  const output = runAiSync(program, args, {
100
103
  input: prompt,
101
104
  encoding: 'utf-8',
@@ -9,10 +9,23 @@
9
9
  import { matchParen } from '../gates/validate.js';
10
10
  const TEST_CALL = /(^|[^\w.])(?:test|it)(?:\.(?:only|fixme|skip|slow|fail))?\s*\(\s*(['"`])((?:(?!\2).)*)\2/gm;
11
11
  const RULE_ID = /\boracle:\s*((?:R\d+[\s,/&]*(?:and\s+)?)+)/g;
12
+ /**
13
+ * Names a file gives to test itself: `const maybe = kit.roleAvailable('run') ?
14
+ * test : test.skip`. A test written as maybe('…', …) is a test all the same.
15
+ */
16
+ function testAliases(content) {
17
+ return [...content.matchAll(/\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*[^;\n]*?\b(?:test|it)(?:\.(?:skip|only|fixme))?\s*(?:[;:\n]|$)/gm)]
18
+ .map((m) => m[1])
19
+ .filter((n) => n !== 'test' && n !== 'it');
20
+ }
12
21
  /** Every test(…) call in the file, describe blocks and hooks excluded. */
13
22
  export function findTestBlocks(content) {
14
23
  const out = [];
15
- for (const m of content.matchAll(TEST_CALL)) {
24
+ const aliases = testAliases(content);
25
+ const calls = aliases.length
26
+ ? new RegExp(`(^|[^\\w.])(?:test|it|${aliases.join('|')})(?:\\.(?:only|fixme|skip|slow|fail))?\\s*\\(\\s*(['"\`])((?:(?!\\2).)*)\\2`, 'gm')
27
+ : TEST_CALL;
28
+ for (const m of content.matchAll(calls)) {
16
29
  const callAt = m.index + m[1].length;
17
30
  const open = content.indexOf('(', callAt);
18
31
  const close = matchParen(content, open);
@@ -29,7 +29,8 @@ import { readFileSync, existsSync } from 'node:fs';
29
29
  import { join, resolve } from 'node:path';
30
30
  const CONFIG_PATH = 'testforge.config.json';
31
31
  const DEFAULT_CONFIG = {
32
- aiCommand: 'claude -p',
32
+ // Sonnet unless a project picks otherwise: the user's own default may be a far pricier model.
33
+ aiCommand: 'claude -p --model sonnet',
33
34
  aiTimeoutMs: 300_000,
34
35
  aiConcurrency: 4,
35
36
  testOutput: {},
@@ -42,7 +42,7 @@ export function buildInitPrompt() {
42
42
  ` "playwright", "lambda": "jest" }.`,
43
43
  `- testExtension: ".test.ts" or ".test.js" — match what existing`,
44
44
  ` tests use, or the project's language (TypeScript vs JavaScript).`,
45
- `- aiCommand: keep it as "claude -p" unless there is a clear reason`,
45
+ `- aiCommand: keep it as "claude -p --model sonnet" unless there is a clear reason`,
46
46
  ` to change it.`,
47
47
  `- project: a short slug for this repo (kebab-case, e.g. the repo folder`,
48
48
  ` name lower-cased). Identifies the project on the TestForge dashboard.`,
@@ -52,7 +52,7 @@ export function buildInitPrompt() {
52
52
  `Write valid JSON only, in this shape:`,
53
53
  ``,
54
54
  `{`,
55
- ` "aiCommand": "claude -p",`,
55
+ ` "aiCommand": "claude -p --model sonnet",`,
56
56
  ` "testOutput": { "<area>": "<path>" },`,
57
57
  ` "framework": { "<area>": "<jest|playwright|...>" },`,
58
58
  ` "seedOutput": "<path>",`,
@@ -11,6 +11,7 @@
11
11
  import { existsSync, readdirSync, readFileSync } from 'node:fs';
12
12
  import { join } from 'node:path';
13
13
  import { usedLabels } from '../gates/validate.js';
14
+ import { blankNames, neededNames } from '../generate/interface-file.js';
14
15
  /** A name as the code would write it: no required-field star, no closing punctuation. */
15
16
  const bare = (n) => n.replace(/\s*\*\s*$/, '').replace(/[\s.:!?…]+$/, '').replace(/\s+/g, ' ').trim();
16
17
  /**
@@ -80,8 +81,10 @@ export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
80
81
  const content = readFileSync(file, 'utf-8');
81
82
  // The doc says the feature is not live: held back whatever the code has.
82
83
  const pending = PENDING_STAMP.exec(content)?.[1];
83
- if (pending)
84
- return { file, state: 'not-built', missing: [pending.trim()], renamed: [] };
84
+ // A name the product has not given yet (blank in testforge/interface.json).
85
+ const blank = blankNames(neededNames(content)).map((n) => `needs ${n}`);
86
+ if (pending || blank.length)
87
+ return { file, state: 'not-built', missing: [...(pending ? [pending.trim()] : []), ...blank], renamed: [] };
85
88
  const { names, pages } = namesInTest(content);
86
89
  const missing = [];
87
90
  const renamed = [];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.1.1",
3
+ "version": "2.2.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -0,0 +1,46 @@
1
+ /**
2
+ * testforge — the names a test needs from the product, kept outside the test.
3
+ *
4
+ * A test is written from the journey doc. Where it needs something only the
5
+ * product can say — a route, a button's label, a command, a table or field —
6
+ * and the code has it, TestForge writes the value into testforge/interface.json.
7
+ * Where the code does not have it yet, the entry is left blank for the
8
+ * developer to fill once it exists:
9
+ *
10
+ * const { need } = require('@activate-core/test-forge/reporters/interface.cjs');
11
+ * const ORDER_SUMMARY = need('order summary URL');
12
+ *
13
+ * `testforge run` holds back a file while one of its names is blank, and
14
+ * lists the names it waits for. Run anyway (--all), a blank name fails the
15
+ * test with what to fill, never with a guess.
16
+ */
17
+ const fs = require('fs');
18
+ const path = require('path');
19
+
20
+ function interfacePath() {
21
+ for (let dir = process.cwd(); ; dir = path.dirname(dir)) {
22
+ const p = path.join(dir, 'testforge', 'interface.json');
23
+ if (fs.existsSync(p)) return p;
24
+ if (path.dirname(dir) === dir) return path.join(process.cwd(), 'testforge', 'interface.json');
25
+ }
26
+ }
27
+
28
+ let cache;
29
+ function values() {
30
+ if (cache) return cache;
31
+ try {
32
+ cache = JSON.parse(fs.readFileSync(interfacePath(), 'utf-8'));
33
+ } catch {
34
+ cache = {};
35
+ }
36
+ return cache;
37
+ }
38
+
39
+ /** The product's value for `name`, or an error naming what to fill. */
40
+ function need(name) {
41
+ const v = values()[name];
42
+ if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
43
+ throw new Error(`"${name}" is not filled in testforge/interface.json: the product does not have it yet, or nobody has written it down. Fill it, then run again.`);
44
+ }
45
+
46
+ module.exports = { need };