@activate-core/test-forge 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +98 -34
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +51 -4
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +51 -24
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +68 -5
  10. package/dist/core/docs/review.js +81 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +56 -0
  13. package/dist/core/gates/coverage.js +12 -19
  14. package/dist/core/gates/validate.js +277 -4
  15. package/dist/core/generate/ai-generator.js +243 -96
  16. package/dist/core/generate/ai-session.js +8 -0
  17. package/dist/core/generate/ai-usage.js +150 -0
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/defect-review.js +213 -20
  20. package/dist/core/generate/doc-diff.js +139 -0
  21. package/dist/core/generate/harness.js +52 -32
  22. package/dist/core/generate/journey-memory.js +325 -0
  23. package/dist/core/generate/plan-cache.js +114 -7
  24. package/dist/core/generate/required-data.js +75 -0
  25. package/dist/core/generate/review-journey.js +104 -0
  26. package/dist/core/generate/run-generate.js +193 -80
  27. package/dist/core/generate/seed-generator.js +2 -2
  28. package/dist/core/generate/suite-planner.js +138 -25
  29. package/dist/core/generate/test-blocks.js +84 -0
  30. package/dist/core/generate/update-journey.js +115 -0
  31. package/dist/core/generate/verify.js +9 -4
  32. package/dist/core/paths.js +90 -13
  33. package/dist/core/project/config.js +25 -2
  34. package/dist/core/project/env-diff.js +122 -0
  35. package/dist/core/project/env-template.js +3 -2
  36. package/dist/core/project/init.js +2 -2
  37. package/dist/core/project/map.js +2 -2
  38. package/dist/core/run/ci.js +94 -0
  39. package/dist/core/run/clean.js +111 -0
  40. package/dist/core/run/readiness.js +119 -0
  41. package/dist/core/run/report.js +2 -0
  42. package/dist/core/run/run-tests.js +23 -1
  43. package/package.json +1 -1
  44. package/reporters/_shared.cjs +24 -3
  45. package/reporters/jest-reporter.cjs +3 -1
  46. package/reporters/playwright-reporter.cjs +3 -1
  47. package/reporters/run-data.cjs +66 -0
  48. package/reporters/ui-helpers.cjs +133 -0
  49. package/dist/core/generate/verify-journey.js +0 -174
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Where a journey's starting data comes from.
3
+ *
4
+ * A repository that owns its data store gets a generated seed that writes
5
+ * it. One that does not (a web client talking to someone else's API) cannot
6
+ * run such a seed, so it gets no seed code at all: the testkit sets up what
7
+ * the product's API can create, and REQUIRED-DATA.md lists, per journey, what
8
+ * a person has to provide — taken word for word from the journey's own
9
+ * "Test setup" section, never invented.
10
+ */
11
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ /** Packages that only a repository talking to its own data store depends on. */
14
+ const STORE_PACKAGES = /^(?:mysql2?|pg|postgres|mongoose|mongodb|prisma|@prisma\/client|typeorm|sequelize|knex|drizzle-orm|better-sqlite3|sqlite3|ioredis|redis|@mikro-orm\/core|objection)$/;
15
+ const STORE_LIBS = /\b(?:sqlalchemy|django|psycopg2?|pymongo|peewee|activerecord|mongoid|sequel|doctrine\/orm|illuminate\/database|laravel\/framework|gorm|sqlx)\b/i;
16
+ /**
17
+ * "db" when this repository talks to its own data store, so a seed it
18
+ * generates can run; else "api". The config's `dataAccess` decides when set.
19
+ * Otherwise the repository itself is read — its dependencies and whether it
20
+ * has migrations — never the env file, whose variable names can be left over.
21
+ */
22
+ export function dataAccess(config, root = '.') {
23
+ if (config.dataAccess === 'db' || config.dataAccess === 'api')
24
+ return config.dataAccess;
25
+ const read = (f) => { try {
26
+ return readFileSync(join(root, f), 'utf-8');
27
+ }
28
+ catch {
29
+ return '';
30
+ } };
31
+ try {
32
+ const pkg = JSON.parse(read('package.json') || '{}');
33
+ const deps = Object.keys({ ...pkg.dependencies, ...pkg.devDependencies });
34
+ if (deps.some((d) => STORE_PACKAGES.test(d)))
35
+ return 'db';
36
+ }
37
+ catch { /* not a node project, or unreadable package.json */ }
38
+ if (['composer.json', 'requirements.txt', 'pyproject.toml', 'Gemfile', 'go.mod'].some((f) => STORE_LIBS.test(read(f))))
39
+ return 'db';
40
+ if (['migrations', 'database/migrations', 'db/migrate', 'prisma/migrations'].some((d) => existsSync(join(root, d))))
41
+ return 'db';
42
+ return 'api';
43
+ }
44
+ /** The journey's "Test setup" section, heading excluded, or undefined when it has none. */
45
+ export function testSetupSection(markdown) {
46
+ const m = /^##\s+Test setup\s*$/im.exec(markdown);
47
+ if (!m)
48
+ return undefined;
49
+ const rest = markdown.slice(m.index + m[0].length);
50
+ const end = rest.search(/^##\s/m);
51
+ const body = (end === -1 ? rest : rest.slice(0, end)).trim();
52
+ return body || undefined;
53
+ }
54
+ const HEAD = [
55
+ '# Data these tests need',
56
+ '',
57
+ 'This repository does not own its data store, so TestForge writes no seed code for it.',
58
+ 'The testkit creates what the product\'s API can create. Everything below must exist',
59
+ 'in the environment the tests run against; each section is the journey\'s own "Test setup".',
60
+ 'Ids and values go in the env file named in testforge.config.json.',
61
+ '',
62
+ ].join('\n');
63
+ /** Write or replace one journey's section of <seedDir>/REQUIRED-DATA.md. Returns the file path. */
64
+ export function writeRequiredData(seedDir, journeyId, journeyName, markdown) {
65
+ const path = join(seedDir, 'REQUIRED-DATA.md');
66
+ const open = `<!-- journey: ${journeyId} -->`, close = `<!-- /journey: ${journeyId} -->`;
67
+ const body = testSetupSection(markdown) ?? '_The journey doc has no "Test setup" section: nothing is listed._';
68
+ const block = `${open}\n## ${journeyName}\n\n${body}\n${close}`;
69
+ let text = existsSync(path) ? readFileSync(path, 'utf-8') : HEAD;
70
+ const at = text.indexOf(open), end = text.indexOf(close);
71
+ text = at !== -1 && end > at ? text.slice(0, at) + block + text.slice(end + close.length) : `${text.trimEnd()}\n\n${block}\n`;
72
+ mkdirSync(seedDir, { recursive: true });
73
+ writeFileSync(path, text, 'utf-8');
74
+ return path;
75
+ }
@@ -0,0 +1,104 @@
1
+ /**
2
+ * `testforge review` — judge a journey's failures again, without the verify
3
+ * and repair pass that produced them.
4
+ *
5
+ * Verify keeps its last results per journey. Review reads them (or runs the
6
+ * journey once, without repairing anything, when there are none or --rerun
7
+ * asks for it), reviews every failure — a claimed defect against its claim,
8
+ * an unclaimed one against the test's own oracle — and writes the verdicts
9
+ * where the reporters pick them up for the dashboard.
10
+ */
11
+ import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
12
+ import { basename, extname, join } from 'node:path';
13
+ import { RUNS_DIR } from '../paths.js';
14
+ import { findMarkdownFiles } from '../find-files.js';
15
+ import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
16
+ import { loadConfig, testDirForType } from '../project/config.js';
17
+ import { matchesJourney } from './plan-cache.js';
18
+ import { claimedDefects, verifyRun } from './verify.js';
19
+ import { groupClaims, oracleClaim, reviewDefects } from './defect-review.js';
20
+ const RESULTS = join(RUNS_DIR, 'verify-results');
21
+ const REVIEWS = join(RUNS_DIR, 'defect-review');
22
+ export function saveVerifyResults(journeyId, failures, passes) {
23
+ mkdirSync(RESULTS, { recursive: true });
24
+ const saved = { at: new Date().toISOString(), failures, passes };
25
+ writeFileSync(join(RESULTS, `${journeyId}.json`), JSON.stringify(saved, null, 2), 'utf-8');
26
+ }
27
+ export function loadVerifyResults(journeyId) {
28
+ try {
29
+ return JSON.parse(readFileSync(join(RESULTS, `${journeyId}.json`), 'utf-8'));
30
+ }
31
+ catch {
32
+ return undefined;
33
+ }
34
+ }
35
+ /** The claim a failure is reviewed under: the generator's own, else the test's oracle. */
36
+ export function claimFor(t) {
37
+ let content = '';
38
+ try {
39
+ content = readFileSync(t.file, 'utf-8');
40
+ }
41
+ catch {
42
+ return oracleClaim(t, '');
43
+ }
44
+ return claimedDefects(content).get(t.line) ?? oracleClaim(t, content);
45
+ }
46
+ const testFiles = (dir) => {
47
+ const out = [];
48
+ const walk = (d) => {
49
+ for (const e of readdirSync(d, { withFileTypes: true })) {
50
+ const p = join(d, e.name);
51
+ if (e.isDirectory())
52
+ walk(p);
53
+ else if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(e.name))
54
+ out.push(p);
55
+ }
56
+ };
57
+ walk(dir);
58
+ return out;
59
+ };
60
+ export async function runReview(opts = {}) {
61
+ const config = loadConfig();
62
+ const docsDir = opts.docsDir ?? 'docs';
63
+ const out = [];
64
+ for (const file of findMarkdownFiles(docsDir)) {
65
+ const markdown = readFileSync(join(docsDir, file), 'utf-8');
66
+ if (notAJourneyDoc(file, markdown))
67
+ continue;
68
+ const area = /[\\/]/.test(file) ? file.split(/[\\/]/)[0] : undefined;
69
+ const journey = parseJourney(markdown, area, basename(file, extname(file)));
70
+ if (opts.journey && !matchesJourney(file, journey.id, opts.journey))
71
+ continue;
72
+ const testDir = join(testDirForType(config, journey.area), journey.id);
73
+ if (!existsSync(testDir))
74
+ continue;
75
+ let saved = opts.rerun ? undefined : loadVerifyResults(journey.id);
76
+ if (!saved) {
77
+ console.log(` ${journey.id}: running the journey once (no repair) to get its failures…`);
78
+ // A whole journey runs far longer than one file: give it the time a full run takes.
79
+ const files = testFiles(testDir);
80
+ const run = await verifyRun(config, journey.area, files, Math.max(config.verifyTimeoutMs ?? 600_000, files.length * 180_000));
81
+ if (!run.ran || run.infra) {
82
+ out.push({ journey: journey.id, failures: 0, verdicts: {}, skipped: run.infra ?? run.reason ?? 'the run produced no results' });
83
+ continue;
84
+ }
85
+ saveVerifyResults(journey.id, run.failures, run.passes);
86
+ saved = loadVerifyResults(journey.id);
87
+ }
88
+ if (!saved.failures.length) {
89
+ out.push({ journey: journey.id, failures: 0, verdicts: {} });
90
+ continue;
91
+ }
92
+ const groups = groupClaims(saved.failures, claimFor);
93
+ console.log(` ${journey.id}: reviewing ${saved.failures.length} failure(s) in ${groups.length} group(s)…`);
94
+ const reviews = await reviewDefects(journey, groups, saved.passes, undefined, join(REVIEWS, `${journey.id}.partial.json`));
95
+ mkdirSync(REVIEWS, { recursive: true });
96
+ writeFileSync(join(REVIEWS, `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
97
+ rmSync(join(REVIEWS, `${journey.id}.partial.json`), { force: true });
98
+ const verdicts = {};
99
+ for (const r of reviews)
100
+ verdicts[r.verdict] = (verdicts[r.verdict] ?? 0) + r.tests.length;
101
+ out.push({ journey: journey.id, failures: saved.failures.length, verdicts });
102
+ }
103
+ return out;
104
+ }
@@ -18,27 +18,49 @@
18
18
  */
19
19
  import { readFileSync, writeFileSync, mkdirSync, existsSync, } from 'node:fs';
20
20
  import { join, dirname, basename, extname, relative, resolve, sep } from 'node:path';
21
- import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored } from '../paths.js';
21
+ import { CACHE_PATH, RESULTS_DIR, ensureStateDirIgnored, journeyDir } from '../paths.js';
22
22
  import { lintGenerated } from '../gates/lint.js';
23
23
  import { parseJourney, notAJourneyDoc } from '../docs/parser.js';
24
24
  import { writeCoverageManifest } from '../gates/coverage.js';
25
- import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
25
+ import { cleanStaleFiles, collisionWarning, hasDeferredStubs, isDeferredStub, isStubDoc, loadOrPlan, readPlan, readPlanSkips, writePlan, readPlanSlugs, GEN_MARKER, matchesJourney, } from './plan-cache.js';
26
26
  import { aiFailureNote, aiUnavailableReason, confirmAiAlive, lastAiFailureWasProcess, noteAiFailure, noteAiRejected, noteAiSuccess, resetAiSession, waitForAi, } from './ai-session.js';
27
27
  import { generateTestCases } from './generator.js';
28
28
  import { writeDeferredStub } from './playwright-writer.js';
29
- import { generateWithAI, generateWithAIAsync, repairWithAIAsync, } from './ai-generator.js';
30
- import { browserPreflight, missingPackages } from './verify.js';
31
- import { verifyJourney } from './verify-journey.js';
29
+ import { buildPrompt, generateWithAI, generateWithAIAsync, writeFilesWithAI, } from './ai-generator.js';
32
30
  import { normalisePending } from './pending.js';
33
31
  import { packageRoot } from '../run/run-tests.js';
34
32
  import { groundJourney } from '../docs/grounding.js';
35
33
  import { generateSeed } from './seed-generator.js';
36
- import { isSyntaxValid, checkTestGates } from '../gates/validate.js';
34
+ import { dataAccess, writeRequiredData } from './required-data.js';
35
+ import { aiUsage, usageSince, describeUsage } from './ai-usage.js';
36
+ import { testedTags } from './suite-planner.js';
37
+ import { factsSlice, helperApi, journeySlice, readRules, saveMemory, testedElsewhere } from './journey-memory.js';
38
+ import { loadSourceChoice } from '../docs/source-of-truth.js';
39
+ import { alignRules, describeDiff, loadMemory } from './doc-diff.js';
40
+ import { updateJourney } from './update-journey.js';
41
+ import { sourceText } from '../docs/excerpts.js';
42
+ import { isSyntaxValid, checkTestGates, duplicatedHelpers, unbalancedAt } from '../gates/validate.js';
37
43
  import { installedModules, transportHint, projectSelectors } from '../project/detect.js';
38
44
  import { harnessInfo, generateHarnessAsync, harnessIsCurrent, stampHarness, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, } from './harness.js';
39
45
  import { hashContent, loadCache, saveCache } from '../docs/cache.js';
40
46
  import { findMarkdownFiles } from '../find-files.js';
41
47
  import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
48
+ /**
49
+ * kit.find is a shared helper the testkit re-exports beside settledClick. A
50
+ * testkit written before it existed gets the one line here, with no AI and no
51
+ * rebuild.
52
+ */
53
+ export function exportFind(harnessPath) {
54
+ const src = readFileSync(harnessPath, 'utf-8');
55
+ if (/^\s*find\s*[:,]/m.test(src) || !/settledClick:\s*(\w+)\.settledClick,?/.test(src))
56
+ return;
57
+ writeFileSync(harnessPath, src.replace(/^(\s*)settledClick:\s*(\w+)\.settledClick,?/m, (m, pad, mod) => `${m.endsWith(',') ? m : `${m},`}\n${pad}find: ${mod}.find,`), 'utf-8');
58
+ }
59
+ /** Why a file is not valid code, as precisely as the check can say. */
60
+ function syntaxReason(content, ext) {
61
+ const where = /\.(c|m)?js$/i.test(ext) ? null : unbalancedAt(content);
62
+ return `not valid ${ext}${where ? `: ${where}` : ' (syntax)'} — the answer must be the whole file, code only`;
63
+ }
42
64
  /** How many times to re-ask the AI when output won't parse. */
43
65
  const MAX_GEN_ATTEMPTS = 2;
44
66
  /** How many focused files to generate concurrently within a journey. */
@@ -64,13 +86,17 @@ export async function pool(items, limit, fn) {
64
86
  return results;
65
87
  }
66
88
  /** Async version of generateValidated (for concurrent per-file gen). */
67
- async function generateValidatedAsync(generate, ext, gate) {
89
+ async function generateValidatedAsync(generate, ext, gate,
90
+ /** Why an earlier draft of the same file (from a batch) was refused. */
91
+ firstRefusal) {
68
92
  let content = '';
93
+ let refusal = firstRefusal;
69
94
  for (let attempt = 0; attempt < MAX_GEN_ATTEMPTS; attempt++) {
70
95
  if (aiUnavailableReason())
71
96
  return null; // retrying a dead CLI only burns time
72
97
  try {
73
- content = await generate();
98
+ // A second try is told why the first was refused; asked blind, it made the same file again.
99
+ content = await generate(refusal);
74
100
  }
75
101
  catch (e) {
76
102
  noteAiFailure(e);
@@ -80,7 +106,8 @@ async function generateValidatedAsync(generate, ext, gate) {
80
106
  const gateMsg = gate ? gate(content) : null;
81
107
  if (isSyntaxValid(content, ext) && !gateMsg)
82
108
  return content;
83
- noteAiRejected(gateMsg || `not valid ${ext} (syntax)`);
109
+ refusal = gateMsg || syntaxReason(content, ext);
110
+ noteAiRejected(refusal);
84
111
  }
85
112
  return null;
86
113
  }
@@ -163,6 +190,7 @@ export async function runHarness(only) {
163
190
  }
164
191
  mkdirSync(dirname(info.path), { recursive: true });
165
192
  writeFileSync(info.path, stampHarness(kit), 'utf-8');
193
+ exportFind(info.path);
166
194
  writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
167
195
  writeHarnessConfigs(framework, areaDir, info.dir, area);
168
196
  written.push(info.path);
@@ -177,27 +205,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
177
205
  return [];
178
206
  }
179
207
  const config = loadConfig();
180
- // Verify is on unless the project or the caller turns it off. verifyFiles
181
- // itself reports (and skips) the cases where the app cannot be reached, so
182
- // "on" never means "fail the run because nothing is running".
183
- const verifyOn = opts.verify ?? config.verify ?? true;
184
- const verifyAttempts = Math.max(0, config.verifyAttempts ?? 2);
185
208
  const ext = config.testExtension ?? '.test.ts';
186
209
  const seedDir = config.seedOutput ?? 'generated-tests/seed';
187
210
  const seedExt = ext.endsWith('.js') ? '.seed.js' : '.seed.ts';
188
- // A browser that cannot start fails every verify, and each failure would be
189
- // sent for repair. Find out once, before any AI call is spent.
190
- if (useAI && verifyOn) {
191
- for (const area of Object.keys(config.testOutput ?? {})) {
192
- if (resolveFramework(config, area, area) !== 'playwright')
193
- continue;
194
- const problem = browserPreflight(packageRoot(testDirForType(config, area)));
195
- if (problem) {
196
- throw new Error(`Cannot verify "${area}" tests: ${problem}.\n` +
197
- 'Fix that and re-run, or pass --no-verify to write tests without running them.');
198
- }
199
- }
200
- }
201
211
  ensureStateDirIgnored();
202
212
  const cache = loadCache(CACHE_PATH);
203
213
  const results = [];
@@ -245,9 +255,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
245
255
  // makes one-by-one (`generate --journey X`) cheap: the first journey
246
256
  // in an area builds the testkit, every later journey reuses it — no
247
257
  // extra AI call (and no risk of that call eating the token limit).
248
- // Pass --force to rebuild a stale testkit (e.g. after upgrading).
249
- if (!force && existsSync(info.path)) {
258
+ // --force rewrites a journey's tests, not the testkit every journey in the
259
+ // area shares: a current testkit is kept even then. A stale one is rebuilt.
260
+ if (existsSync(info.path)) {
250
261
  if (harnessIsCurrent(readFileSync(info.path, 'utf-8'))) {
262
+ exportFind(info.path);
251
263
  harnessByArea.set(area, info.path);
252
264
  return info.path;
253
265
  }
@@ -263,6 +275,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
263
275
  }
264
276
  mkdirSync(dirname(info.path), { recursive: true });
265
277
  writeFileSync(info.path, stampHarness(kit), 'utf-8');
278
+ exportFind(info.path);
266
279
  writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
267
280
  // Config + globalSetup (playwright) / config + setup (jest) so the specs run
268
281
  // AND get their tokens minted once, plus the runner config scaffolding.
@@ -408,13 +421,21 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
408
421
  });
409
422
  // A freshly written file also may not hold placeholders; a repair is held to
410
423
  // the repair guard instead (it may not add a skip), so old files stay repairable.
424
+ // What the app can show: its source and the doc. A fresh test's names and
425
+ // labels are checked against it before the test is run.
426
+ let knownTexts;
411
427
  const genGate = (content, filePath) => testGate(content) ??
412
- checkTestGates(content, ext, { framework, forbidPlaceholders: true, filePath });
428
+ checkTestGates(content, ext, {
429
+ framework, forbidPlaceholders: true, filePath,
430
+ maxTestTimeoutMs: config.maxTestTimeoutMs ?? 90_000, maxTestsPerRule: config.maxTestsPerRule ?? 2,
431
+ knownTexts: framework === 'playwright'
432
+ ? (knownTexts ??= `${sourceText([join(packageRoot(areaDir), 'src'), ...(config.relatedDirs ?? [])])} ${markdown.toLowerCase().replace(/\s+/g, ' ')}`)
433
+ : undefined,
434
+ });
413
435
  // Skip if unchanged — but NOT if the journey still has deferred stubs
414
436
  // to retry (a resume pass fills those in without touching real files).
415
437
  if (!force &&
416
438
  cache[file] === hash &&
417
- (!useAI || !verifyOn || cache[`verified:${file}`] === hash) &&
418
439
  !hasDeferredStubs(testDir, ext)) {
419
440
  results.push({
420
441
  journeyName: journey.name,
@@ -427,6 +448,12 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
427
448
  continue;
428
449
  }
429
450
  const warnings = [];
451
+ const usageStart = aiUsage();
452
+ // A feature written up before it is built: its tests come from the doc
453
+ // alone. There is no code to read for facts and no app to run them on.
454
+ const notBuilt = /^status:\s*not built/im.test(markdown);
455
+ if (notBuilt)
456
+ warnings.push('written before the feature exists (Status: not built yet) — run these tests once it ships');
430
457
  const collision = collisionWarning(areaDir, journey.id, ext);
431
458
  if (collision)
432
459
  warnings.push(collision);
@@ -459,9 +486,70 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
459
486
  // Capture the PREVIOUS plan's slugs first so we can clean up any
460
487
  // files whose slug no longer exists (F6 replace-don't-duplicate).
461
488
  const prevSlugs = useAI ? readPlanSlugs(journey) : [];
489
+ // The doc read ONCE: rules, controls and setup, plus the testkit's exports.
490
+ // From here on no AI call gets the whole doc; each gets its own slice.
491
+ // A doc without tagged rules keeps the old whole-doc flow.
492
+ const parsed = useAI ? readRules(markdown) : undefined;
493
+ const tested = testedTags();
494
+ let memory = parsed && parsed.rules.some((r) => r.tag && tested.includes(r.tag)) ? parsed : undefined;
495
+ const helpers = memory && harnessPath && existsSync(harnessPath) ? helperApi(readFileSync(harnessPath, 'utf-8')) : '';
496
+ // A doc that changed since its suite was written: only what changed is
497
+ // touched. Renames and removals are edits without AI; reworded and added
498
+ // rules get one small call each file; new sections become new files below.
499
+ const prevRecord = memory ? loadMemory(journey) : undefined;
500
+ const prevMemory = !force && !opts.planOnly ? prevRecord : undefined;
501
+ const prevPlan = prevMemory && prevMemory.hash !== hash ? readPlan(journey) : undefined;
502
+ let changes;
503
+ if (memory && prevMemory && prevPlan && prevPlan.length && prevPlan.every((f) => Array.isArray(f.rules))) {
504
+ const { memory: aligned, diff } = alignRules(prevMemory.rules, memory);
505
+ memory = aligned;
506
+ changes = { renamed: diff.renamed.map((r) => r.id), changed: diff.changed, removed: diff.removed };
507
+ console.log(` ${journey.id}: doc changed — ${describeDiff(diff)}`);
508
+ const facts = notBuilt ? undefined : (await groundJourney(journey, hash, 'src', markdown, true)) ?? undefined;
509
+ const up = await updateJourney({
510
+ journey, memory: aligned, diff, plan: prevPlan, tested, helpers, facts, harnessPath, transport,
511
+ pathOf: (f) => join(testDir, `${flat ? `${journey.id}-${f.slug}` : f.slug}${ext}`),
512
+ check: (c, p) => (isSyntaxValid(c, ext) ? genGate(c, p) : syntaxReason(c, ext)),
513
+ });
514
+ for (const n of up.notes)
515
+ console.log(` ${n}`);
516
+ warnings.push(...up.notes);
517
+ writePlan(journey, hash, up.plan);
518
+ }
462
519
  const planned = useAI
463
- ? loadOrPlan(journey, hash)
520
+ ? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
464
521
  : [{ slug: journey.id, title: journey.name, covers: '' }];
522
+ if (memory) {
523
+ const choice = loadSourceChoice();
524
+ saveMemory(journey, hash, memory, helpers, {
525
+ doc: file, reference: choice?.journeys?.[file] ?? choice?.reference,
526
+ plan: planned, skipped: readPlanSkips(journey), prev: prevRecord, changes,
527
+ });
528
+ }
529
+ if (opts.planOnly) {
530
+ results.push({
531
+ journeyName: journey.name,
532
+ testDir,
533
+ testFiles: [],
534
+ seedPath,
535
+ status: 'skipped',
536
+ seedStatus: 'none',
537
+ warnings: [`plan only: ${planned.length} file(s) planned, nothing written — see ${join(journeyDir(journey), 'plan-check.md')}`],
538
+ });
539
+ continue;
540
+ }
541
+ // What one file is given: its rules with their context, the facts for
542
+ // its controls, the testkit list. A plan made before rules existed
543
+ // gives every file all the rules that get tests.
544
+ const allTested = memory ? memory.rules.filter((r) => r.tag && tested.includes(r.tag)).map((r) => r.id) : [];
545
+ const fileMemory = async (pf) => {
546
+ const facts = await grounding();
547
+ if (!memory)
548
+ return { facts };
549
+ const ids = pf.rules?.length ? pf.rules : allTested;
550
+ const slice = journeySlice(memory, ids);
551
+ return { mem: { journey: slice.text, helpers, rules: ids }, facts: factsSlice(facts, slice.controls) };
552
+ };
465
553
  const plannedSlugs = planned.map((p) => p.slug);
466
554
  // Read the journey's real code ONCE, before any file is written, and hand
467
555
  // the same facts to every file. Without this each file re-opened the same
@@ -472,7 +560,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
472
560
  // Only a file that has to be written or repaired needs it, so it is read
473
561
  // on first use: re-checking an existing suite costs no grounding at all.
474
562
  let groundingP;
475
- const grounding = () => (groundingP ??= useAI
563
+ const grounding = () => (groundingP ??= useAI && !notBuilt
476
564
  ? groundJourney(journey, hash, 'src', markdown).then((g) => g ?? undefined)
477
565
  : Promise.resolve(undefined));
478
566
  const staleRemoved = cleanStaleFiles(testDir, journey.id, ext, flat, plannedSlugs, prevSlugs);
@@ -485,6 +573,50 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
485
573
  // parallel agentic AI runs starve each other and time out, so a flaky
486
574
  // environment can lower `aiConcurrency` (e.g. 2) for reliability.
487
575
  const concurrency = Math.max(1, config.aiConcurrency ?? GEN_CONCURRENCY);
576
+ // Files to write, a few to one AI call: each call carries the shared rules
577
+ // once. A drafted file that fails its checks is written again on its own,
578
+ // told why; a file the batch left out is written on its own.
579
+ const pathOf = (pf) => join(testDir, `${flat ? `${journey.id}-${pf.slug}` : pf.slug}${ext}`);
580
+ const keeps = (p) => {
581
+ if ((force && !redone.has(i)) || !existsSync(p))
582
+ return false;
583
+ try {
584
+ const c = readFileSync(p, 'utf-8');
585
+ return isSyntaxValid(c, ext) && !isDeferredStub(c);
586
+ }
587
+ catch {
588
+ return false;
589
+ }
590
+ };
591
+ const drafts = new Map();
592
+ const perCall = Math.max(1, config.filesPerCall ?? 3);
593
+ if (useAI && memory && perCall > 1) {
594
+ const toWrite = planned.filter((pf) => !keeps(pathOf(pf)));
595
+ const batches = [];
596
+ for (let k = 0; k < toWrite.length; k += perCall)
597
+ batches.push(toWrite.slice(k, k + perCall));
598
+ const writeBatch = async (batch) => {
599
+ const files = await Promise.all(batch.map(async (pf) => {
600
+ const { mem, facts } = await fileMemory(pf);
601
+ return { path: pathOf(pf), prompt: buildPrompt(journey, '', pathOf(pf), { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem) };
602
+ }));
603
+ try {
604
+ const got = await writeFilesWithAI(files);
605
+ noteAiSuccess();
606
+ for (const [p, c] of got)
607
+ drafts.set(p, c);
608
+ }
609
+ catch (e) {
610
+ noteAiFailure(e);
611
+ }
612
+ };
613
+ // The first call alone writes the shared prompt to the cache; the rest
614
+ // then read it instead of each writing its own copy at the same moment.
615
+ const multi = batches.filter((b) => b.length > 1);
616
+ if (multi.length)
617
+ await writeBatch(multi[0]);
618
+ await pool(multi.slice(1), concurrency, writeBatch);
619
+ }
488
620
  const generated = await pool(planned, concurrency, async (pf) => {
489
621
  // Flat (jest): prefix the file with the journey so many journeys can
490
622
  // share one dir without clashing. Folder (playwright): the journey
@@ -495,7 +627,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
495
627
  // — don't spend tokens regenerating it. So a journey interrupted
496
628
  // midway resumes from exactly the files it hadn't written yet.
497
629
  // (--force ignores this and regenerates.)
498
- if (!force && existsSync(filePath)) {
630
+ // A redo after stubs fills in only the stubs: --force already rewrote the rest once this run.
631
+ if ((!force || redone.has(i)) && existsSync(filePath)) {
499
632
  try {
500
633
  const existing = readFileSync(filePath, 'utf-8');
501
634
  // Keep a real, parsing file — but a DEFERRED stub is not real
@@ -516,8 +649,10 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
516
649
  let content;
517
650
  let status;
518
651
  if (useAI) {
519
- const facts = await grounding();
520
- const ai = await generateValidatedAsync(() => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts), ext, (c) => genGate(c, filePath));
652
+ const draft = drafts.get(filePath);
653
+ const draftRefusal = draft === undefined ? undefined : isSyntaxValid(draft, ext) ? genGate(draft, filePath) : syntaxReason(draft, ext);
654
+ const { mem, facts } = await fileMemory(pf);
655
+ const ai = draft !== undefined && !draftRefusal ? draft : await generateValidatedAsync((refusal) => generateWithAIAsync(journey, filePath, { title: pf.title, covers: pf.covers }, harnessPath, transport, facts, mem, refusal), ext, (c) => genGate(c, filePath), draftRefusal ?? undefined);
521
656
  if (ai !== null) {
522
657
  content = framework === 'playwright' ? normalisePending(ai) : ai;
523
658
  status = 'ai';
@@ -568,12 +703,22 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
568
703
  .join(', ')}${aiFailureNote()}`);
569
704
  }
570
705
  // 2. Seed file (only when using AI — basic mode has no seed concept).
706
+ // A repository without its own data store gets no seed code it could
707
+ // never run: its journey's "Test setup" goes into REQUIRED-DATA.md.
571
708
  let seedStatus = 'none';
572
- if (useAI && !force && existsSync(seedPath)) {
709
+ // Per area: in a monorepo the api package owns the store, the web package does not.
710
+ if (useAI && dataAccess(config, packageRoot(areaDir)) === 'api') {
711
+ writeRequiredData(seedDir, journey.id, journey.name, markdown);
712
+ }
713
+ else if (useAI && !force && existsSync(seedPath)) {
573
714
  seedStatus = 'ok'; // kept, like an existing test file
574
715
  }
575
716
  else if (useAI) {
576
- const seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
717
+ let seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
718
+ // A usage limit is not a failed seed: wait for the AI like every other step, then try again.
719
+ while (seed === null && aiUnavailableReason() && waitForLimit && (await waitForAi(config))) {
720
+ seed = generateValidated(() => generateSeed(markdown, seedPath), seedExt);
721
+ }
577
722
  if (seed !== null) {
578
723
  mkdirSync(dirname(seedPath), { recursive: true });
579
724
  writeFileSync(seedPath, seed, 'utf-8');
@@ -605,6 +750,11 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
605
750
  // Only files testforge generated: a project's own config is never edited.
606
751
  ...[harnessPath, join(dirname(testDir), 'playwright.config.js')].filter((f) => !!f && existsSync(f) && readFileSync(f, 'utf-8').includes(GEN_MARKER)),
607
752
  ];
753
+ // One helper copied into several files diverges and fails them all at once: name each copy.
754
+ const dupes = duplicatedHelpers(testFiles.map((t) => (typeof t === 'string' ? t : t.path)).filter((p) => existsSync(p)).map((p) => ({ path: relative(process.cwd(), p), content: readFileSync(p, 'utf-8') })));
755
+ for (const d of dupes) {
756
+ warnings.push(`helper "${d.name}" is copied into ${d.files.length} files (${d.files.join(', ')}) — it belongs in the testkit`);
757
+ }
608
758
  const lint = lintGenerated(lintTargets);
609
759
  if (lint.fixed > 0) {
610
760
  console.log(` tidied ${lint.fixed} lint problem(s) in ${lint.checked} generated file(s)`);
@@ -613,54 +763,15 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
613
763
  warnings.push(`lint: ${lint.remaining.length} problem(s) the linter could not fix — ` +
614
764
  lint.remaining.slice(0, 5).join('; '));
615
765
  }
616
- // Run what we just wrote against the real application before calling it
617
- // done — file by file, then the whole journey together, then an independent
618
- // review of every claimed defect (see verify-journey.ts). Anything that
619
- // could not run, or still fails unexplained, leaves the journey UNVERIFIED
620
- // and uncached, so the next run checks it again.
621
- let verified;
622
- let reviewed = [];
623
- const targets = testFiles.filter((t) => t.status === 'ai').map((t) => t.path);
624
- const missing = harnessPath ? missingPackages(harnessPath, packageRoot(testDir)) : [];
625
- if (useAI && verifyOn && targets.length && missing.length) {
626
- // The suite cannot even start; running it would only feed repair a
627
- // failure no test edit can fix.
628
- verified = false;
629
- warnings.push(`verify: not run — install the packages the harness needs: npm i -D ${missing.join(' ')}`);
630
- }
631
- else if (useAI && verifyOn && targets.length) {
632
- const result = await verifyJourney({
633
- config,
634
- journey,
635
- targets,
636
- attempts: verifyAttempts,
637
- repair: async (target, current, failure) => {
638
- const facts = await grounding();
639
- const out = await generateValidatedAsync(() => repairWithAIAsync(journey, target, current, failure, harnessPath, transport, facts), ext, testGate);
640
- if (out === null)
641
- return null;
642
- const stamped = out.includes(GEN_MARKER) ? out : `// ${GEN_MARKER}\n${out}`;
643
- return framework === 'playwright' ? normalisePending(stamped) : stamped;
644
- },
645
- });
646
- verified = result.verified;
647
- reviewed = result.reviewed;
648
- warnings.push(...result.warnings);
649
- }
650
- const degraded = useAI && (status === 'basic' || seedStatus === 'failed' || verified === false);
766
+ const degraded = useAI && (status === 'basic' || seedStatus === 'failed');
651
767
  if (!degraded) {
652
768
  cache[file] = hash;
653
- // Remembered apart, so a journey cached before it was ever run is
654
- // re-checked (not regenerated) once verification is on.
655
- if (verified)
656
- cache[`verified:${file}`] = hash;
657
769
  }
658
770
  else {
659
771
  // Actively UN-mark a degraded journey (even under --force, which may
660
772
  // be re-processing a stale "done" from an earlier buggy run), so the
661
773
  // next `generate` re-does it instead of skipping a thin fallback.
662
774
  delete cache[file];
663
- delete cache[`verified:${file}`];
664
775
  }
665
776
  saveCache(CACHE_PATH, cache);
666
777
  resultRow.set(file, results.length);
@@ -668,6 +779,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
668
779
  stubbed.add(file);
669
780
  else
670
781
  stubbed.delete(file);
782
+ const ai = usageSince(usageStart);
783
+ if (ai.calls)
784
+ console.log(` ${journey.id}: ${describeUsage(ai)}`);
671
785
  results.push({
672
786
  journeyName: journey.name,
673
787
  testDir,
@@ -675,9 +789,8 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
675
789
  seedPath,
676
790
  status,
677
791
  seedStatus,
678
- verified,
679
792
  warnings: warnings.length ? warnings : undefined,
680
- reviewed: reviewed.length ? reviewed : undefined,
793
+ ai,
681
794
  });
682
795
  }
683
796
  for (const [area, b] of blockedAreas) {
@@ -14,10 +14,10 @@
14
14
  * Secrets are never embedded: the connection comes from the env var
15
15
  * named in the map (e.g. process.env.DB_URL), not a literal string.
16
16
  */
17
- import { execFileSync } from 'node:child_process';
18
17
  import { readFileSync, existsSync } from 'node:fs';
19
18
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
20
19
  import { cleanOutput } from './ai-generator.js';
20
+ import { runAiSync } from './ai-usage.js';
21
21
  const MAP_PATH = 'testforge.md';
22
22
  /** Build the seed-generation prompt for one journey. */
23
23
  export function buildSeedPrompt(journeyDoc, seedPath) {
@@ -96,7 +96,7 @@ export function generateSeed(journeyDoc, seedPath) {
96
96
  // Allow read-only exploration so the seed matches the REAL schema
97
97
  // (models/collections/enums) read from the code; Write stays denied.
98
98
  const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
99
- const output = execFileSync(program, args, {
99
+ const output = runAiSync(program, args, {
100
100
  input: prompt,
101
101
  encoding: 'utf-8',
102
102
  maxBuffer: 10 * 1024 * 1024,