@activate-core/test-forge 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +98 -34
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +51 -4
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +51 -24
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +68 -5
  10. package/dist/core/docs/review.js +81 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +56 -0
  13. package/dist/core/gates/coverage.js +12 -19
  14. package/dist/core/gates/validate.js +277 -4
  15. package/dist/core/generate/ai-generator.js +243 -96
  16. package/dist/core/generate/ai-session.js +8 -0
  17. package/dist/core/generate/ai-usage.js +150 -0
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/defect-review.js +213 -20
  20. package/dist/core/generate/doc-diff.js +139 -0
  21. package/dist/core/generate/harness.js +52 -32
  22. package/dist/core/generate/journey-memory.js +325 -0
  23. package/dist/core/generate/plan-cache.js +114 -7
  24. package/dist/core/generate/required-data.js +75 -0
  25. package/dist/core/generate/review-journey.js +104 -0
  26. package/dist/core/generate/run-generate.js +193 -80
  27. package/dist/core/generate/seed-generator.js +2 -2
  28. package/dist/core/generate/suite-planner.js +138 -25
  29. package/dist/core/generate/test-blocks.js +84 -0
  30. package/dist/core/generate/update-journey.js +115 -0
  31. package/dist/core/generate/verify.js +9 -4
  32. package/dist/core/paths.js +90 -13
  33. package/dist/core/project/config.js +25 -2
  34. package/dist/core/project/env-diff.js +122 -0
  35. package/dist/core/project/env-template.js +3 -2
  36. package/dist/core/project/init.js +2 -2
  37. package/dist/core/project/map.js +2 -2
  38. package/dist/core/run/ci.js +94 -0
  39. package/dist/core/run/clean.js +111 -0
  40. package/dist/core/run/readiness.js +119 -0
  41. package/dist/core/run/report.js +2 -0
  42. package/dist/core/run/run-tests.js +23 -1
  43. package/package.json +1 -1
  44. package/reporters/_shared.cjs +24 -3
  45. package/reporters/jest-reporter.cjs +3 -1
  46. package/reporters/playwright-reporter.cjs +3 -1
  47. package/reporters/run-data.cjs +66 -0
  48. package/reporters/ui-helpers.cjs +133 -0
  49. package/dist/core/generate/verify-journey.js +0 -174
@@ -11,11 +11,11 @@
11
11
  * auth (read-only tools), so it connects and seeds against the actual
12
12
  * schema — not a guess.
13
13
  */
14
- import { execFileSync, spawn } from 'node:child_process';
15
14
  import { existsSync, readFileSync } from 'node:fs';
16
15
  import { createHash } from 'node:crypto';
17
16
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
18
- import { cleanOutput } from './ai-generator.js';
17
+ import { cleanOutput, runWithStdin } from './ai-generator.js';
18
+ import { runAiSync } from './ai-usage.js';
19
19
  const MAP_PATH = 'testforge.md';
20
20
  /**
21
21
  * The fixed contract every testkit exposes and every generated test
@@ -39,7 +39,16 @@ export const HARNESS_CONTRACT = [
39
39
  ` - REQUIRED_PACKAGES: string[] packages the testkit needs (see H6).`,
40
40
  ` - resetRateLimits(), bypassVerification(), suppressPopups(), and one`,
41
41
  ` resolver per seeded entity — see H7 and H3.`,
42
+ ` - unique(label?): string a value no other run produces (H8).`,
43
+ ` - created(kind, id, cleanup?, role?): void record what the run made (H8).`,
44
+ ` - the "run" role: token('run'), ensureToken('run'), login(page, 'run'),`,
45
+ ` apiContext(request, 'run') — the account tests that CREATE data use (H8).`,
42
46
  `Plus, for Playwright suites:`,
47
+ ` - openStep(page, step, input?): Promise<true | string> open a later step`,
48
+ ` of a multi-step flow with what the earlier steps would have left (H9).`,
49
+ ` - settledClick, fillFrameInputs, watchDialogs: re-exported unchanged from`,
50
+ ` require('@activate-core/test-forge/reporters/ui-helpers.cjs') — never`,
51
+ ` rewritten here.`,
43
52
  ` - apiContext(request): an authenticated API request context, created from`,
44
53
  ` the imported request MODULE (\`const { request: apiRequest } =`,
45
54
  ` require('@playwright/test'); apiRequest.newContext({ baseURL,`,
@@ -144,6 +153,40 @@ export const HARNESS_CONTRACT = [
144
153
  ` real login. When the environment cannot SEND the code, \`sendUrl\` is`,
145
154
  ` answered with the real success shape too. Scope each to exactly that`,
146
155
  ` request; everything after it runs against the real backend.`,
156
+ `H8. WHAT A RUN CREATES. Runs share accounts and environments, so data one`,
157
+ ` run leaves behind must never change another run's result.`,
158
+ ` - Load the run-data runtime once:`,
159
+ ` const runData = require('@activate-core/test-forge/reporters/run-data.cjs').`,
160
+ ` unique(label) returns runData.unique(label). created(kind, id, cleanup,`,
161
+ ` role) calls runData.record({ kind, id, role, cleanup }), where cleanup is`,
162
+ ` { method, url } of the product's own request that removes or cancels that`,
163
+ ` record (absolute url; read the routes), or undefined when there is none.`,
164
+ ` - Every testkit helper that creates a record through the API calls`,
165
+ ` created() with it.`,
166
+ ` - The "run" role. TEST_RUN_ACCOUNT_MODE picks the account:`,
167
+ ` fixture (default) — the "run" role IS the customer role;`,
168
+ ` pool — TEST_RUN_ACCOUNTS is a comma list of identifiers sharing`,
169
+ ` TEST_RUN_ACCOUNTS_PASSWORD; pick one by a stable hash of runData.runId();`,
170
+ ` signup — create a fresh account through the product's own sign-up, only`,
171
+ ` when this environment can complete it (no code it cannot read and`,
172
+ ` cannot bypass under H7); otherwise fall back to fixture and say so once.`,
173
+ ` Mint and cache its token like any role.`,
174
+ `H9. STEPS OF A FLOW. A later step of a multi-step flow (onboarding, a`,
175
+ ` booking or checkout wizard, a sign-up in stages) depends on what the`,
176
+ ` earlier steps stored — a choice, an address, a cart, a plan — in URL`,
177
+ ` parameters, browser storage or the server. Opened by its URL alone it`,
178
+ ` often sends the visitor back to an earlier step without a word, and a`,
179
+ ` test then "fails" on the product's correct guard.`,
180
+ ` - Read the flow's step components, its router guard and its store, and`,
181
+ ` give openStep(page, step, input?) the way the product reaches that step:`,
182
+ ` the parameters a real entry link carries, or the stored state the`,
183
+ ` earlier steps write (pre-set with an init script before the app runs).`,
184
+ ` \`input\` overrides the defaults (which choice, which address).`,
185
+ ` - It waits until the step is really shown and resolves true; when the`,
186
+ ` product still sends the visitor elsewhere it resolves a string naming`,
187
+ ` the step it landed on, so the test can skip with that reason.`,
188
+ ` - Offer only the steps the flow has; when the project has no such flow,`,
189
+ ` export openStep as a function resolving 'this product has no multi-step flow'.`,
147
190
  ].join('\n');
148
191
  /**
149
192
  * Stamp written into every generated testkit: the contract it was built
@@ -239,7 +282,7 @@ function buildHarnessPrompt(harnessPath, framework) {
239
282
  export function generateHarness(harnessPath, framework) {
240
283
  const prompt = buildHarnessPrompt(harnessPath, framework);
241
284
  const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
242
- const output = execFileSync(program, args, {
285
+ const output = runAiSync(program, args, {
243
286
  input: prompt,
244
287
  encoding: 'utf-8',
245
288
  maxBuffer: 10 * 1024 * 1024,
@@ -251,35 +294,8 @@ export function generateHarnessAsync(harnessPath, framework) {
251
294
  const config = loadConfig();
252
295
  const prompt = buildHarnessPrompt(harnessPath, framework);
253
296
  const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
254
- const timeoutMs = config.aiTimeoutMs ?? 300_000;
255
- return new Promise((resolve, reject) => {
256
- const child = spawn(program, args, { stdio: ['pipe', 'pipe', 'pipe'] });
257
- let out = '';
258
- let err = '';
259
- let done = false;
260
- const finish = (fn) => {
261
- if (done)
262
- return;
263
- done = true;
264
- clearTimeout(timer);
265
- fn();
266
- };
267
- // Without this, a harness call that hangs (flaky network, a stalled AI
268
- // CLI) never closes and the whole run wedges on step 1 forever — kill it
269
- // and surface a resumable error so the caller can retry.
270
- const timer = setTimeout(() => {
271
- child.kill('SIGKILL');
272
- finish(() => reject(new Error(`harness gen timed out after ${timeoutMs}ms — the AI CLI did not respond`)));
273
- }, timeoutMs);
274
- child.stdout.setEncoding('utf-8');
275
- child.stdout.on('data', (d) => (out += d));
276
- child.stderr.on('data', (d) => (err += d));
277
- child.on('error', (e) => finish(() => reject(e)));
278
- child.on('close', (code) => finish(() => code === 0
279
- ? resolve(cleanOutput(out))
280
- : reject(new Error(`harness gen exited ${code}: ${err.slice(0, 300)}`))));
281
- child.stdin.end(prompt);
282
- });
297
+ // Same runner as every other async AI call: one timeout guard, one usage count, one trace.
298
+ return runWithStdin(program, args, prompt, config.aiTimeoutMs ?? 300_000).then(cleanOutput);
283
299
  }
284
300
  /**
285
301
  * A minimal Playwright config so a webapp/ops area that currently uses
@@ -295,6 +311,8 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, c
295
311
  `const { defineConfig } = require('@playwright/test');`,
296
312
  `const fs = require('fs');`,
297
313
  `const path = require('path');`,
314
+ `// One run id for this run, set here in the main process so every worker inherits it.`,
315
+ `try { require('@activate-core/test-forge/reporters/run-data.cjs').runId(); } catch (_) {}`,
298
316
  ``,
299
317
  `// Load the env file named in testforge.config.json, so a plain`,
300
318
  `// \`npx playwright test\` (or an editor's test runner) gets the same env as`,
@@ -352,6 +370,8 @@ export function jestConfig(testDirRelToPackage, setupRelToPackage, category = 'a
352
370
  return [
353
371
  `// @generated by testforge`,
354
372
  `// Install once: npm i -D jest`,
373
+ `// One run id for this run, set before the workers start so each inherits it.`,
374
+ `try { require('@activate-core/test-forge/reporters/run-data.cjs').runId(); } catch (_) {}`,
355
375
  `module.exports = {`,
356
376
  ` testEnvironment: 'node',`,
357
377
  ` setupFiles: ['<rootDir>/${setupRelToPackage}'],`,
@@ -0,0 +1,325 @@
1
+ /**
2
+ * testforge — a journey read once, handed out in pieces.
3
+ *
4
+ * The journey doc is parsed into its rules, controls and test setup, the
5
+ * testkit into the list of helpers it exports, and both are saved next to the
6
+ * plan. Every later AI call receives only the rules, controls and facts its
7
+ * own file needs, and never the whole doc again. Measured on one small
8
+ * journey, re-sending the doc, the map and the facts to every file and every
9
+ * repair was most of 9.6M tokens; the code below needs no AI at all.
10
+ */
11
+ import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
12
+ import { dirname, join } from 'node:path';
13
+ import { journeyDir } from '../paths.js';
14
+ /** Sections that are indexes or open questions, never something to test. */
15
+ const NOT_RULES = /^(controls|coverage|test setup|questions for product|not requirements|pending decisions|suggested)\b/i;
16
+ const TAG = /\[(spec|baseline|observed)\]/;
17
+ const CONTROL_REF = /\bC\d+[a-z]?\b/g;
18
+ /** Split a journey doc into header, controls, rules and setup. Pure: no AI, no I/O. */
19
+ export function readRules(markdown) {
20
+ const lines = markdown.split('\n');
21
+ const header = [];
22
+ const setup = [];
23
+ const controls = {};
24
+ const rules = [];
25
+ let section = '';
26
+ let group = '';
27
+ let mode = 'header';
28
+ let current = null;
29
+ const close = () => {
30
+ if (!current)
31
+ return;
32
+ const r = current.rule;
33
+ r.text = r.text.trimEnd();
34
+ r.controls = [...new Set(r.text.match(CONTROL_REF) ?? [])];
35
+ const marked = /\]\s*\((must|should|could)\b/i.exec(r.text)?.[1]?.toLowerCase();
36
+ r.level = r.tag === 'spec' ? 'must' : marked;
37
+ current = null;
38
+ };
39
+ for (const line of lines) {
40
+ const heading = /^(#{2,3})\s+(.*)$/.exec(line);
41
+ if (heading) {
42
+ close();
43
+ const title = heading[2].trim();
44
+ if (heading[1] === '##') {
45
+ if (/^controls\b/i.test(title))
46
+ mode = 'controls';
47
+ else if (/^test setup\b/i.test(title))
48
+ mode = 'setup';
49
+ else if (NOT_RULES.test(title))
50
+ mode = 'skip';
51
+ else
52
+ mode = 'rules';
53
+ }
54
+ else if (mode === 'skip' || mode === 'controls' || mode === 'setup') {
55
+ continue;
56
+ }
57
+ const num = /^(\d+(?:\.\d+)*)\.?\s+(.*)$/.exec(title);
58
+ if (mode === 'rules') {
59
+ section = num ? `§${num[1]} ${num[2]}` : title;
60
+ if (heading[1] === '##')
61
+ group = section;
62
+ }
63
+ continue;
64
+ }
65
+ if (/^#\s/.test(line))
66
+ continue; // the journey title
67
+ if (mode === 'header') {
68
+ header.push(line);
69
+ continue;
70
+ }
71
+ if (mode === 'setup') {
72
+ setup.push(line);
73
+ continue;
74
+ }
75
+ if (mode === 'controls') {
76
+ const row = /^\|\s*(C\d+[a-z]?)\s*\|(.*)\|\s*$/.exec(line);
77
+ if (row)
78
+ controls[row[1]] = row[2].split('|').map((c) => c.trim()).filter(Boolean).join(' · ');
79
+ continue;
80
+ }
81
+ if (mode !== 'rules')
82
+ continue;
83
+ const bullet = /^(\s*)[-*]\s+(.*)$/.exec(line);
84
+ if (bullet) {
85
+ const indent = bullet[1].length;
86
+ // A sub-bullet under a rule (its list of cases, its note) belongs to it.
87
+ if (current && indent > current.indent) {
88
+ current.rule.text += `\n${line}`;
89
+ continue;
90
+ }
91
+ close();
92
+ const tag = (TAG.exec(bullet[2])?.[1] ?? '');
93
+ current = { rule: { id: `R${rules.length + 1}`, section, group, tag, text: bullet[2], controls: [] }, indent };
94
+ rules.push(current.rule);
95
+ continue;
96
+ }
97
+ // A paragraph, a table or a blank line inside a rule's block stays with it.
98
+ if (current && (line.trim() === '' || /^\s/.test(line) || line.startsWith('|'))) {
99
+ current.rule.text += `\n${line}`;
100
+ continue;
101
+ }
102
+ close();
103
+ // Free text between rules (a section's lead-in) becomes context for the next rule's section.
104
+ if (line.trim())
105
+ rules.push({ id: `R${rules.length + 1}`, section, group, tag: '', text: line.trim(), controls: [...new Set(line.match(CONTROL_REF) ?? [])] });
106
+ }
107
+ close();
108
+ return { header: header.join('\n').trim(), rules, controls, setup: setup.join('\n').trim() };
109
+ }
110
+ /** One line per rule — what the planner and the gate see instead of the doc. */
111
+ export function ruleIndex(memory, tags = ['spec', 'baseline', 'observed']) {
112
+ return memory.rules
113
+ .filter((r) => r.tag && tags.includes(r.tag))
114
+ .map((r) => {
115
+ const oneLine = r.text.split('\n')[0].replace(TAG, '').replace(/\(source:[^)]*\)/g, '').replace(/\s+/g, ' ').trim();
116
+ return `${r.id} [${r.tag}] ${r.section} — ${oneLine.length > 160 ? `${oneLine.slice(0, 157)}…` : oneLine}`;
117
+ })
118
+ .join('\n');
119
+ }
120
+ /**
121
+ * The part of the journey one file needs: its own rules in full, the
122
+ * untagged notes and [observed] mechanics of the same sections, and the
123
+ * controls all of those name. Nothing else of the doc.
124
+ */
125
+ export function journeySlice(memory, ruleIds) {
126
+ const own = memory.rules.filter((r) => ruleIds.includes(r.id));
127
+ const sections = new Set(own.map((r) => r.section));
128
+ const context = memory.rules.filter((r) => !ruleIds.includes(r.id) && sections.has(r.section) && (r.tag === '' || r.tag === 'observed'));
129
+ const controlIds = [...new Set([...own, ...context].flatMap((r) => r.controls))].sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
130
+ return { text: sliceText(memory, own, context, controlIds), controls: controlIds };
131
+ }
132
+ function sliceText(memory, own, context, controlIds) {
133
+ const bySection = (rs) => [...new Set(rs.map((r) => r.section))].map((s) => [`### ${s}`, ...rs.filter((r) => r.section === s).map((r) => `- ${r.id}: ${r.text}`)].join('\n')).join('\n\n');
134
+ return [
135
+ memory.header,
136
+ '',
137
+ controlIds.length ? ['## Controls', ...controlIds.map((c) => `- ${c}: ${memory.controls[c] ?? '(not in the Controls table)'}`)].join('\n') : '',
138
+ '',
139
+ '## Rules this file tests',
140
+ bySection(own),
141
+ ...(context.length ? ['', '## Context from the same sections (not tested here, explains how the app works)', bySection(context)] : []),
142
+ ...(memory.setup ? ['', '## Test setup', memory.setup] : []),
143
+ ].filter((x, i, a) => !(x === '' && a[i - 1] === '')).join('\n').trim();
144
+ }
145
+ /**
146
+ * The testkit's exports, one line each: signature plus its comment. Read from
147
+ * the file, not asked of the AI, so every writer gets the same true list
148
+ * without opening the testkit itself.
149
+ */
150
+ export function helperApi(harnessSource) {
151
+ const names = new Set();
152
+ const cjs = /module\.exports\s*=\s*\{([\s\S]*?)\}/.exec(harnessSource);
153
+ if (cjs)
154
+ for (const n of cjs[1].split(',')) {
155
+ const m = /^\s*(\w+)/.exec(n);
156
+ if (m)
157
+ names.add(m[1]);
158
+ }
159
+ for (const m of harnessSource.matchAll(/^export\s+(?:async\s+)?(?:function\*?|const|let|class)\s+(\w+)/gm))
160
+ names.add(m[1]);
161
+ for (const m of harnessSource.matchAll(/^export\s*\{([^}]*)\}/gm))
162
+ for (const n of m[1].split(',')) {
163
+ const w = /(\w+)\s*$/.exec(n.trim());
164
+ if (w)
165
+ names.add(w[1]);
166
+ }
167
+ const lines = harnessSource.split('\n');
168
+ const out = [];
169
+ for (const name of names) {
170
+ const at = lines.findIndex((l) => new RegExp(`^\\s*(?:export\\s+)?(?:async\\s+)?(?:function\\*?\\s+${name}\\s*\\(|(?:const|let|var)\\s+${name}\\s*=)`).test(l));
171
+ if (at === -1) {
172
+ out.push(`- ${name}`);
173
+ continue;
174
+ }
175
+ let sig = lines[at].trim().replace(/^export\s+/, '').replace(/\s*\{\s*\}?\s*$/, '');
176
+ // A constant's value is what the writer needs (TEST_IDS, a URL map): keep it whole, within reason.
177
+ if (/^(const|let|var)\s/.test(sig) && /[{[]\s*$/.test(lines[at])) {
178
+ let depth = 0;
179
+ const body = [];
180
+ for (let i = at; i < lines.length && body.join('\n').length < 1500; i++) {
181
+ body.push(lines[i]);
182
+ depth += (lines[i].match(/[{[]/g) ?? []).length - (lines[i].match(/[}\]]/g) ?? []).length;
183
+ if (depth <= 0)
184
+ break;
185
+ }
186
+ out.push(`- ${body.join('\n ').replace(/^export\s+/, '')}`);
187
+ continue;
188
+ }
189
+ const doc = [];
190
+ for (let i = at - 1; i >= 0 && doc.length < 3; i--) {
191
+ const t = lines[i].trim();
192
+ if (!t || t === '*/' || t === '/**')
193
+ continue;
194
+ if (!/^(\/\/|\*|\/\*\*)/.test(t))
195
+ break;
196
+ const text = t.replace(/^(\/\/+|\/\*\*|\*\/?)\s?/, '').replace(/^-{4,}.*$/, '').trim();
197
+ if (text)
198
+ doc.unshift(text);
199
+ }
200
+ out.push(`- ${sig.length > 200 ? `${sig.slice(0, 197)}…` : sig}${doc.length ? ` — ${doc.join(' ')}` : ''}`);
201
+ }
202
+ return out.join('\n');
203
+ }
204
+ /**
205
+ * The code facts a file needs: every section of the grounding as it is,
206
+ * except SCREENS, of which only the lines for this file's controls (and the
207
+ * untagged ones) are kept. Facts without control tags are passed whole.
208
+ */
209
+ export function factsSlice(facts, controlIds) {
210
+ if (!facts)
211
+ return facts;
212
+ const lines = facts.split('\n');
213
+ const tagged = /^\s*[-*]?\s*\[(C\d+[a-z]?(?:\s*,\s*C\d+[a-z]?)*)\]/;
214
+ if (!lines.some((l) => tagged.test(l)))
215
+ return facts;
216
+ const want = new Set(controlIds);
217
+ return lines
218
+ .filter((l) => {
219
+ const m = tagged.exec(l);
220
+ return !m || m[1].split(',').some((c) => want.has(c.trim()));
221
+ })
222
+ .join('\n');
223
+ }
224
+ /**
225
+ * Write the journey's memory: its rules with source, tests, skips and history.
226
+ * Earlier wordings and removed rules are kept, so "why did this test change"
227
+ * has an answer. Pure bookkeeping, no AI.
228
+ */
229
+ export function saveMemory(journey, docHash, memory, helpers, extra) {
230
+ const path = join(journeyDir(journey), 'rules.json');
231
+ const now = new Date().toISOString();
232
+ const prev = extra.prev;
233
+ const prevRule = new Map((prev?.rules ?? []).map((r) => [r.id, r]));
234
+ const renamed = new Set(extra.changes?.renamed ?? []);
235
+ const changed = new Set(extra.changes?.changed ?? []);
236
+ const skip = new Map((extra.skipped ?? []).map((x) => [x.rule, x.why]));
237
+ const rules = memory.rules.map((r) => {
238
+ const before = prevRule.get(r.id);
239
+ const moved = renamed.has(r.id) || changed.has(r.id);
240
+ const quote = /\(source:[^"“]*["“]([^"”]+)["”]/.exec(r.text)?.[1];
241
+ return {
242
+ ...r,
243
+ category: /\]\s*\((?:must|should|could),\s*([^)]+)\)/i.exec(r.text)?.[1]?.trim(),
244
+ source: extra.reference || quote ? { reference: extra.reference, quote } : undefined,
245
+ testedBy: (extra.plan ?? []).filter((f) => f.rules?.includes(r.id)).map((f) => f.slug),
246
+ skipped: skip.get(r.id),
247
+ since: before?.since ?? now,
248
+ updated: moved || !before ? now : before.updated,
249
+ history: moved && before
250
+ ? [...(before.history ?? []), { at: now, change: renamed.has(r.id) ? 'renamed' : 'changed', was: before.text }]
251
+ : before?.history,
252
+ };
253
+ });
254
+ const removedNow = (extra.changes?.removed ?? []).flatMap((id) => {
255
+ const r = prevRule.get(id);
256
+ return r ? [{ id, text: r.text, at: now }] : [];
257
+ });
258
+ const record = {
259
+ journey: journey.id,
260
+ name: journey.name,
261
+ doc: extra.doc,
262
+ hash: docHash,
263
+ reference: extra.reference,
264
+ updated: now,
265
+ header: memory.header,
266
+ controls: memory.controls,
267
+ setup: memory.setup,
268
+ helpers,
269
+ rules,
270
+ removed: [...(prev?.removed ?? []), ...removedNow],
271
+ };
272
+ try {
273
+ mkdirSync(dirname(path), { recursive: true });
274
+ writeFileSync(path, JSON.stringify(record, null, 2), 'utf-8');
275
+ }
276
+ catch {
277
+ // best-effort: the memory is rebuilt from the doc when it is missing
278
+ }
279
+ return path;
280
+ }
281
+ /**
282
+ * Rules of this journey that another journey already tests, found without the
283
+ * AI: the same tag and nearly the same words (≥ 85 % of the words shared).
284
+ * They are not planned again; the plan check names where they are tested.
285
+ */
286
+ export function testedElsewhere(journey, memory, tested) {
287
+ const others = [];
288
+ const root = join(journeyDir(journey), '..', '..');
289
+ for (const area of existsSync(root) ? readdirSync(root) : []) {
290
+ for (const id of existsSync(join(root, area)) ? readdirSync(join(root, area)) : []) {
291
+ if (id === journey.id)
292
+ continue;
293
+ try {
294
+ others.push(JSON.parse(readFileSync(join(root, area, id, 'rules.json'), 'utf-8')));
295
+ }
296
+ catch {
297
+ // no memory for that journey yet
298
+ }
299
+ }
300
+ }
301
+ const words = (t) => new Set(t.replace(/\(source:[^)]*\)/g, '').toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 2));
302
+ const out = [];
303
+ for (const r of memory.rules) {
304
+ if (!r.tag || !tested.includes(r.tag))
305
+ continue;
306
+ const mine = words(r.text);
307
+ for (const o of others) {
308
+ const hit = o.rules.find((x) => {
309
+ if (x.tag !== r.tag || !x.testedBy?.length)
310
+ return false;
311
+ const theirs = words(x.text);
312
+ let both = 0;
313
+ for (const w of mine)
314
+ if (theirs.has(w))
315
+ both += 1;
316
+ return mine.size + theirs.size > 0 && both / (mine.size + theirs.size - both) >= 0.85;
317
+ });
318
+ if (hit) {
319
+ out.push({ rule: r.id, why: `already tested in ${o.journey} (${hit.id}, ${hit.testedBy.join(', ')})` });
320
+ break;
321
+ }
322
+ }
323
+ }
324
+ return out;
325
+ }
@@ -9,8 +9,9 @@
9
9
  */
10
10
  import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
11
11
  import { dirname, join } from 'node:path';
12
- import { planSuite } from './suite-planner.js';
13
- import { PLANS_DIR } from '../paths.js';
12
+ import { planBySections, planSuite, testedTags } from './suite-planner.js';
13
+ import { loadConfig } from '../project/config.js';
14
+ import { journeyDir } from '../paths.js';
14
15
  /** The line every generated file carries, so cleanup never touches a hand-written one. */
15
16
  export const GEN_MARKER = '@generated by testforge';
16
17
  /**
@@ -48,9 +49,9 @@ export function collisionWarning(testDir, id, ext) {
48
49
  return `an existing test with different naming already covers this journey: ${clashes.join(', ')} — new file may duplicate it`;
49
50
  }
50
51
  /** Plan cache path for a journey, organised per area:
51
- * `<PLANS_DIR>/<area>/<id>.json` (e.g. test-artifacts/test-plans/api/payment-api.json). */
52
+ * testforge/memory/<area>/<id>/plan.json. */
52
53
  export function planPath(journey) {
53
- return join(PLANS_DIR, journey.area, `${journey.id}.json`);
54
+ return join(journeyDir(journey), 'plan.json');
54
55
  }
55
56
  /**
56
57
  * F9 — a journey doc that is still an unconfirmed stub. Deep-generating
@@ -146,7 +147,9 @@ export function cleanStaleFiles(testDir, journeyId, ext, flat, keepSlugs, dropSl
146
147
  * actually matches — and it saves the planner AI call on a resume. When
147
148
  * the doc changes, the hash changes and we re-plan.
148
149
  */
149
- export function loadOrPlan(journey, docHash) {
150
+ export function loadOrPlan(journey, docHash, memory,
151
+ /** Rules not to plan, with why (another journey already tests them). */
152
+ preSkips = []) {
150
153
  const p = planPath(journey);
151
154
  if (existsSync(p)) {
152
155
  try {
@@ -159,16 +162,120 @@ export function loadOrPlan(journey, docHash) {
159
162
  // corrupt plan cache — re-plan
160
163
  }
161
164
  }
162
- const files = planSuite(journey);
165
+ const skipped = [...preSkips];
166
+ const pre = new Set(preSkips.map((x) => x.rule));
167
+ // The doc's sections are the plan unless the project asks the AI to plan.
168
+ let files = memory && loadConfig().planner !== 'ai'
169
+ ? planBySections(journey, memory, skipped)
170
+ : planSuite(journey, memory && pre.size ? { ...memory, rules: memory.rules.filter((r) => !pre.has(r.id)) } : memory, skipped);
171
+ if (memory) {
172
+ const gate = gatePlan(files, skipped, memory, journey.name);
173
+ files = gate.files;
174
+ try {
175
+ mkdirSync(dirname(p), { recursive: true });
176
+ writeFileSync(join(dirname(p), 'plan-check.md'), gate.report, 'utf-8');
177
+ }
178
+ catch {
179
+ // the report is for people; the plan below is what runs
180
+ }
181
+ console.log(` ${journey.id}: ${gate.summary}`);
182
+ }
163
183
  try {
164
184
  mkdirSync(dirname(p), { recursive: true });
165
- writeFileSync(p, JSON.stringify({ hash: docHash, files }, null, 2), 'utf-8');
185
+ writeFileSync(p, JSON.stringify({ hash: docHash, files, skipped }, null, 2), 'utf-8');
166
186
  }
167
187
  catch {
168
188
  // best-effort cache
169
189
  }
170
190
  return files;
171
191
  }
192
+ /** Rules the saved plan left out, with why. */
193
+ export function readPlanSkips(journey) {
194
+ try {
195
+ const c = JSON.parse(readFileSync(planPath(journey), 'utf-8'));
196
+ return Array.isArray(c.skipped) ? c.skipped : [];
197
+ }
198
+ catch {
199
+ return [];
200
+ }
201
+ }
202
+ /**
203
+ * The plan, checked before a line of test code is written: every file tests
204
+ * at least one rule that gets tests, no rule is tested twice, and anything not
205
+ * planned is listed with the reason. A file the gate drops costs nothing; the
206
+ * same file written, run and repaired cost most of a journey's AI budget.
207
+ */
208
+ export function gatePlan(planned, skipped, memory, name = 'journey') {
209
+ const tags = testedTags();
210
+ const byId = new Map(memory.rules.map((r) => [r.id, r]));
211
+ const testable = memory.rules.filter((r) => r.tag && tags.includes(r.tag));
212
+ const owner = new Map();
213
+ const dropped = [];
214
+ const duplicates = [];
215
+ const files = [];
216
+ for (const f of planned) {
217
+ const rules = [];
218
+ for (const id of f.rules ?? []) {
219
+ const r = byId.get(id);
220
+ if (!r || !r.tag || !tags.includes(r.tag))
221
+ continue;
222
+ if (owner.has(id)) {
223
+ duplicates.push(`${id} (in ${owner.get(id)} and ${f.slug})`);
224
+ continue;
225
+ }
226
+ owner.set(id, f.slug);
227
+ rules.push(id);
228
+ }
229
+ if (rules.length)
230
+ files.push({ ...f, rules });
231
+ else
232
+ dropped.push(f.slug);
233
+ }
234
+ const skippedIds = new Set(skipped.map((s) => s.rule));
235
+ const missing = testable.filter((r) => !owner.has(r.id) && !skippedIds.has(r.id));
236
+ // A "must" (every [spec], and security, money, concurrency and error-boundary
237
+ // [baseline]) may be left out only with a reason the planner gave.
238
+ const mustMissing = missing.filter((r) => r.level === 'must');
239
+ const line = (id) => {
240
+ const r = byId.get(id);
241
+ return r ? `${id} [${r.tag}] ${r.section} — ${r.text.split('\n')[0].replace(/\(source:[^)]*\)/g, '').trim().slice(0, 140)}` : id;
242
+ };
243
+ const report = [
244
+ `# Plan check — ${name}`,
245
+ '',
246
+ `Rules that get tests (${tags.join(', ')}): ${testable.length}. Planned: ${owner.size}. Skipped with a reason: ${skippedIds.size}. Not planned: ${missing.length}.`,
247
+ '',
248
+ '## Files',
249
+ ...files.map((f) => `- **${f.slug}** — ${f.covers}\n ${(f.rules ?? []).join(', ')}`),
250
+ ...(skipped.length ? ['', '## Skipped by the planner', ...skipped.map((s) => `- ${line(s.rule)}\n why: ${s.why || '(no reason given)'}`)] : []),
251
+ ...(mustMissing.length ? ['', '## MUST rules not planned', ...mustMissing.map((r) => `- ${line(r.id)}`)] : []),
252
+ ...(missing.length > mustMissing.length ? ['', '## Not planned (no file, no reason)', ...missing.filter((r) => r.level !== 'must').map((r) => `- ${line(r.id)}`)] : []),
253
+ ...(duplicates.length ? ['', '## Planned twice (kept in the first file)', ...duplicates.map((d) => `- ${d}`)] : []),
254
+ ...(dropped.length ? ['', '## Files dropped (no rule that gets tests)', ...dropped.map((d) => `- ${d}`)] : []),
255
+ ].join('\n');
256
+ const summary = `plan check: ${files.length} file(s), ${owner.size}/${testable.length} rules planned, ${skippedIds.size} skipped${missing.length ? `, ${missing.length} NOT planned${mustMissing.length ? ` (${mustMissing.length} MUST)` : ''}` : ''}${dropped.length ? `, ${dropped.length} file(s) dropped` : ''}${duplicates.length ? `, ${duplicates.length} duplicate(s) removed` : ''}`;
257
+ return { files, report, summary };
258
+ }
259
+ /** The saved plan's files, whatever doc version it was made for. */
260
+ export function readPlan(journey) {
261
+ try {
262
+ const c = JSON.parse(readFileSync(planPath(journey), 'utf-8'));
263
+ return Array.isArray(c.files) ? c.files : undefined;
264
+ }
265
+ catch {
266
+ return undefined;
267
+ }
268
+ }
269
+ /** Save a plan as the one for this version of the doc. */
270
+ export function writePlan(journey, docHash, files, skipped = readPlanSkips(journey)) {
271
+ try {
272
+ mkdirSync(dirname(planPath(journey)), { recursive: true });
273
+ writeFileSync(planPath(journey), JSON.stringify({ hash: docHash, files, skipped }, null, 2), 'utf-8');
274
+ }
275
+ catch {
276
+ // best-effort cache
277
+ }
278
+ }
172
279
  /** Does this doc file / journey match a --journey selector? */
173
280
  export function matchesJourney(file, journeyId, sel) {
174
281
  const norm = (s) => s.toLowerCase().replace(/[^a-z0-9]+/g, '-');