@activate-core/test-forge 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +98 -34
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +51 -4
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +51 -24
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +68 -5
  10. package/dist/core/docs/review.js +81 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +56 -0
  13. package/dist/core/gates/coverage.js +12 -19
  14. package/dist/core/gates/validate.js +277 -4
  15. package/dist/core/generate/ai-generator.js +243 -96
  16. package/dist/core/generate/ai-session.js +8 -0
  17. package/dist/core/generate/ai-usage.js +150 -0
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/defect-review.js +213 -20
  20. package/dist/core/generate/doc-diff.js +139 -0
  21. package/dist/core/generate/harness.js +52 -32
  22. package/dist/core/generate/journey-memory.js +325 -0
  23. package/dist/core/generate/plan-cache.js +114 -7
  24. package/dist/core/generate/required-data.js +75 -0
  25. package/dist/core/generate/review-journey.js +104 -0
  26. package/dist/core/generate/run-generate.js +193 -80
  27. package/dist/core/generate/seed-generator.js +2 -2
  28. package/dist/core/generate/suite-planner.js +138 -25
  29. package/dist/core/generate/test-blocks.js +84 -0
  30. package/dist/core/generate/update-journey.js +115 -0
  31. package/dist/core/generate/verify.js +9 -4
  32. package/dist/core/paths.js +90 -13
  33. package/dist/core/project/config.js +25 -2
  34. package/dist/core/project/env-diff.js +122 -0
  35. package/dist/core/project/env-template.js +3 -2
  36. package/dist/core/project/init.js +2 -2
  37. package/dist/core/project/map.js +2 -2
  38. package/dist/core/run/ci.js +94 -0
  39. package/dist/core/run/clean.js +111 -0
  40. package/dist/core/run/readiness.js +119 -0
  41. package/dist/core/run/report.js +2 -0
  42. package/dist/core/run/run-tests.js +23 -1
  43. package/package.json +1 -1
  44. package/reporters/_shared.cjs +24 -3
  45. package/reporters/jest-reporter.cjs +3 -1
  46. package/reporters/playwright-reporter.cjs +3 -1
  47. package/reporters/run-data.cjs +66 -0
  48. package/reporters/ui-helpers.cjs +133 -0
  49. package/dist/core/generate/verify-journey.js +0 -174
@@ -0,0 +1,245 @@
1
+ /**
2
+ * testforge — the source a journey names, collected without the AI.
3
+ *
4
+ * A journey doc already says where its behaviour lives: the [observed] rules
5
+ * cite files and lines, and the rules quote the texts a screen shows. Grepping
6
+ * for those is free; an AI doing the same with its tools took 25–32 turns, each
7
+ * re-sending everything read so far. The excerpts below go into ONE call.
8
+ */
9
+ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
10
+ import { join, relative, resolve } from 'node:path';
11
+ const CODE = /\.(tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go)$/;
12
+ const SKIP_DIR = /^(node_modules|\.next|dist|build|coverage|vendor|storage|\.git|tests?|__tests__|__mocks__|test-artifacts|testforge|generated-tests)$/;
13
+ const FILE_REF = /((?:[\w.-]+\/)+[\w.-]+\.(?:tsx?|jsx?|mjs|cjs|vue|svelte|php|py|rb|go))(?::(\d+)(?:[-–](\d+))?)?/g;
14
+ const QUOTED = /"([^"\n]{4,60})"/g;
15
+ /** Words in a control's name that name no component in particular. */
16
+ const GENERIC = new Set(['button', 'buttons', 'field', 'input', 'inputs', 'header', 'footer', 'items', 'entry', 'entries', 'shown', 'guest', 'logged', 'customer', 'computer', 'phone', 'sized', 'screen', 'page', 'pages', 'where', 'opens', 'close', 'link', 'links', 'text', 'label', 'click', 'state', 'result', 'neighbour', 'neighbouring', 'other', 'while', 'their', 'there', 'which', 'every', 'itself', 'changes', 'layout']);
17
+ function walk(dir, out) {
18
+ let entries;
19
+ try {
20
+ entries = readdirSync(dir);
21
+ }
22
+ catch {
23
+ return;
24
+ }
25
+ for (const e of entries) {
26
+ const p = join(dir, e);
27
+ let st;
28
+ try {
29
+ st = statSync(p);
30
+ }
31
+ catch {
32
+ continue;
33
+ }
34
+ if (st.isDirectory()) {
35
+ if (!SKIP_DIR.test(e))
36
+ walk(p, out);
37
+ }
38
+ else if (CODE.test(e) && !/\.(test|spec)\.\w+$/.test(e) && st.size < 300_000)
39
+ out.push(p);
40
+ }
41
+ }
42
+ /**
43
+ * A file named in the doc, found under the project or a related repo. Docs
44
+ * often shorten a path ("header/index.tsx"): that is matched as a suffix, and
45
+ * kept only when it points at no more than two files.
46
+ */
47
+ function locate(ref, roots, all) {
48
+ for (const base of [process.cwd(), ...roots]) {
49
+ for (const cand of [join(base, ref), join(base, '..', ref)])
50
+ if (existsSync(cand) && statSync(cand).isFile())
51
+ return [resolve(cand)];
52
+ }
53
+ const hits = all.filter((f) => f.endsWith(`/${ref}`));
54
+ return hits.length <= 2 ? hits : [];
55
+ }
56
+ /**
57
+ * The lines of the project's source a journey doc points at: every file it
58
+ * cites (around the cited lines), then every file that contains a text the
59
+ * doc quotes, most hits first, until `budget` characters.
60
+ */
61
+ export function sourceExcerpts(markdown, roots, budget = 50_000) {
62
+ const windows = new Map();
63
+ const add = (file, from, to) => {
64
+ const w = windows.get(file) ?? [];
65
+ w.push([Math.max(1, from), to]);
66
+ windows.set(file, w);
67
+ };
68
+ const order = [];
69
+ const files = [];
70
+ for (const r of roots)
71
+ walk(resolve(r), files);
72
+ for (const m of markdown.matchAll(FILE_REF)) {
73
+ for (const file of locate(m[1], roots, files)) {
74
+ const a = Number(m[2] ?? 0);
75
+ const b = Number(m[3] ?? a);
76
+ if (a)
77
+ add(file, a - 20, b + 20);
78
+ else
79
+ add(file, 1, 80);
80
+ if (!order.includes(file))
81
+ order.push(file);
82
+ }
83
+ }
84
+ // A file named after a control ("Escalate button" → escalateDisputeButton/) is
85
+ // the component itself: found by its path, before any text search.
86
+ const controlWords = [...markdown.matchAll(/^\|\s*C\d+[a-z]?\s*\|([^|]+)\|/gm)]
87
+ .flatMap((m) => m[1].replace(/["“”(][^"”)]*["”)]?/g, ' ').match(/[A-Za-z]{5,}/g) ?? [])
88
+ .map((w) => w.toLowerCase())
89
+ .filter((w) => !GENERIC.has(w));
90
+ const pathHits = files.filter((f) => {
91
+ const p = relative(process.cwd(), f).toLowerCase();
92
+ return [...new Set(controlWords)].some((w) => p.includes(w));
93
+ });
94
+ // How a component is reached and fed: the files that import it, up to the
95
+ // page that serves it (its route), and the hooks and API modules they use.
96
+ const importersOf = (target) => {
97
+ const stem = target.replace(/[\\/]index\.\w+$/, '').replace(/\.\w+$/, '');
98
+ const name = stem.replace(/^.*[\\/]/, '');
99
+ return files.filter((u) => {
100
+ if (u === target)
101
+ return false;
102
+ try {
103
+ return new RegExp(`from\\s+['"][^'"]*[\\/]${name}['"]`).test(readFileSync(u, 'utf-8'));
104
+ }
105
+ catch {
106
+ return false;
107
+ }
108
+ });
109
+ };
110
+ const feeds = (file) => {
111
+ let text = '';
112
+ try {
113
+ text = readFileSync(file, 'utf-8');
114
+ }
115
+ catch {
116
+ return [];
117
+ }
118
+ const specs = [...text.matchAll(/from\s+['"]([^'"]+)['"]/g)].map((m) => m[1]).filter((sp) => /(^|[\\/])(use[A-Z]\w*|api|services?|types?|dataProvider|queries)([\\/]|$)/.test(sp));
119
+ return files.filter((f) => specs.some((sp) => f.replace(/\.\w+$/, '').replace(/[\\/]index$/, '').endsWith(sp.replace(/^[@~.]+[\\/]?/, '').replace(/^src[\\/]/, ''))));
120
+ };
121
+ for (const f of pathHits.slice(0, 12)) {
122
+ let level = [f];
123
+ for (let hop = 0; hop < 3 && level.length; hop++) {
124
+ const next = level.flatMap(importersOf).filter((u) => !order.includes(u)).slice(0, 4);
125
+ for (const u of next) {
126
+ add(u, 1, /[\\/](app|pages)[\\/]/.test(u) ? 40 : 15);
127
+ order.push(u);
128
+ for (const h of feeds(u).slice(0, 3))
129
+ if (!order.includes(h)) {
130
+ add(h, 1, 80);
131
+ order.push(h);
132
+ }
133
+ }
134
+ level = next;
135
+ }
136
+ }
137
+ for (const f of pathHits.slice(0, 12)) {
138
+ add(f, 1, 120);
139
+ if (!order.includes(f))
140
+ order.push(f);
141
+ // And where it is used: the page that renders it says how to reach it.
142
+ const name = /[\\/]([^\\/]+)[\\/]index\.\w+$/.exec(f)?.[1] ?? f.replace(/^.*[\\/]|\.\w+$/g, '');
143
+ for (const user of files) {
144
+ if (user === f)
145
+ continue;
146
+ let text;
147
+ try {
148
+ text = readFileSync(user, 'utf-8');
149
+ }
150
+ catch {
151
+ continue;
152
+ }
153
+ if (!new RegExp(`from\\s+['"][^'"]*\\b${name}['"]`).test(text))
154
+ continue;
155
+ text.split('\n').forEach((l, i) => { if (l.includes(name) || /<[A-Z]\w*/.test(l) && l.includes(name.charAt(0).toUpperCase() + name.slice(1)))
156
+ add(user, i + 1 - 10, i + 1 + 10); });
157
+ if (!order.includes(user))
158
+ order.push(user);
159
+ }
160
+ }
161
+ const quoted = [...markdown.matchAll(QUOTED)].map((m) => m[1].trim()).filter((t) => /[a-z]/i.test(t) && !/\.\s/.test(t));
162
+ const named = [...markdown.matchAll(/`([A-Za-z_$][\w$]*(?:\.[\w$]+)*)`/g)].map((m) => m[1]).filter((t) => t.length >= 4 && /[A-Z_.]/.test(t));
163
+ const terms = [...new Set([...quoted, ...named])];
164
+ if (terms.length) {
165
+ const hits = [];
166
+ for (const f of files) {
167
+ let text;
168
+ try {
169
+ text = readFileSync(f, 'utf-8');
170
+ }
171
+ catch {
172
+ continue;
173
+ }
174
+ const found = terms.filter((t) => text.includes(t));
175
+ if (!found.length)
176
+ continue;
177
+ const lines = text.split('\n');
178
+ hits.push({ file: f, terms: found, lines: lines.flatMap((l, i) => (found.some((t) => l.includes(t)) ? [i + 1] : [])) });
179
+ }
180
+ // A text found in one file says more about it than one found in twenty; one
181
+ // found in more than six ("Log In", "Signup") points at nothing in particular.
182
+ const spread = new Map(terms.map((t) => [t, hits.filter((h) => h.terms.includes(t)).length]));
183
+ for (const h of hits) {
184
+ h.terms = h.terms.filter((t) => (spread.get(t) ?? 0) <= 6);
185
+ h.lines = h.lines.filter((n) => h.terms.length > 0 && n > 0);
186
+ }
187
+ for (let i = hits.length - 1; i >= 0; i--)
188
+ if (!hits[i].terms.length)
189
+ hits.splice(i, 1);
190
+ const score = (h) => h.terms.reduce((n, t) => n + 1 / (spread.get(t) || 1), 0);
191
+ hits.sort((x, y) => score(y) - score(x));
192
+ for (const h of hits) {
193
+ const text = readFileSync(h.file, 'utf-8').split('\n');
194
+ const at = text.flatMap((l, i) => (h.terms.some((t) => l.includes(t)) ? [i + 1] : []));
195
+ for (const n of at)
196
+ add(h.file, n - 8, n + 8);
197
+ if (!order.includes(h.file))
198
+ order.push(h.file);
199
+ }
200
+ }
201
+ const out = [];
202
+ let used = 0;
203
+ for (const file of order) {
204
+ const lines = readFileSync(file, 'utf-8').split('\n');
205
+ const merged = (windows.get(file) ?? [])
206
+ .map(([a, b]) => [a, Math.min(b, lines.length)])
207
+ .sort((x, y) => x[0] - y[0])
208
+ .reduce((acc, w) => {
209
+ const last = acc[acc.length - 1];
210
+ if (last && w[0] <= last[1] + 1)
211
+ last[1] = Math.max(last[1], w[1]);
212
+ else
213
+ acc.push([...w]);
214
+ return acc;
215
+ }, []);
216
+ const body = merged.map(([a, b]) => lines.slice(a - 1, b).map((l, i) => `${a + i}: ${l}`).join('\n')).join('\n…\n');
217
+ const block = `### ${relative(process.cwd(), file)}\n${body}\n`;
218
+ if (used + block.length > budget) {
219
+ if (used === 0)
220
+ out.push(block.slice(0, budget));
221
+ break;
222
+ }
223
+ out.push(block);
224
+ used += block.length;
225
+ }
226
+ return out.join('\n');
227
+ }
228
+ /** All of the source under `roots`, lower-cased with whitespace collapsed: what a label check searches. */
229
+ export function sourceText(roots) {
230
+ const files = [];
231
+ for (const r of roots)
232
+ walk(resolve(r), files);
233
+ return files
234
+ .map((f) => {
235
+ try {
236
+ return readFileSync(f, 'utf-8');
237
+ }
238
+ catch {
239
+ return '';
240
+ }
241
+ })
242
+ .join('\n')
243
+ .toLowerCase()
244
+ .replace(/\s+/g, ' ');
245
+ }
@@ -17,15 +17,17 @@
17
17
  * journey document remains the sole source of what is CORRECT, and the
18
18
  * generation prompt says so in the same breath as it presents these facts.
19
19
  */
20
- import { execFileSync } from 'node:child_process';
21
20
  import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
22
21
  import { dirname, join } from 'node:path';
23
- import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
22
+ import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
23
+ import { readRules, ruleIndex } from '../generate/journey-memory.js';
24
+ import { sourceExcerpts } from './excerpts.js';
24
25
  import { hashProjectSource } from './cache.js';
25
- import { STATE_DIR } from '../paths.js';
26
+ import { journeyDir } from '../paths.js';
26
27
  import { loadEnvFile } from '../project/config.js';
27
28
  import { originsFrom, pathsFrom, observeApp } from './observe.js';
28
29
  import { observeSchema } from './schema.js';
30
+ import { runAiSync } from '../generate/ai-usage.js';
29
31
  const MAP_PATH = 'testforge.md';
30
32
  /**
31
33
  * Bumped whenever the SHAPE of a facts block changes.
@@ -35,27 +37,31 @@ const MAP_PATH = 'testforge.md';
35
37
  * that upgrades keeps serving the facts it cached under the old version —
36
38
  * so the new facts never reach a single prompt and the upgrade does nothing.
37
39
  */
38
- const FACTS_VERSION = 2;
40
+ const FACTS_VERSION = 4;
39
41
  /** Where a journey's grounded facts live, mirroring the plan cache layout. */
40
42
  function groundingPath(journey) {
41
- const area = (journey.area || 'misc').replace(/[^a-z0-9-]/gi, '-');
42
- const slug = (journey.name || 'journey')
43
- .toLowerCase()
44
- .replace(/[^a-z0-9]+/g, '-')
45
- .replace(/^-|-$/g, '');
46
- return join(STATE_DIR, 'grounding', area, `${slug}.json`);
43
+ return join(journeyDir(journey), 'facts.json');
47
44
  }
48
- function buildGroundingPrompt(journey, map, srcDir) {
45
+ function buildGroundingPrompt(journey, map, srcDir, excerpts = '') {
49
46
  const isUi = journey.type !== 'api';
50
47
  return [
51
48
  `You are a senior engineer establishing the MECHANICS of one user`,
52
49
  `journey so tests can be written against it without every writer`,
53
50
  `re-reading the same source.`,
54
51
  ``,
55
- `You are inside the project's repository with read-only tools (Read,`,
56
- `Grep, Glob, LS). Read the real code for this journey under "${srcDir}":`,
57
- `start from the route/endpoint the journey names, then the`,
58
- `controller/handler, the service it calls, and the models it touches.`,
52
+ ...(excerpts
53
+ ? [
54
+ `The parts of this project's source that the journey names (the files`,
55
+ `it cites, and the files that contain the texts it quotes) are under`,
56
+ `SOURCE EXCERPTS below, with line numbers. You have no tools: work`,
57
+ `from those excerpts only.`,
58
+ ]
59
+ : [
60
+ `You are inside the project's repository with read-only tools (Read,`,
61
+ `Grep, Glob, LS). Read the real code for this journey under "${srcDir}":`,
62
+ `start from the route/endpoint the journey names, then the`,
63
+ `controller/handler, the service it calls, and the models it touches.`,
64
+ ]),
59
65
  ``,
60
66
  `Report ONLY what you actually read. Never guess a path, a status code`,
61
67
  `or a field name: a fact invented here is repeated into every test file`,
@@ -95,6 +101,11 @@ function buildGroundingPrompt(journey, map, srcDir) {
95
101
  ` label/placeholder text, on-screen text, input name/id`,
96
102
  ` attributes, and any data-testid that truly exists in the`,
97
103
  ` source. Never list a selector you did not see.`,
104
+ ` Start every SCREENS line with the ids from the journey's Controls`,
105
+ ` table that it serves, in brackets: "[C3,C4] header/index.tsx: link`,
106
+ ` 'FAQs' (role link)". Each test file is given only the lines for its`,
107
+ ` own controls, so a line without its ids reaches every file and a`,
108
+ ` line with the wrong ids reaches none.`,
98
109
  ]
99
110
  : []),
100
111
  ``,
@@ -102,13 +113,19 @@ function buildGroundingPrompt(journey, map, srcDir) {
102
113
  `restating the journey, no advice about testing. Aim for well under 200`,
103
114
  `lines. Output the headings and their contents only.`,
104
115
  ``,
105
- `--- PROJECT MAP ---`,
106
- map,
107
- ``,
116
+ ...(excerpts ? [] : [`--- PROJECT MAP ---`, map, ``]),
108
117
  `--- JOURNEY ---`,
109
- journey.rawMarkdown || journey.name,
118
+ ...(excerpts ? [compactJourney(journey.rawMarkdown || journey.name)] : [journey.rawMarkdown || journey.name]),
119
+ ...(excerpts ? [``, `--- SOURCE EXCERPTS ---`, excerpts] : []),
110
120
  ].join('\n');
111
121
  }
122
+ /** The journey as its controls and one line per rule: what the facts must serve, without the prose. */
123
+ function compactJourney(markdown) {
124
+ const m = readRules(markdown);
125
+ if (!m.rules.length)
126
+ return markdown;
127
+ return [m.header, ``, `CONTROLS:`, ...Object.entries(m.controls).map(([id, c]) => `${id}: ${c}`), ``, `RULES:`, ruleIndex(m)].join('\n');
128
+ }
112
129
  /**
113
130
  * Establish (or reuse) the journey's code facts. Cached against the doc hash
114
131
  * AND the source hash: the mechanics only move when the code or the journey
@@ -117,7 +134,9 @@ function buildGroundingPrompt(journey, map, srcDir) {
117
134
  * Never throws. If the AI call fails we return null and the caller falls back
118
135
  * to telling each file to read the source itself — slower, but still correct.
119
136
  */
120
- export async function groundJourney(journey, docHash, srcDir = 'src', markdown = '') {
137
+ export async function groundJourney(journey, docHash, srcDir = 'src', markdown = '',
138
+ /** A doc edit, not a new journey: reuse the facts already read, whatever version of the doc they came from. */
139
+ reuseOnly = false) {
121
140
  const path = groundingPath(journey);
122
141
  const map = existsSync(MAP_PATH) ? readFileSync(MAP_PATH, 'utf-8') : '';
123
142
  let sourceHash = '';
@@ -131,7 +150,7 @@ export async function groundJourney(journey, docHash, srcDir = 'src', markdown =
131
150
  if (existsSync(path)) {
132
151
  try {
133
152
  const cached = JSON.parse(readFileSync(path, 'utf-8'));
134
- if (cached.key === key && typeof cached.facts === 'string' && cached.facts.trim()) {
153
+ if (typeof cached.facts === 'string' && cached.facts.trim() && (cached.key === key || (reuseOnly && String(cached.key).startsWith(`v${FACTS_VERSION}:`)))) {
135
154
  return cached.facts;
136
155
  }
137
156
  }
@@ -139,12 +158,20 @@ export async function groundJourney(journey, docHash, srcDir = 'src', markdown =
139
158
  // corrupt cache — re-ground
140
159
  }
141
160
  }
161
+ if (reuseOnly)
162
+ return null;
142
163
  const config = loadConfig();
143
- const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
164
+ // The source the doc points at, collected here, goes into one tool-less call.
165
+ // A doc that points at nothing findable gets no facts: its tests are written
166
+ // from the doc, which is the contract, rather than paying an AI to go looking.
167
+ const excerpts = sourceExcerpts(journey.rawMarkdown || '', [srcDir, ...(config.relatedDirs ?? [])]);
168
+ if (!excerpts)
169
+ return null;
170
+ const { program, args } = withNoTools(splitCommand(config.aiCommand));
144
171
  let facts;
145
172
  try {
146
- facts = execFileSync(program, args, {
147
- input: buildGroundingPrompt(journey, map, srcDir),
173
+ facts = runAiSync(program, args, {
174
+ input: buildGroundingPrompt(journey, map, srcDir, excerpts),
148
175
  encoding: 'utf-8',
149
176
  maxBuffer: 10 * 1024 * 1024,
150
177
  timeout: config.aiTimeoutMs ?? 300_000,
@@ -0,0 +1,78 @@
1
+ /**
2
+ * testforge — a reference read once, and only its relevant part handed on.
3
+ *
4
+ * Drafting one journey used to fetch the same Notion page three times (to
5
+ * compare, to draft, to review), each inside a long tool session. The page is
6
+ * now fetched by one short call, kept on disk, and cut down to the journey
7
+ * being drafted, its shared rules and the page's introduction. A file in the
8
+ * project is simply read. `--refresh` fetches again.
9
+ */
10
+ import { createHash } from 'node:crypto';
11
+ import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ import { REFERENCE_DIR } from '../paths.js';
14
+ import { loadConfig, splitCommand, withReferenceTools } from '../project/config.js';
15
+ import { runAiSync } from '../generate/ai-usage.js';
16
+ const cachePath = (reference) => join(REFERENCE_DIR, `${createHash('sha1').update(reference).digest('hex').slice(0, 12)}.md`);
17
+ /** The reference's text: a local file as it is, a page or URL fetched once and kept. */
18
+ export function readReference(reference, refresh = false) {
19
+ if (existsSync(reference) && statSync(reference).isFile() && !/\.pdf$/i.test(reference))
20
+ return readFileSync(reference, 'utf-8');
21
+ const path = cachePath(reference);
22
+ if (!refresh && existsSync(path))
23
+ return readFileSync(path, 'utf-8');
24
+ const { program, args } = withReferenceTools(splitCommand(loadConfig().aiCommand));
25
+ const text = runAiSync(program, args, {
26
+ input: [
27
+ `Fetch this reference and print its FULL content as markdown, word for word:`,
28
+ ` ${reference}`,
29
+ `It may be a Notion page, a URL or a document. Keep every heading, list,`,
30
+ `table and sentence; summarise nothing, add nothing. Include its sub-pages`,
31
+ `only if the page's own text is just a list of links to them. Print only`,
32
+ `the content. If you cannot read it, print one line starting "ERROR:".`,
33
+ ].join('\n'),
34
+ encoding: 'utf-8',
35
+ maxBuffer: 10 * 1024 * 1024,
36
+ }).trim();
37
+ if (!text || /^ERROR:/m.test(text.slice(0, 300)))
38
+ throw new Error(`Could not read the reference "${reference}":\n${text.slice(0, 500)}`);
39
+ mkdirSync(REFERENCE_DIR, { recursive: true });
40
+ writeFileSync(path, text, 'utf-8');
41
+ return text;
42
+ }
43
+ const norm = (t) => t.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
44
+ /**
45
+ * The part of a reference one journey needs: everything before the first
46
+ * section that names a journey (the page's introduction and shared rules),
47
+ * the sections that are not journeys, and the section whose heading names
48
+ * this one. A reference with no such heading is returned whole.
49
+ */
50
+ export function sectionFor(text, journey) {
51
+ const lines = text.split('\n');
52
+ const headings = lines.flatMap((l, i) => {
53
+ const m = /^(#{1,4})\s+(.*)$/.exec(l);
54
+ return m ? [{ i, level: m[1].length, title: m[2] }] : [];
55
+ });
56
+ const want = norm(journey);
57
+ const hit = headings.find((h) => norm(h.title).includes(want));
58
+ if (!hit)
59
+ return text;
60
+ const isJourney = (t) => /\bjourney\b|^\d+[.)]\s/i.test(t);
61
+ const keep = new Set();
62
+ const span = (h) => {
63
+ const next = headings.find((x) => x.i > h.i && x.level <= h.level);
64
+ return [h.i, next ? next.i : lines.length];
65
+ };
66
+ const firstJourney = headings.find((h) => isJourney(h.title) && h.level === hit.level);
67
+ for (let i = 0; i < (firstJourney ? firstJourney.i : 0); i++)
68
+ keep.add(i);
69
+ for (const h of headings.filter((x) => x.level === hit.level && !isJourney(x.title))) {
70
+ const [a, b] = span(h);
71
+ for (let i = a; i < b; i++)
72
+ keep.add(i);
73
+ }
74
+ const [a, b] = span(hit);
75
+ for (let i = a; i < b; i++)
76
+ keep.add(i);
77
+ return lines.filter((_, i) => keep.has(i)).join('\n').replace(/\n{3,}/g, '\n\n').trim();
78
+ }
@@ -21,16 +21,16 @@
21
21
  * URLs/Notion, web) access. As with discover/map this is tool-specific
22
22
  * and tuned for the Claude CLI.
23
23
  */
24
- import { execFileSync } from 'node:child_process';
25
- import { loadConfig, splitCommand, withReferenceTools } from '../project/config.js';
24
+ import { loadConfig, splitCommand, withNoTools, withReferenceTools } from '../project/config.js';
26
25
  import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
26
+ import { runAiSync } from '../generate/ai-usage.js';
27
27
  /** Where the comparison report is written. */
28
- export const DIFF_PATH = 'testforge.diff.md';
28
+ export const DIFF_PATH = 'testforge/reference/diff.md';
29
29
  const MAP_PATH = 'testforge.md';
30
30
  /** Run the configured AI command with read/write/web (+ Notion) access. */
31
31
  function runAI(prompt) {
32
32
  const { program, args } = withReferenceTools(splitCommand(loadConfig().aiCommand));
33
- const output = execFileSync(program, args, {
33
+ const output = runAiSync(program, args, {
34
34
  input: prompt,
35
35
  encoding: 'utf-8',
36
36
  maxBuffer: 10 * 1024 * 1024,
@@ -67,7 +67,7 @@ export function buildAnalyzePrompt(reference, docsDir) {
67
67
  ` that the code-derived view omits or gets wrong`,
68
68
  ` - anything the code-derived view asserts that the reference contradicts`,
69
69
  ``,
70
- `Write a comparison report to "${DIFF_PATH}" at the project root with`,
70
+ `Write a comparison report to "${DIFF_PATH}" with`,
71
71
  `these sections:`,
72
72
  ` # Source-of-truth comparison`,
73
73
  ` ## Reference (what you read, and where from)`,
@@ -109,6 +109,12 @@ export function buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey
109
109
  DEEP_DOC_GUIDANCE,
110
110
  ``,
111
111
  `Reference: ${reference}`,
112
+ ...(reference.endsWith('tickets-reference.md')
113
+ ? [
114
+ `The reference is a bundle of exported tickets, one section per ticket id.`,
115
+ `Every [spec] source names its ticket: (source: <TICKET-ID>: "<quoted sentence>").`,
116
+ ]
117
+ : []),
112
118
  ...(loadConfig().decisions
113
119
  ? [
114
120
  `NOT YET REQUIREMENTS: the project keeps its open product decisions and`,
@@ -211,3 +217,60 @@ export function analyzeReference(reference, docsDir) {
211
217
  export function draftDocsFromReference(srcDir, docsDir, reference, journey, existingDocs) {
212
218
  return runAI(buildReferenceDiscoverPrompt(srcDir, docsDir, reference, journey, existingDocs));
213
219
  }
220
+ /**
221
+ * Draft ONE journey's doc in a single call with no tools: the reference's
222
+ * part for this journey and the source lines it points at are in the prompt.
223
+ * The answer is the doc itself, after one AREA line.
224
+ */
225
+ export function buildJourneyDraftPrompt(journey, section, excerpts, areas, otherDocs) {
226
+ return [
227
+ `You are a senior QA engineer writing ONE journey doc, "${journey}", so a QA`,
228
+ `tool can generate tests from it. The REFERENCE below is the source of truth`,
229
+ `for how the product is EXPECTED to behave; where the code does something`,
230
+ `else, document the reference (the code may be buggy).`,
231
+ ``,
232
+ DEEP_DOC_GUIDANCE,
233
+ ``,
234
+ `You have no tools. Everything you may use is below: the reference's part`,
235
+ `for this journey (with the page's shared rules), and SOURCE EXCERPTS, the`,
236
+ `lines of this project's code the journey's texts and files point at. Use the`,
237
+ `excerpts only for TECHNICAL detail (routes, field names, labels the screen`,
238
+ `really shows) and for [observed] rules; [spec] comes only from the reference.`,
239
+ `Where the excerpts do not show something, do not invent it.`,
240
+ ``,
241
+ ...(otherDocs.length
242
+ ? [`Other journeys already have docs; a rule they own is not repeated here, only named:`, ...otherDocs.map((d) => ` - ${d}`), ``]
243
+ : []),
244
+ `The doc starts with this header:`,
245
+ `# Journey: <Name>`,
246
+ ``,
247
+ `Type: web ("web" for a page/UI, "api" for an endpoint)`,
248
+ `Route: <path>`,
249
+ ``,
250
+ `Coverage:`,
251
+ `- functional`,
252
+ `- security`,
253
+ ``,
254
+ `Rules: tag every criterion as described above; [spec] only where the`,
255
+ `reference states it, with its quote. When the code seems to behave`,
256
+ `differently from a [spec] criterion, keep the criterion and add a line`,
257
+ `"Likely defect: <what the code does>" under it. No "# TODO", no secrets.`,
258
+ ``,
259
+ `OUTPUT: first one line "AREA: <one of ${areas.join(', ')}>", then the doc`,
260
+ `itself, starting with "# Journey:". Nothing before or after.`,
261
+ ``,
262
+ `--- REFERENCE ---`,
263
+ section,
264
+ ``,
265
+ `--- SOURCE EXCERPTS ---`,
266
+ excerpts || '(no source the reference points at was found: the feature may not be built yet)',
267
+ ].join('\n');
268
+ }
269
+ /** One no-tools call that answers with a doc; returns its area and markdown. */
270
+ export function draftJourneyDoc(prompt) {
271
+ const { program, args } = withNoTools(splitCommand(loadConfig().aiCommand));
272
+ const out = runAiSync(program, args, { input: prompt, encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 }).trim();
273
+ const area = /^AREA:\s*([\w-]+)/m.exec(out)?.[1];
274
+ const at = out.indexOf('# Journey:');
275
+ return { area, markdown: at >= 0 ? `${out.slice(at).trim()}\n` : '' };
276
+ }
@@ -10,9 +10,11 @@
10
10
  * tiers that claim more than their source says. The reviewer edits the file
11
11
  * in place and never deletes a sound criterion.
12
12
  */
13
+ import { readFileSync, writeFileSync } from 'node:fs';
13
14
  import { join } from 'node:path';
14
15
  import { loadConfig, splitCommand, withEditTools, withReferenceTools } from '../project/config.js';
15
16
  import { runWithStdin } from '../generate/ai-generator.js';
17
+ import { HAS_SOURCE } from '../generate/defect-review.js';
16
18
  import { pool } from '../generate/run-generate.js';
17
19
  import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
18
20
  /** Build the prompt that reviews and completes ONE journey doc. */
@@ -59,6 +61,84 @@ export function buildReviewPrompt(docPath, srcDir, reference) {
59
61
  `with counts.`,
60
62
  ].join('\n');
61
63
  }
64
+ /**
65
+ * The same review in one call with no tools: the reference's part, the source
66
+ * excerpts and the draft are in the prompt, and the answer is the whole
67
+ * revised doc.
68
+ */
69
+ export function buildLeanReviewPrompt(doc, section, excerpts, otherDocs) {
70
+ return [
71
+ `You are a senior QA engineer reviewing ONE journey doc that a first pass`,
72
+ `drafted. Make it COMPLETE, so no scenario a careful QA engineer, or a user`,
73
+ `trying to break it, would test is missing. You have no tools: the`,
74
+ `reference's part for this journey, the source excerpts it points at and the`,
75
+ `draft are below. [spec] comes only from the reference; the excerpts give`,
76
+ `technical detail and [observed] rules.`,
77
+ ``,
78
+ `The doc must follow this guidance:`,
79
+ DEEP_DOC_GUIDANCE,
80
+ ``,
81
+ `Check it against the METHOD above and fix what you find: every control in`,
82
+ `the table; every [baseline] item that fits each control, with its level`,
83
+ `and category, and every MUST that fits; every combination the reference`,
84
+ `states; every branch the excerpts show, as [observed] or a Question; every`,
85
+ `[spec] quoting a real reference sentence (re-tag any that does not);`,
86
+ `the Coverage table with no blank cell. Keep every sound criterion, the`,
87
+ `header and the section numbering. Never write "# TODO".`,
88
+ ...(otherDocs.length ? [`Other journeys' docs (shared rules stay with them):`, ...otherDocs.map((d) => ` - ${d}`)] : []),
89
+ ``,
90
+ `OUTPUT: the complete revised doc, starting with "# Journey:". Nothing else.`,
91
+ ``,
92
+ `--- REFERENCE ---`,
93
+ section,
94
+ ``,
95
+ `--- SOURCE EXCERPTS ---`,
96
+ excerpts || '(none found)',
97
+ ``,
98
+ `--- THE DRAFT ---`,
99
+ doc,
100
+ ].join('\n');
101
+ }
102
+ /**
103
+ * A [spec] line is a requirement the client wrote, so it must carry the
104
+ * client's words: `(source: "…")`, or `(source as above)` pointing at the quote before it. A [spec] line without that quote is the
105
+ * drafter's reading, and it is re-tagged [observed] — never given an invented
106
+ * quote. Fenced code blocks are left alone.
107
+ */
108
+ export function enforceSpecSources(markdown) {
109
+ let inFence = false;
110
+ let retagged = 0;
111
+ const lines = markdown.split('\n').map((line) => {
112
+ if (/^\s*```/.test(line))
113
+ inFence = !inFence;
114
+ if (inFence || !/\[spec\]/i.test(line) || HAS_SOURCE.test(line))
115
+ return line;
116
+ retagged += 1;
117
+ return line.replace(/\[spec\]/gi, '[observed]');
118
+ });
119
+ return { markdown: lines.join('\n'), retagged };
120
+ }
121
+ /** Apply the source-quote rule to docs on disk; returns how many lines were re-tagged. */
122
+ export function enforceSpecSourcesOnDisk(docsDir, docs) {
123
+ let total = 0;
124
+ for (const doc of docs) {
125
+ const path = join(docsDir, doc);
126
+ let text;
127
+ try {
128
+ text = readFileSync(path, 'utf-8');
129
+ }
130
+ catch {
131
+ continue;
132
+ }
133
+ const { markdown, retagged } = enforceSpecSources(text);
134
+ if (!retagged)
135
+ continue;
136
+ writeFileSync(path, markdown, 'utf-8');
137
+ console.log(` ${doc}: ${retagged} [spec] line(s) without a quoted source re-tagged [observed]`);
138
+ total += retagged;
139
+ }
140
+ return total;
141
+ }
62
142
  /**
63
143
  * Review each doc (paths relative to docsDir) in its own AI pass, a few at
64
144
  * a time. A failed review leaves that doc as drafted and is reported, never
@@ -81,4 +161,5 @@ export async function reviewDocs(docsDir, docs, srcDir, reference) {
81
161
  console.log(` ${doc}: review failed, kept as drafted (${err.message.split('\n')[0]})`);
82
162
  }
83
163
  });
164
+ enforceSpecSourcesOnDisk(docsDir, docs);
84
165
  }