@activate-core/test-forge 2.4.6 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +47 -11
  2. package/dist/cli/index.js +12 -12
  3. package/dist/core/docs/cache.js +1 -1
  4. package/dist/core/docs/discover.js +5 -5
  5. package/dist/core/docs/excerpts.js +139 -5
  6. package/dist/core/docs/grounding.js +78 -16
  7. package/dist/core/docs/reference-text.js +310 -52
  8. package/dist/core/gates/coverage.js +9 -0
  9. package/dist/core/gates/validate.js +1 -0
  10. package/dist/core/generate/ai-generator.js +9 -3
  11. package/dist/core/generate/ai-usage.js +11 -4
  12. package/dist/core/generate/bind.js +3 -2
  13. package/dist/core/generate/complete-file.js +45 -7
  14. package/dist/core/generate/defect-review.js +2 -1
  15. package/dist/core/generate/harness.js +34 -0
  16. package/dist/core/generate/interface-file.js +12 -0
  17. package/dist/core/generate/journey-memory.js +7 -1
  18. package/dist/core/generate/passed.js +24 -0
  19. package/dist/core/generate/review-journey.js +2 -2
  20. package/dist/core/generate/run-generate.js +117 -13
  21. package/dist/core/generate/suite-planner.js +2 -2
  22. package/dist/core/generate/test-blocks.js +34 -10
  23. package/dist/core/paths.js +41 -6
  24. package/dist/core/project/config.js +5 -0
  25. package/dist/core/project/setup.js +9 -9
  26. package/dist/core/project/update.js +6 -6
  27. package/dist/core/prompts/blocks.js +24 -3
  28. package/dist/core/run/name-scan.js +120 -14
  29. package/dist/core/run/report.js +7 -4
  30. package/dist/core/run/run-tests.js +53 -4
  31. package/package.json +3 -2
  32. package/reporters/_shared.cjs +56 -3
  33. package/reporters/interface.cjs +14 -0
  34. package/reporters/jest-reporter.cjs +7 -3
  35. package/reporters/playwright-reporter.cjs +6 -2
  36. package/reporters/run-data.cjs +6 -1
@@ -1,63 +1,321 @@
1
1
  /**
2
- * testforge — a reference read once, and only its relevant part handed on.
2
+ * testforge — a reference read fresh, once per command, word for word, and
3
+ * only its relevant part handed on. No AI ever re-types a reference:
3
4
  *
4
- * Drafting one journey used to fetch the same Notion page three times (to
5
- * compare, to draft, to review), each inside a long tool session. The page is
6
- * now fetched by one short call, kept on disk, and cut down to the journey
7
- * being drafted, its shared rules and the page's introduction. A file in the
8
- * project is simply read. `--refresh` fetches again.
5
+ * - a file in the project is read as it is (a PDF through pdftotext);
6
+ * - a Notion page is fetched by a short AI call that only runs the Notion
7
+ * fetch tool (one more per level of child pages); the text is taken from
8
+ * the tools' results in the event stream (or the file the CLI saves a large
9
+ * one to);
10
+ * - any other link is downloaded directly.
11
+ *
12
+ * Anything that would leave part of a reference out (a page Notion cut short,
13
+ * a child page not fetched, a database, a login page) is an error, never a
14
+ * quietly shorter reference. Nothing is kept between commands, so an edited
15
+ * page is always the one used; the last text read is written under
16
+ * testforge/runs/reference/ to look at, never read back.
9
17
  */
10
18
  import { createHash } from 'node:crypto';
11
- import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
19
+ import { spawnSync } from 'node:child_process';
20
+ import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
21
+ import { tmpdir } from 'node:os';
12
22
  import { join } from 'node:path';
13
- import { REFERENCE_DIR } from '../paths.js';
14
- import { loadConfig, splitCommand, withReferenceTools } from '../project/config.js';
15
- import { runAiSync } from '../generate/ai-usage.js';
16
- const cachePath = (reference) => join(REFERENCE_DIR, `${createHash('sha1').update(reference).digest('hex').slice(0, 12)}.md`);
17
- /** The reference's text: a local file as it is, a page or URL fetched once and kept. */
18
- export function readReference(reference, refresh = false) {
19
- if (existsSync(reference) && statSync(reference).isFile() && !/\.pdf$/i.test(reference))
20
- return readFileSync(reference, 'utf-8');
21
- const path = cachePath(reference);
22
- if (!refresh && existsSync(path))
23
- return readFileSync(path, 'utf-8');
24
- const { program, args } = withReferenceTools(splitCommand(loadConfig().aiCommand));
25
- const ask = (after) => runAiSync(program, args, {
26
- input: [
27
- after
28
- ? `Continue printing this reference as Markdown, word for word, from the section that comes after the heading "${after}":`
29
- : `Fetch this reference and print its full content as Markdown, word for word:`,
30
- ` ${reference}`,
31
- `It may be a Notion page, a URL or a document. Keep every heading, list, table`,
32
- `and sentence; summarise nothing and add nothing. Include sub-pages only when`,
33
- `the page's own text is just a list of links to them. Print only the content.`,
34
- ``,
35
- `If you cannot read it, print one line starting "ERROR:".`,
36
- `If it is too long to print in one answer, stop at the end of a heading's`,
37
- `section and print a last line "CONTINUE AFTER: <that heading>". You will be`,
38
- `asked for the rest.`,
39
- ].join('\n'),
40
- encoding: 'utf-8',
41
- maxBuffer: 10 * 1024 * 1024,
42
- }).trim();
43
- // A long page comes in parts; each part says where the next one starts.
44
- const parts = [];
45
- let after;
46
- for (let k = 0; k < 12; k++) {
47
- const part = ask(after);
48
- const cont = /\n?CONTINUE AFTER:\s*(.+)\s*$/.exec(part);
49
- parts.push(cont ? part.slice(0, cont.index).trimEnd() : part);
50
- if (!cont || cont[1].trim() === after)
51
- break;
52
- after = cont[1].trim();
23
+ import { RUNS_DIR } from '../paths.js';
24
+ import { isClaude, loadConfig, splitCommand } from '../project/config.js';
25
+ import { runAiWithStream } from '../generate/ai-usage.js';
26
+ import { looksLikeAppShell } from './cache.js';
27
+ const read = new Map();
28
+ /** The reference's text, word for word: a file, a folder of files, a Notion page with its child pages, or a link. */
29
+ export async function readReference(reference) {
30
+ const known = read.get(reference);
31
+ if (known !== undefined)
32
+ return known;
33
+ const text = existsSync(reference) ? readLocal(reference)
34
+ : notionId(reference) && !/^https?:\/\//i.test(reference) || isNotionUrl(reference) ? readNotion(reference)
35
+ : /^https?:\/\//i.test(reference) ? await readUrl(reference)
36
+ : fail(reference, 'it is neither a file in this project nor a link');
37
+ if (!text.trim())
38
+ fail(reference, 'it is empty');
39
+ read.set(reference, text);
40
+ if (existsSync(reference))
41
+ return text; // a file is its own copy
42
+ try {
43
+ const dir = join(RUNS_DIR, 'reference');
44
+ mkdirSync(dir, { recursive: true });
45
+ writeFileSync(join(dir, `${createHash('sha1').update(reference).digest('hex').slice(0, 12)}.md`), text, 'utf-8');
53
46
  }
54
- const text = parts.join('\n\n').trim();
55
- if (!text || /^ERROR:/m.test(text.slice(0, 300)))
56
- throw new Error(`Could not read the reference "${reference}":\n${text.slice(0, 500)}`);
57
- mkdirSync(REFERENCE_DIR, { recursive: true });
58
- writeFileSync(path, text, 'utf-8');
47
+ catch { /* only a copy to look at */ }
59
48
  return text;
60
49
  }
50
+ function fail(reference, why) {
51
+ throw new Error(`Could not read the reference "${reference}": ${why}.`);
52
+ }
53
+ // ---- files ----
54
+ function readLocal(path) {
55
+ if (statSync(path).isDirectory()) {
56
+ return readdirSync(path).sort()
57
+ .filter((f) => /\.(md|markdown|txt|pdf)$/i.test(f) && statSync(join(path, f)).isFile())
58
+ .map((f) => `# ${f}\n\n${readLocal(join(path, f))}`)
59
+ .join('\n\n');
60
+ }
61
+ if (/\.pdf$/i.test(path))
62
+ return pdfText(path, path);
63
+ const buf = readFileSync(path);
64
+ if (buf.includes(0))
65
+ fail(path, 'it is not a text file — save it as .md, .txt or .pdf');
66
+ return buf.toString('utf-8');
67
+ }
68
+ function pdfText(file, reference) {
69
+ const r = spawnSync('pdftotext', [file, '-'], { encoding: 'utf-8', maxBuffer: 64 * 1024 * 1024 });
70
+ if (r.error)
71
+ fail(reference, 'reading a PDF needs pdftotext (poppler-utils: `apt install poppler-utils`, `brew install poppler`)');
72
+ if (r.status !== 0)
73
+ fail(reference, `pdftotext failed: ${String(r.stderr).trim().slice(0, 200)}`);
74
+ if (!r.stdout.trim())
75
+ fail(reference, 'the PDF has no text layer (a scan) — export the text instead');
76
+ return r.stdout;
77
+ }
78
+ // ---- links ----
79
+ async function readUrl(url) {
80
+ let res;
81
+ try {
82
+ res = await fetch(url, { redirect: 'follow' });
83
+ }
84
+ catch (e) {
85
+ return fail(url, e.message);
86
+ }
87
+ if (!res.ok)
88
+ fail(url, `the link returned ${res.status} — if it needs a sign-in, export it as .md, .txt or .pdf and pass the file`);
89
+ const type = res.headers.get('content-type') ?? '';
90
+ if (/pdf/i.test(type) || /\.pdf(\?|$)/i.test(url)) {
91
+ const file = join(mkdtempSync(join(tmpdir(), 'testforge-')), 'reference.pdf');
92
+ writeFileSync(file, Buffer.from(await res.arrayBuffer()));
93
+ return pdfText(file, url);
94
+ }
95
+ const body = await res.text();
96
+ if (!/html/i.test(type))
97
+ return body;
98
+ const shell = looksLikeAppShell(url, body);
99
+ if (shell)
100
+ fail(url, `${shell} — export it as .md, .txt or .pdf and pass the file`);
101
+ return htmlText(body);
102
+ }
103
+ /** A web page's readable text, its headings and list items kept as Markdown. */
104
+ export function htmlText(html) {
105
+ return entities(html
106
+ .replace(/<(script|style|noscript|svg|head)\b[\s\S]*?<\/\1>/gi, '')
107
+ .replace(/<h([1-6])\b[^>]*>/gi, (_, n) => `\n\n${'#'.repeat(Number(n))} `)
108
+ .replace(/<li\b[^>]*>/gi, '\n- ')
109
+ .replace(/<\/t[dh]>/gi, ' | ')
110
+ .replace(/<(?:br|\/p|\/div|\/h[1-6]|\/tr|\/ul|\/ol|\/section|\/article)\b[^>]*>/gi, '\n')
111
+ .replace(/<[^>]+>/g, ''))
112
+ .replace(/[ \t]+/g, ' ')
113
+ .replace(/\n\s*\n\s*\n+/g, '\n\n')
114
+ .trim();
115
+ }
116
+ function entities(s) {
117
+ return s.replace(/&nbsp;/g, ' ').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;|&apos;/g, "'")
118
+ .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))).replace(/&amp;/g, '&');
119
+ }
120
+ // ---- Notion ----
121
+ const isNotionUrl = (s) => /^https?:\/\/([\w-]+\.)*notion\.(so|site|com)\//i.test(s);
122
+ /** A Notion page's 32-hex id, from a URL or a bare id. */
123
+ export function notionId(s) {
124
+ const path = s.split(/[?#]/)[0].replace(/-/g, '');
125
+ return /([0-9a-f]{32})(?![0-9a-f])/i.exec(path)?.[1]?.toLowerCase();
126
+ }
127
+ /**
128
+ * The fetch only runs tools, and the pages are taken from the tools' results,
129
+ * not the AI's words: Claude's smallest model does it for about a fifteenth
130
+ * of the cost of the configured one.
131
+ */
132
+ export function fetchModel(cmd) {
133
+ if (!isClaude(cmd.program))
134
+ return cmd;
135
+ const args = cmd.args.filter((a, k) => a !== '--model' && cmd.args[k - 1] !== '--model' && !a.startsWith('--model='));
136
+ return { program: cmd.program, args: [...args, '--model', 'haiku'] };
137
+ }
138
+ /**
139
+ * One haiku call per level of child pages (one call for a page without any):
140
+ * the AI is given the exact pages to fetch, and the code — not the AI — finds
141
+ * the child pages in what came back.
142
+ */
143
+ function readNotion(reference) {
144
+ const cmd = splitCommand(loadConfig().aiCommand);
145
+ if (!isClaude(cmd.program))
146
+ fail(reference, 'a Notion page is read through the Claude CLI and its Notion connection; set aiCommand to claude, or export the page as .md and pass the file');
147
+ const { program, args } = fetchModel(cmd);
148
+ const tools = ['--allowedTools', 'mcp__claude_ai_Notion__notion-fetch mcp__notion__notion-fetch',
149
+ '--disallowedTools', 'Bash Read Grep Glob Write Edit NotebookEdit WebFetch WebSearch Task Agent'];
150
+ const pages = new Map();
151
+ const root = notionId(reference) ?? reference;
152
+ let want = [reference];
153
+ let aiSaid = '';
154
+ for (let round = 0; want.length && round < 12; round++) {
155
+ const { text, raw } = runAiWithStream(program, [...args, ...tools], {
156
+ input: [
157
+ `Fetch each of these Notion pages with the Notion fetch tool (load it with`,
158
+ `ToolSearch "select:mcp__claude_ai_Notion__notion-fetch" if it is not loaded),`,
159
+ `each once, and nothing else:`,
160
+ ...want.map((u) => ` ${u}`),
161
+ `Do not print any page: they are read from the tool results. A result the`,
162
+ `tool saved to a file instead of returning counts as fetched. Reply with the`,
163
+ `single word DONE, or one line starting "ERROR:" if a page cannot be fetched.`,
164
+ ].join('\n'),
165
+ encoding: 'utf-8',
166
+ maxBuffer: 10 * 1024 * 1024,
167
+ });
168
+ // The AI's word is not the verdict: what the tool results hold is.
169
+ if (/^ERROR:/m.test(text.slice(0, 300)))
170
+ aiSaid = text.trim().slice(0, 300);
171
+ const before = new Set(pages.keys());
172
+ for (const [id, p] of notionPages(raw, reference))
173
+ if (!pages.has(id))
174
+ pages.set(id, p);
175
+ const got = [...pages.keys()].filter((id) => !before.has(id));
176
+ if (!got.length)
177
+ break; // nothing new: assembly names what is missing
178
+ // Next: every child page the reference reaches that is not read yet.
179
+ want = [...new Set(reachable(pages, root).filter((u) => { const id = notionId(u); return id && !pages.has(id); }))];
180
+ }
181
+ try {
182
+ return assemble(pages, reference);
183
+ }
184
+ catch (e) {
185
+ throw aiSaid ? new Error(`${e.message}\n The AI said: ${aiSaid}`) : e;
186
+ }
187
+ }
188
+ const CHILD = /<page\s+url="([^"]+)"[^>]*>([\s\S]*?)<\/page>/g;
189
+ /** The child-page links of every page the reference reaches through pages already read. */
190
+ function reachable(pages, root) {
191
+ const out = [];
192
+ const seen = new Set();
193
+ const walk = (id) => {
194
+ if (seen.has(id))
195
+ return;
196
+ seen.add(id);
197
+ const page = pages.get(id);
198
+ if (!page)
199
+ return;
200
+ for (const m of page.body.matchAll(CHILD)) {
201
+ out.push(m[1]);
202
+ const c = notionId(m[1]);
203
+ if (c)
204
+ walk(c);
205
+ }
206
+ };
207
+ walk(root);
208
+ return out;
209
+ }
210
+ /** The whole reference from a CLI event stream (one round). Throws when any part would be missing. */
211
+ export function notionText(raw, reference) {
212
+ return assemble(notionPages(raw, reference), reference);
213
+ }
214
+ /** The page with each child page in the place its link stood. Throws when any part would be missing. */
215
+ function assemble(pages, reference) {
216
+ const root = notionId(reference);
217
+ const top = root ? pages.get(root) : undefined;
218
+ if (!top)
219
+ fail(reference, 'the Notion page was not fetched — check that Claude\'s Notion connection can open it');
220
+ const missing = [];
221
+ const cut = [];
222
+ const expand = (page, name, seen) => {
223
+ if (page.cut)
224
+ cut.push(name);
225
+ return page.body.replace(CHILD, (_, url, title) => {
226
+ const id = notionId(url);
227
+ const child = id ? pages.get(id) : undefined;
228
+ if (!id || seen.has(id))
229
+ return title;
230
+ if (!child) {
231
+ missing.push(title.trim() || url);
232
+ return title;
233
+ }
234
+ const childName = child.title || title.trim();
235
+ return `\n# ${childName}\n\n${expand(child, childName, new Set([...seen, id])).trim()}\n`;
236
+ });
237
+ };
238
+ const body = expand(top, top.title || reference, new Set([root]));
239
+ if (cut.length)
240
+ fail(reference, `Notion returned ${cut.map((c) => `"${c}"`).join(', ')} with part of it left out (a page too large for one fetch). Split it into child pages — each child page is read whole`);
241
+ if (missing.length)
242
+ fail(reference, `its child page(s) could not be fetched: ${missing.join(', ')}. Check that Claude's Notion connection can open them, then run the command again`);
243
+ const unread = [...new Set([...body.matchAll(/<(database|folder|data-source)\b[^>]*>([^<]*)/g)].map((m) => `${m[1]} "${m[2].trim()}"`))];
244
+ if (unread.length)
245
+ fail(reference, `it holds ${unread.join(', ')}, whose rows are not part of the page text. Copy what the tests need into the page, or export it as .md and pass the file`);
246
+ const files = (body.match(/<(?:pdf|file|audio|video|image)\b|!\[[^\]]*\]\(/g) ?? []).length;
247
+ if (files)
248
+ console.warn(` ${reference}: ${files} image(s)/attachment(s) on the page are not text and are not read.`);
249
+ const out = plainNotion(body).replace(/\n{3,}/g, '\n\n').trim();
250
+ return top.title ? `# ${top.title}\n\n${out}` : out;
251
+ }
252
+ /** Every Notion page fetched in a CLI event stream, by id. */
253
+ function notionPages(raw, reference) {
254
+ const tools = new Map();
255
+ const pages = new Map();
256
+ for (const line of raw.split('\n')) {
257
+ if (!line.startsWith('{'))
258
+ continue;
259
+ let e;
260
+ try {
261
+ e = JSON.parse(line);
262
+ }
263
+ catch {
264
+ continue;
265
+ }
266
+ for (const c of e.message?.content ?? []) {
267
+ if (c.type === 'tool_use' && c.id && c.name)
268
+ tools.set(c.id, { name: c.name, id: typeof c.input?.id === 'string' ? c.input.id : undefined });
269
+ if (c.type !== 'tool_result')
270
+ continue;
271
+ const tool = tools.get(c.tool_use_id ?? '');
272
+ if (!tool || !/notion.*fetch/i.test(tool.name))
273
+ continue;
274
+ let body = typeof c.content === 'string' ? c.content
275
+ : Array.isArray(c.content) ? c.content.map((x) => x.text ?? '').join('') : '';
276
+ // A result over the CLI's limit is saved whole to a file; the result names it.
277
+ const saved = /saved to (\/\S+?\.txt)\b/.exec(body)?.[1];
278
+ if (saved) {
279
+ if (!existsSync(saved))
280
+ fail(reference, `the CLI saved a Notion page to ${saved}, which is gone`);
281
+ body = readFileSync(saved, 'utf-8');
282
+ }
283
+ let j = {};
284
+ try {
285
+ j = JSON.parse(body);
286
+ }
287
+ catch {
288
+ continue;
289
+ } // an error message, not a page
290
+ const text = String(j.text ?? '');
291
+ const open = text.indexOf('<content>');
292
+ if (open < 0)
293
+ continue;
294
+ const close = text.lastIndexOf('</content>');
295
+ const id = notionId(String(j.url ?? '')) ?? notionId(tool.id ?? '');
296
+ if (!id)
297
+ continue;
298
+ pages.set(id, {
299
+ title: String(j.title ?? '').trim(),
300
+ body: text.slice(open + '<content>'.length, close < 0 ? undefined : close),
301
+ cut: close < 0 || j.truncated === true || Number(j.unknown_block_count) > 0,
302
+ });
303
+ }
304
+ }
305
+ return pages;
306
+ }
307
+ /** Notion's tags as Markdown, so quotes match the page as people read it. */
308
+ export function plainNotion(s) {
309
+ return entities(s
310
+ .replace(/<\/?code>/g, '`')
311
+ .replace(/<\/?(?:b|strong)>/g, '**')
312
+ .replace(/<\/?(?:i|em)>/g, '*')
313
+ .replace(/<br\s*\/?>/g, '\n')
314
+ .replace(/<\/t[dh]>\s*/g, ' | ')
315
+ .replace(/<\/tr>/g, '\n')
316
+ .replace(/<date\s+start="([^"]+)"(?:\s+end="([^"]+)")?[^>]*\/?>/g, (_, a, b) => (b ? `${a} to ${b}` : a))
317
+ .replace(/<\/?[a-z][\w-]*(?:\s[^>]*)?\/?>/g, ''));
318
+ }
61
319
  const norm = (t) => t.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
62
320
  /**
63
321
  * The part of a reference one journey needs: everything before the first
@@ -44,6 +44,15 @@ export function writeCoverageManifest(results) {
44
44
  });
45
45
  if (!journeys.length)
46
46
  return;
47
+ // Journeys this run did not touch keep their earlier entry: the manifest is
48
+ // the project's coverage, not this run's.
49
+ try {
50
+ const prev = JSON.parse(readFileSync(COVERAGE_PATH, 'utf-8'));
51
+ for (const j of prev.journeys ?? [])
52
+ if (!journeys.some((n) => n.journey === j.journey))
53
+ journeys.push(j);
54
+ }
55
+ catch { /* no earlier manifest */ }
47
56
  const totalReal = journeys.reduce((n, j) => n + j.real, 0);
48
57
  const totalDeferred = journeys.reduce((n, j) => n + j.deferred, 0);
49
58
  try {
@@ -490,6 +490,7 @@ function looksStructurallyClean(content) {
490
490
  t.startsWith('function ') ||
491
491
  t.startsWith('async ') ||
492
492
  t.startsWith('require(') ||
493
+ /^(?:describe|test|it|beforeAll|beforeEach)\s*\(/.test(t) ||
493
494
  t.startsWith("'use strict'") ||
494
495
  t.startsWith('"use strict"');
495
496
  if (!startsWithCode)
@@ -12,7 +12,7 @@
12
12
  */
13
13
  import { spawn } from 'node:child_process';
14
14
  import { readFileSync, existsSync } from 'node:fs';
15
- import { loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
15
+ import { isClaude, loadConfig, splitCommand, withNoTools, withReadOnlyTools, resolveFramework, } from '../project/config.js';
16
16
  import { runAiSync, noteAiCall, traceArgs, traceCall } from './ai-usage.js';
17
17
  import { API, WEB, codeOutput, evidence } from '../prompts/blocks.js';
18
18
  import { stripStrayComments } from '../gates/validate.js';
@@ -339,7 +339,9 @@ export function buildPrompt(journey, map, testPath, focus, harnessPath, transpor
339
339
  ``,
340
340
  `NAMES THE PRODUCT GIVES. The doc says WHAT happens; the product names HOW to`,
341
341
  `reach it — a route, a button's label, a command, a table or a field.`,
342
- ` - When the CODE FACTS show that name, use it as written there.`,
342
+ ` - When the CODE FACTS show that name (in ROUTES, SHAPES or DATA), use it`,
343
+ ` as written there: the request fields a handler validates, a table's`,
344
+ ` columns, a status value. need() is never for a name the facts show.`,
343
345
  ` - When they do not (the code is not built yet, or the facts missed it),`,
344
346
  ` write the test anyway and take the name from need('<plain name>'):`,
345
347
  ` const { need } = require('@activate-core/test-forge/reporters/interface.cjs');`,
@@ -753,6 +755,9 @@ function writerCommand(prompt, memory) {
753
755
  if (!memory)
754
756
  return { ...withReadOnlyTools(cmd), input: prompt };
755
757
  const { system, message } = splitShared(prompt);
758
+ // Only the Claude CLI takes a system prompt; any other AI gets the rules at the top of its input.
759
+ if (system && !isClaude(cmd.program))
760
+ return { ...withNoTools(cmd), input: `${system}\n\n${message}` };
756
761
  return system ? { ...withNoTools(cmd, system), input: message } : { ...withNoTools(cmd), input: prompt };
757
762
  }
758
763
  /**
@@ -843,7 +848,8 @@ export async function writeFilesWithAI(files) {
843
848
  const config = loadConfig();
844
849
  const cmd = splitCommand(config.aiCommand);
845
850
  const { program, args } = system ? withNoTools(cmd, system) : withNoTools(cmd);
846
- return splitBatchAnswer(await runWithStdin(program, args, message, config.aiTimeoutMs), files.map((f) => f.path));
851
+ const input = system && !isClaude(cmd.program) ? `${system}\n\n${message}` : message;
852
+ return splitBatchAnswer(await runWithStdin(program, args, input, config.aiTimeoutMs), files.map((f) => f.path));
847
853
  }
848
854
  /** Each file's code from a batch answer, by its marker line. */
849
855
  export function splitBatchAnswer(out, paths) {
@@ -53,12 +53,16 @@ export function describeUsage(u) {
53
53
  }
54
54
  /** execFileSync for an AI CLI, counted. Same signature as the call it replaces. */
55
55
  export function runAiSync(program, args, options) {
56
+ return runAiWithStream(program, args, options).text;
57
+ }
58
+ /** The same call, also returning the raw event stream (tool results included) for a caller that needs it. */
59
+ export function runAiWithStream(program, args, options) {
56
60
  const started = Date.now();
57
61
  let out = '';
58
62
  try {
59
63
  // The event stream is larger than the answer alone: give it room.
60
64
  out = execFileSync(program, traceArgs(program, args), { ...options, maxBuffer: Math.max(options.maxBuffer ?? 0, 64 * 1024 * 1024) });
61
- return traceCall(String(options.input ?? ''), out, started);
65
+ return { text: traceCall(String(options.input ?? ''), out, started), raw: out };
62
66
  }
63
67
  catch (e) {
64
68
  traceCall(String(options.input ?? ''), String(e.stdout ?? ''), started, String(e));
@@ -77,6 +81,7 @@ let traceSeq = 0;
77
81
  const STAGES = [
78
82
  [/^=== REPAIR ===$/m, 'repair'],
79
83
  [/^You are a senior QA engineer\. Generate a complete/, 'write'],
84
+ [/^You are a senior QA engineer writing one .* test file/, 'write'],
80
85
  [/^You are a senior test architect/, 'plan'],
81
86
  [/^You are a senior engineer establishing the MECHANICS/, 'grounding'],
82
87
  [/^You review ONE claimed product defect/, 'defect-review'],
@@ -85,7 +90,7 @@ const STAGES = [
85
90
  [/^You are a senior QA engineer creating per-journey documentation/, 'discover'],
86
91
  [/^You are a senior QA engineer writing ONE journey doc/, 'discover'],
87
92
  [/^You are a senior QA engineer writing the journey doc for/, 'discover'],
88
- [/^(?:Fetch this reference and print its (?:FULL|full) content|Continue printing this reference)/, 'reference-fetch'],
93
+ [/^(?:Fetch this reference|Continue printing this reference)/, 'reference-fetch'],
89
94
  [/^You are a senior QA engineer reviewing ONE journey doc/, 'doc-review'],
90
95
  [/^You check ONE journey doc/, 'doc-review'],
91
96
  [/^You are setting up TestForge in this repository/, 'init'],
@@ -124,7 +129,9 @@ export function traceCall(prompt, raw, started, error) {
124
129
  }
125
130
  });
126
131
  const result = events.find((e) => e.type === 'result');
127
- const text = events.length ? String(result?.result ?? '') : raw;
132
+ // Only a Claude event stream has typed events; another AI's JSON answer is the answer itself.
133
+ const isStream = events.some((e) => typeof e.type === 'string');
134
+ const text = isStream ? String(result?.result ?? '') : raw;
128
135
  const ru = result?.usage ?? {};
129
136
  const tokens = ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'].reduce((n, k) => n + (Number(ru[k]) || 0), 0);
130
137
  const cost = Number(result?.total_cost_usd) || 0;
@@ -150,7 +157,7 @@ export function traceCall(prompt, raw, started, error) {
150
157
  try {
151
158
  mkdirSync(TRACE_DIR, { recursive: true });
152
159
  writeFileSync(join(TRACE_DIR, `${name}.prompt.md`), prompt);
153
- writeFileSync(join(TRACE_DIR, `${name}.${events.length ? 'stream.jsonl' : 'out.txt'}`), raw);
160
+ writeFileSync(join(TRACE_DIR, `${name}.${isStream ? 'stream.jsonl' : 'out.txt'}`), raw);
154
161
  const u = result?.usage ?? {};
155
162
  appendFileSync(join(TRACE_DIR, 'index.jsonl'), JSON.stringify({
156
163
  seq, stage, file: file ? basename(file) : undefined,
@@ -8,6 +8,7 @@
8
8
  * tests take the code's wording, and the next `generate` records it in the
9
9
  * journey's memory as a rename. No AI.
10
10
  */
11
+ import { DOCS_DIR } from '../paths.js';
11
12
  import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
12
13
  import { basename, extname, join, relative } from 'node:path';
13
14
  import { findMarkdownFiles } from '../find-files.js';
@@ -59,7 +60,7 @@ export function bindJourneys(opts = {}) {
59
60
  const haystack = sourceText(roots);
60
61
  let texts;
61
62
  const texter = () => (texts ??= shownTexts(rawSource(roots)));
62
- return journeys(opts.docsDir ?? 'docs', opts.journey).map((j) => ({
63
+ return journeys(opts.docsDir ?? DOCS_DIR, opts.journey).map((j) => ({
63
64
  journey: j.id,
64
65
  files: checkReadiness(testFilesIn(j.dir), haystack, texter).map((r) => ({ ...r, file: relative(process.cwd(), r.file) })),
65
66
  }));
@@ -71,7 +72,7 @@ export function bindJourneys(opts = {}) {
71
72
  */
72
73
  export function acceptNames(pairs, opts = {}) {
73
74
  const notes = [];
74
- for (const j of journeys(opts.docsDir ?? 'docs', opts.journey)) {
75
+ for (const j of journeys(opts.docsDir ?? DOCS_DIR, opts.journey)) {
75
76
  const files = [j.doc, ...testFilesIn(j.dir)];
76
77
  let touched = 0;
77
78
  for (const f of files) {
@@ -32,6 +32,20 @@ function withoutBlocks(content, problems) {
32
32
  const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
33
33
  return blocks.length ? spliceBlocks(content, blocks.map((block) => ({ block, text: '' }))) : content;
34
34
  }
35
+ /**
36
+ * What an answer adds to the file: its test blocks for the rules asked for.
37
+ * An answer that came back as a whole file (imports, an outer describe) or
38
+ * repeats a test the file already has would otherwise land nested or twice.
39
+ */
40
+ export function onlyNewTests(answer, file, rules) {
41
+ if (!/^\s*(?:import\s|const\s+\w+\s*=\s*require\(|(?:test\.)?describe\s*\()/m.test(answer))
42
+ return answer;
43
+ const have = new Set(findTestBlocks(file).map((b) => b.title));
44
+ return findTestBlocks(answer)
45
+ .filter((b) => !have.has(b.title) && (!b.rules.length || b.rules.some((r) => rules.includes(r))))
46
+ .map((b) => answer.slice(b.start, b.end).replace(/^\s*\n/, ''))
47
+ .join('\n\n');
48
+ }
35
49
  /** Is the file complete as it is? */
36
50
  export function fileIsComplete(content, rules, steps, knownTexts) {
37
51
  const { cov, problems } = review(content, rules, steps, knownTexts);
@@ -46,12 +60,29 @@ export async function completeFile(content, ctx) {
46
60
  const drop = (ps) => { for (const p of ps)
47
61
  if (!dropped.has(p.title))
48
62
  dropped.set(p.title, p.reason); };
49
- if (first.cov.missing.length || first.problems.length) {
50
- out = withoutNames(withoutBlocks(content, first.problems));
51
- drop(first.problems);
63
+ const locked = ctx.locked ?? new Set();
64
+ const before = new Set(findTestBlocks(content).map((b) => b.title));
65
+ // A test that passed is never touched. A kept file's other tests are only
66
+ // reported: a check added later must not quietly shrink a working suite.
67
+ // Only the tests asked for again (redo) go, and so do new tests just written.
68
+ const removable = (p) => !locked.has(p.title) && (!ctx.kept || !before.has(p.title) || !!ctx.redo?.(p.block));
69
+ const redoBlocks = ctx.kept && ctx.redo
70
+ ? findTestBlocks(content).filter((b) => !locked.has(b.title) && ctx.redo(b))
71
+ : [];
72
+ const redo = redoBlocks
73
+ .filter((b) => !first.problems.some((p) => p.block.start === b.start))
74
+ .map((b) => ({ title: b.title, rules: b.rules, reason: 'it waits on a name the code now has: written again from the code', block: b }));
75
+ const firstDrop = [...first.problems.filter(removable), ...redo];
76
+ const reported = first.problems.filter((p) => !removable(p));
77
+ if (reported.length) {
78
+ notes.push(`left ${reported.length} earlier test(s) in place that a check flags (${[...new Set(reported.map((p) => shortReason(p.reason)))].slice(0, 3).join('; ')}) — not removed, they were already in the file`);
79
+ }
80
+ if (first.cov.missing.length || firstDrop.length) {
81
+ out = withoutNames(withoutBlocks(content, firstDrop));
82
+ drop(firstDrop);
52
83
  // Only rules the doc has are written again: an id with no text makes the writer invent one.
53
84
  const known = new Set(ctx.memory.rules.filter((r) => r.tag).map((r) => r.id));
54
- const left = ruleCoverage(out, ctx.rules).missing.filter((id) => known.has(id));
85
+ const left = [...new Set([...ruleCoverage(out, ctx.rules).missing, ...redo.flatMap((p) => p.rules)])].filter((id) => known.has(id) && ctx.rules.includes(id));
55
86
  if (left.length) {
56
87
  const blocks = findTestBlocks(out);
57
88
  const head = out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000);
@@ -59,8 +90,8 @@ export async function completeFile(content, ctx) {
59
90
  const slice = journeySlice(ctx.memory, left);
60
91
  calls = 1;
61
92
  try {
62
- const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(first.problems, first.cov));
63
- const candidate = appendTests(out, tests);
93
+ const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(firstDrop, first.cov));
94
+ const candidate = appendTests(out, onlyNewTests(tests, out, left));
64
95
  const refused = ctx.check(candidate);
65
96
  if (refused)
66
97
  notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
@@ -71,13 +102,20 @@ export async function completeFile(content, ctx) {
71
102
  notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
72
103
  }
73
104
  // What the second answer still gets wrong is removed, not sent again.
74
- const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts });
105
+ const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts }).filter(removable);
75
106
  if (second.length) {
76
107
  out = withoutNames(withoutBlocks(out, second));
77
108
  drop(second);
78
109
  }
79
110
  }
80
111
  }
112
+ // The last word, checked mechanically: every test that passed is still here.
113
+ const lost = [...locked].filter((t) => before.has(t) && !findTestBlocks(out).some((b) => b.title === t));
114
+ if (lost.length) {
115
+ notes.push(`kept the file as it was: the change would have lost ${lost.length} test(s) that passed, e.g. "${lost[0].slice(0, 60)}"`);
116
+ out = content;
117
+ dropped.clear();
118
+ }
81
119
  // A cannot-test line that is no longer true (its rule has a test now) or never
82
120
  // was a reason ("the locator is unknown") does not stay in the file.
83
121
  const has = new Set(findTestBlocks(out).flatMap((b) => b.rules));
@@ -246,7 +246,8 @@ export function oracleClaim(t, content) {
246
246
  return `${t.title} ${NOT_CLAIMED}`;
247
247
  }
248
248
  /** A [spec] line that carries the client's words, quoted or pointing at the quote above it ("source as above"). */
249
- export const HAS_SOURCE = /\((?:source(?::\s*(?:[A-Z][A-Z0-9]+-\d+:?\s*)?["“]|:?\s*(?:as|same)\b)|same source\b)/i;
249
+ // A quoted source, a ticket key on its own ("source: SHOP-1") — the ticket is the source — or "same as".
250
+ export const HAS_SOURCE = /\((?:source(?::\s*(?:[A-Z][A-Z0-9]+-\d+:?\s*)?["“]|:\s*[A-Z][A-Z0-9]+-\d+\b|:?\s*(?:as|same)\b)|same source\b)/i;
250
251
  /** Playwright's report of a locator that matched more than one element. */
251
252
  const AMBIGUOUS = /strict mode violation|resolved to \d+ elements/i;
252
253
  /** An application file:line, the only evidence a defect may rest on. */