@activate-core/test-forge 1.3.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +39 -33
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +49 -2
  5. package/dist/core/docs/doc-guidance.js +70 -29
  6. package/dist/core/docs/excerpts.js +294 -0
  7. package/dist/core/docs/grounding.js +55 -22
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +60 -3
  10. package/dist/core/docs/review.js +38 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +3 -3
  13. package/dist/core/gates/coverage.js +1 -16
  14. package/dist/core/gates/test-quality.js +295 -0
  15. package/dist/core/gates/validate.js +171 -7
  16. package/dist/core/generate/ai-generator.js +267 -117
  17. package/dist/core/generate/ai-usage.js +116 -5
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/complete-file.js +77 -0
  20. package/dist/core/generate/defect-review.js +59 -9
  21. package/dist/core/generate/doc-diff.js +140 -0
  22. package/dist/core/generate/harness.js +18 -34
  23. package/dist/core/generate/journey-memory.js +394 -0
  24. package/dist/core/generate/plan-cache.js +114 -7
  25. package/dist/core/generate/review-journey.js +3 -3
  26. package/dist/core/generate/run-generate.js +240 -82
  27. package/dist/core/generate/suite-planner.js +128 -23
  28. package/dist/core/generate/test-blocks.js +84 -0
  29. package/dist/core/generate/update-journey.js +118 -0
  30. package/dist/core/paths.js +90 -13
  31. package/dist/core/project/config.js +25 -2
  32. package/dist/core/project/env-template.js +2 -1
  33. package/dist/core/run/ci.js +1 -1
  34. package/dist/core/run/clean.js +1 -1
  35. package/dist/core/run/readiness.js +146 -0
  36. package/dist/core/run/run-tests.js +26 -2
  37. package/package.json +1 -1
  38. package/reporters/_shared.cjs +2 -2
  39. package/reporters/run-data.cjs +1 -1
  40. package/reporters/ui-helpers.cjs +78 -1
  41. package/dist/core/generate/verify-journey.js +0 -215
@@ -3,9 +3,20 @@
3
3
  * one place every AI call goes through. Tokens are estimated as characters
4
4
  * sent and received divided by four — close enough to compare journeys and
5
5
  * runs, not a bill.
6
+ *
7
+ * Every Claude call is run with its event stream, so the counts above use the
8
+ * real token use and cost it reports. TESTFORGE_TRACE=1 also records every call in order under
9
+ * testforge/runs/trace/<run>/: the prompt, the CLI's full event stream, and
10
+ * one index.jsonl line with the stage, the real token usage and cost the
11
+ * Claude CLI reports, and every tool the AI used (which files it read, what
12
+ * it searched). The characters ÷ 4 estimate misses those reads; this does not.
6
13
  */
7
14
  import { execFileSync } from 'node:child_process';
8
- const total = { calls: 0, ms: 0, inChars: 0, outChars: 0 };
15
+ import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs';
16
+ import { basename, join } from 'node:path';
17
+ import { isClaude } from '../project/config.js';
18
+ import { RUNS_DIR } from '../paths.js';
19
+ const total = { calls: 0, ms: 0, inChars: 0, outChars: 0, tokens: 0, costUsd: 0 };
9
20
  export function noteAiCall(inChars, outChars, ms) {
10
21
  total.calls += 1;
11
22
  total.ms += ms;
@@ -18,22 +29,122 @@ export function aiUsage() {
18
29
  }
19
30
  /** What was spent between two snapshots. */
20
31
  export function usageSince(start, now = aiUsage()) {
21
- return { calls: now.calls - start.calls, ms: now.ms - start.ms, inChars: now.inChars - start.inChars, outChars: now.outChars - start.outChars };
32
+ return {
33
+ calls: now.calls - start.calls, ms: now.ms - start.ms, inChars: now.inChars - start.inChars, outChars: now.outChars - start.outChars,
34
+ tokens: now.tokens - start.tokens, costUsd: now.costUsd - start.costUsd,
35
+ };
22
36
  }
23
37
  export const approxTokens = (u) => Math.round((u.inChars + u.outChars) / 4);
24
38
  export function describeUsage(u) {
25
39
  const min = u.ms / 60_000;
26
- return `${u.calls} AI call(s), ${min >= 1 ? `${min.toFixed(1)} min` : `${Math.round(u.ms / 1000)} s`}, ~${approxTokens(u).toLocaleString('en-GB')} tokens`;
40
+ const time = min >= 1 ? `${min.toFixed(1)} min` : `${Math.round(u.ms / 1000)} s`;
41
+ // The CLI's own count includes what the AI read with its tools and every
42
+ // turn's re-sent context; characters ÷ 4 sees only our prompt and its answer.
43
+ const tokens = u.tokens
44
+ ? `${u.tokens.toLocaleString('en-GB')} tokens, $${u.costUsd.toFixed(2)}`
45
+ : `~${approxTokens(u).toLocaleString('en-GB')} tokens (estimate)`;
46
+ return `${u.calls} AI call(s), ${time}, ${tokens}`;
27
47
  }
28
48
  /** execFileSync for an AI CLI, counted. Same signature as the call it replaces. */
29
49
  export function runAiSync(program, args, options) {
30
50
  const started = Date.now();
31
51
  let out = '';
32
52
  try {
33
- out = execFileSync(program, args, options);
34
- return out;
53
+ // The event stream is larger than the answer alone: give it room.
54
+ out = execFileSync(program, traceArgs(program, args), { ...options, maxBuffer: Math.max(options.maxBuffer ?? 0, 64 * 1024 * 1024) });
55
+ return traceCall(String(options.input ?? ''), out, started);
56
+ }
57
+ catch (e) {
58
+ traceCall(String(options.input ?? ''), String(e.stdout ?? ''), started, String(e));
59
+ throw e;
35
60
  }
36
61
  finally {
37
62
  noteAiCall(String(options.input ?? '').length, out.length, Date.now() - started);
38
63
  }
39
64
  }
65
+ // ---- trace (TESTFORGE_TRACE=1) ----
66
+ const TRACE_DIR = process.env.TESTFORGE_TRACE
67
+ ? join(RUNS_DIR, 'trace', new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19))
68
+ : '';
69
+ let traceSeq = 0;
70
+ /** Which step a prompt belongs to, from its opening words. Repair first: it embeds the write prompt. */
71
+ const STAGES = [
72
+ [/^=== REPAIR ===$/m, 'repair'],
73
+ [/^You are a senior QA engineer\. Generate a complete/, 'write'],
74
+ [/^You are a senior test architect/, 'plan'],
75
+ [/^You are a senior engineer establishing the MECHANICS/, 'grounding'],
76
+ [/^You review ONE claimed product defect/, 'defect-review'],
77
+ [/^You are a senior SDET writing the SHARED test harness/, 'harness'],
78
+ [/^You are a senior SDET writing a TEST DATA SEED/, 'seed'],
79
+ [/^You are a senior QA engineer creating per-journey documentation/, 'discover'],
80
+ [/^You are a senior QA engineer writing ONE journey doc/, 'discover'],
81
+ [/^Fetch this reference and print its FULL content/, 'reference-fetch'],
82
+ [/^You are a senior QA engineer reviewing ONE journey doc/, 'doc-review'],
83
+ [/^You are a senior QA engineer\. The developer has given you an/, 'reference'],
84
+ [/^You are a senior engineer mapping a codebase/, 'map'],
85
+ [/^You are setting up the QA tool/, 'init'],
86
+ [/^--- (CODE FACTS|TESTKIT)/, 'write'],
87
+ [/^Reply with exactly: PONG/, 'ping'],
88
+ ];
89
+ /**
90
+ * The CLI args, plus the event-stream flags for a Claude call: its result
91
+ * event carries the real token use and cost, which every run counts, and the
92
+ * stream is what a trace saves.
93
+ */
94
+ export function traceArgs(program, args) {
95
+ return isClaude(program) ? [...args, '--output-format', 'stream-json', '--verbose'] : args;
96
+ }
97
+ /**
98
+ * Record one finished call and return its answer text. Without tracing the
99
+ * raw output IS the answer; with it, the answer is the stream's result event.
100
+ */
101
+ export function traceCall(prompt, raw, started, error) {
102
+ const events = raw.split('\n').flatMap((l) => {
103
+ if (!l.startsWith('{'))
104
+ return [];
105
+ try {
106
+ return [JSON.parse(l)];
107
+ }
108
+ catch {
109
+ return [];
110
+ }
111
+ });
112
+ const result = events.find((e) => e.type === 'result');
113
+ const text = events.length ? String(result?.result ?? '') : raw;
114
+ const ru = result?.usage ?? {};
115
+ total.tokens += ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'].reduce((n, k) => n + (Number(ru[k]) || 0), 0);
116
+ total.costUsd += Number(result?.total_cost_usd) || 0;
117
+ if (!TRACE_DIR)
118
+ return text;
119
+ const tools = events
120
+ .filter((e) => e.type === 'assistant')
121
+ .flatMap((e) => e.message?.content ?? [])
122
+ .filter((c) => c.type === 'tool_use')
123
+ .map((c) => {
124
+ const i = c.input ?? {};
125
+ return `${c.name} ${String(i.file_path ?? i.pattern ?? i.path ?? i.command ?? JSON.stringify(i)).slice(0, 200)}`;
126
+ });
127
+ const seq = String(++traceSeq).padStart(3, '0');
128
+ const stage = STAGES.find(([re]) => re.test(prompt))?.[1] ?? 'ai';
129
+ const file = /Save this file at EXACTLY: (\S+)/.exec(prompt)?.[1];
130
+ const name = `${seq}-${stage}${file ? `-${basename(file).replace(/\.[^.]+$/, '').replace(/\.test$/, '')}` : ''}`;
131
+ try {
132
+ mkdirSync(TRACE_DIR, { recursive: true });
133
+ writeFileSync(join(TRACE_DIR, `${name}.prompt.md`), prompt);
134
+ writeFileSync(join(TRACE_DIR, `${name}.${events.length ? 'stream.jsonl' : 'out.txt'}`), raw);
135
+ const u = result?.usage ?? {};
136
+ appendFileSync(join(TRACE_DIR, 'index.jsonl'), JSON.stringify({
137
+ seq, stage, file: file ? basename(file) : undefined,
138
+ startedAt: new Date(started).toISOString(), ms: Date.now() - started,
139
+ promptChars: prompt.length, answerChars: text.length,
140
+ turns: result?.num_turns, costUsd: result?.total_cost_usd,
141
+ inputTokens: u.input_tokens, outputTokens: u.output_tokens,
142
+ cacheReadTokens: u.cache_read_input_tokens, cacheWriteTokens: u.cache_creation_input_tokens,
143
+ tools, error,
144
+ }) + '\n');
145
+ }
146
+ catch (e) {
147
+ console.warn(` trace: could not record call ${seq}: ${e.message}`);
148
+ }
149
+ return text;
150
+ }
@@ -0,0 +1,91 @@
1
+ /**
2
+ * testforge — do the tests and the shipped code use the same names?
3
+ *
4
+ * `testforge bind` reports, per journey, which test files are ready to run
5
+ * (the code has every name and page they use), which test a feature not built
6
+ * yet, and which use a name the code spells differently. `--accept "A=B"`
7
+ * settles a naming difference in the code's favour: the doc and the journey's
8
+ * tests take the code's wording, and the next `generate` records it in the
9
+ * journey's memory as a rename. No AI.
10
+ */
11
+ import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
12
+ import { basename, extname, join, relative } from 'node:path';
13
+ import { findMarkdownFiles } from '../find-files.js';
14
+ import { notAJourneyDoc, parseJourney } from '../docs/parser.js';
15
+ import { loadConfig, testDirForType } from '../project/config.js';
16
+ import { matchesJourney } from './plan-cache.js';
17
+ import { sourceText } from '../docs/excerpts.js';
18
+ import { checkReadiness, shownTexts, testFilesIn } from '../run/readiness.js';
19
+ function journeys(docsDir, only) {
20
+ const config = loadConfig();
21
+ const out = [];
22
+ for (const file of findMarkdownFiles(docsDir)) {
23
+ const markdown = readFileSync(join(docsDir, file), 'utf-8');
24
+ if (notAJourneyDoc(file, markdown))
25
+ continue;
26
+ const area = /[\\/]/.test(file) ? file.split(/[\\/]/)[0] : undefined;
27
+ const journey = parseJourney(markdown, area, basename(file, extname(file)));
28
+ if (only && !matchesJourney(file, journey.id, only))
29
+ continue;
30
+ const dir = join(testDirForType(config, journey.area), journey.id);
31
+ if (existsSync(dir))
32
+ out.push({ id: journey.id, doc: join(docsDir, file), dir });
33
+ }
34
+ return out;
35
+ }
36
+ /** The raw source under the project's roots, for the texts the code shows. */
37
+ function rawSource(roots) {
38
+ const parts = [];
39
+ const walk = (d) => {
40
+ for (const e of existsSync(d) ? readdirSync(d, { withFileTypes: true }) : []) {
41
+ const p = join(d, e.name);
42
+ if (e.isDirectory()) {
43
+ if (!/^(node_modules|\.next|dist|build|vendor|tests?)$/.test(e.name))
44
+ walk(p);
45
+ }
46
+ else if (/\.(tsx?|jsx?|vue|svelte|php)$/.test(e.name) && !/\.(test|spec)\./.test(e.name))
47
+ parts.push(readFileSync(p, 'utf-8'));
48
+ }
49
+ };
50
+ for (const r of roots)
51
+ walk(r);
52
+ return parts.join('\n');
53
+ }
54
+ export function sourceRoots() {
55
+ return ['src', ...(loadConfig().relatedDirs ?? [])];
56
+ }
57
+ export function bindJourneys(opts = {}) {
58
+ const roots = sourceRoots();
59
+ const haystack = sourceText(roots);
60
+ let texts;
61
+ const texter = () => (texts ??= shownTexts(rawSource(roots)));
62
+ return journeys(opts.docsDir ?? 'docs', opts.journey).map((j) => ({
63
+ journey: j.id,
64
+ files: checkReadiness(testFilesIn(j.dir), haystack, texter).map((r) => ({ ...r, file: relative(process.cwd(), r.file) })),
65
+ }));
66
+ }
67
+ /**
68
+ * Settle naming differences in the code's favour: each "doc wording=code
69
+ * wording" pair is replaced in the journey docs that use it and in their
70
+ * tests. Returns what changed.
71
+ */
72
+ export function acceptNames(pairs, opts = {}) {
73
+ const notes = [];
74
+ for (const j of journeys(opts.docsDir ?? 'docs', opts.journey)) {
75
+ const files = [j.doc, ...testFilesIn(j.dir)];
76
+ let touched = 0;
77
+ for (const f of files) {
78
+ const before = readFileSync(f, 'utf-8');
79
+ let after = before;
80
+ for (const [from, to] of pairs)
81
+ after = after.split(from).join(to);
82
+ if (after !== before) {
83
+ writeFileSync(f, after, 'utf-8');
84
+ touched += 1;
85
+ }
86
+ }
87
+ if (touched)
88
+ notes.push(`${j.id}: ${touched} file(s) updated (doc and tests)`);
89
+ }
90
+ return notes;
91
+ }
@@ -0,0 +1,77 @@
1
+ /**
2
+ * testforge — finish a written file: every rule tested, every test able to fail.
3
+ *
4
+ * A file can pass every file-level gate and still leave rules out, or hold a
5
+ * test that passes whatever the app does. Those tests are taken out, and the
6
+ * rules left without a test are written again in ONE small call that sees only
7
+ * those rules, the file's top and why the earlier tests were refused. What
8
+ * that call still gets wrong is removed and reported, never sent again: a
9
+ * missing test is honest, a test that cannot fail is not.
10
+ */
11
+ import { factsSlice, journeySlice } from './journey-memory.js';
12
+ import { appendTests, findTestBlocks, spliceBlocks } from './test-blocks.js';
13
+ import { acceptedReason, describeProblems, ruleCoverage, testProblems } from '../gates/test-quality.js';
14
+ const review = (content, rules, steps, knownTexts) => ({
15
+ cov: ruleCoverage(content, rules),
16
+ problems: testProblems(content, { steps, knownTexts }),
17
+ });
18
+ /** The file without the tests that have a problem. */
19
+ function withoutBlocks(content, problems) {
20
+ const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
21
+ return blocks.length ? spliceBlocks(content, blocks.map((block) => ({ block, text: '' }))) : content;
22
+ }
23
+ /** Is the file complete as it is? */
24
+ export function fileIsComplete(content, rules, steps, knownTexts) {
25
+ const { cov, problems } = review(content, rules, steps, knownTexts);
26
+ return !cov.missing.length && !problems.length;
27
+ }
28
+ export async function completeFile(content, ctx) {
29
+ const notes = [];
30
+ const first = review(content, ctx.rules, ctx.steps, ctx.knownTexts);
31
+ let calls = 0;
32
+ let out = content;
33
+ const dropped = new Set();
34
+ if (first.cov.missing.length || first.problems.length) {
35
+ out = withoutBlocks(content, first.problems);
36
+ for (const p of first.problems)
37
+ dropped.add(p.title);
38
+ const left = ruleCoverage(out, ctx.rules).missing;
39
+ if (left.length) {
40
+ const blocks = findTestBlocks(out);
41
+ const head = out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000);
42
+ const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(out)?.[2] ?? ctx.journey.name;
43
+ const slice = journeySlice(ctx.memory, left);
44
+ calls = 1;
45
+ try {
46
+ const tests = await ctx.write(ctx.journey, ctx.path, head, title, '', ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), { journey: slice.text, helpers: ctx.helpers, rules: left }, describeProblems(first.problems, first.cov));
47
+ const candidate = appendTests(out, tests);
48
+ const refused = ctx.check(candidate);
49
+ if (refused)
50
+ notes.push(`tests written again for ${left.join(', ')} were refused (${refused.split('\n')[0]})`);
51
+ else
52
+ out = candidate;
53
+ }
54
+ catch (e) {
55
+ notes.push(`the call to write ${left.join(', ')} again failed — ${e.message.split('\n')[0].slice(0, 120)}`);
56
+ }
57
+ // What the second answer still gets wrong is removed, not sent again.
58
+ const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts });
59
+ if (second.length) {
60
+ out = withoutBlocks(out, second);
61
+ for (const p of second)
62
+ dropped.add(p.title);
63
+ }
64
+ }
65
+ }
66
+ // A cannot-test line that is no longer true (its rule has a test now) or never
67
+ // was a reason ("the locator is unknown") does not stay in the file.
68
+ const has = new Set(findTestBlocks(out).flatMap((b) => b.rules));
69
+ out = out.replace(/^[ \t]*\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)\n/gm, (line, ids, why) => ids.split(/\s*,\s*/).every((id) => has.has(id)) || !acceptedReason(why) ? '' : line);
70
+ const cov = ruleCoverage(out, ctx.rules);
71
+ const tested = [...has].filter((id) => ctx.rules.includes(id));
72
+ if (dropped.size)
73
+ notes.push(`took out ${dropped.size} test(s) that could not check their rule`);
74
+ if (cov.missing.length)
75
+ notes.push(`no test for ${cov.missing.join(', ')}`);
76
+ return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped], calls, notes };
77
+ }
@@ -18,7 +18,9 @@
18
18
  */
19
19
  import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
20
20
  import { dirname, relative } from 'node:path';
21
- import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
21
+ import { loadConfig, splitCommand, withNoTools, withReferenceTools } from '../project/config.js';
22
+ import { readRules, ruleIndex } from './journey-memory.js';
23
+ import { sourceExcerpts } from '../docs/excerpts.js';
22
24
  import { runWithStdin, DEFECT_MARK, FINDING_MARK } from './ai-generator.js';
23
25
  import { aiUnavailableReason, noteAiFailure, noteAiSuccess, waitForAi } from './ai-session.js';
24
26
  const IMPACTS = ['blocker', 'major', 'minor'];
@@ -49,7 +51,7 @@ export function decisionsSource(spec) {
49
51
  return { text: readFileSync(spec, 'utf-8') };
50
52
  return { link: spec };
51
53
  }
52
- export function buildDefectReviewPrompt(journey, group, passes, decisions) {
54
+ export function buildDefectReviewPrompt(journey, group, passes, decisions, evidence) {
53
55
  const files = [...new Set(group.tests.map((t) => relative(process.cwd(), t.file)))];
54
56
  const sample = group.tests.slice(0, 3);
55
57
  return [
@@ -79,8 +81,22 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
79
81
  ``,
80
82
  ]
81
83
  : []),
82
- `Read the test file(s) (${files.join(', ')}), the application source the test`,
83
- `exercises, and the server code behind it. You have read-only tools.`,
84
+ ...(evidence
85
+ ? [
86
+ `You have no tools. The failing tests' code and the application code the`,
87
+ `claim points at are below. When they do not show the cause, the verdict`,
88
+ `cannot be "defect" or "finding": say "unproven".`,
89
+ ``,
90
+ `--- TEST CODE ---`,
91
+ evidence.tests,
92
+ `--- APPLICATION CODE ---`,
93
+ evidence.app || '(the claim names no application file that could be found)',
94
+ `--- END CODE ---`,
95
+ ]
96
+ : [
97
+ `Read the test file(s) (${files.join(', ')}), the application source the test`,
98
+ `exercises, and the server code behind it. You have read-only tools.`,
99
+ ]),
84
100
  ``,
85
101
  `Decide exactly one verdict:`,
86
102
  `- "defect": ALL of these hold — (a) the criterion is stated in the journey`,
@@ -116,9 +132,9 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
116
132
  : decisions.link
117
133
  ? [`OPEN DECISIONS AND KNOWN ISSUES (not requirements yet) — read it: ${decisions.link}`, ``]
118
134
  : [`No list of open decisions was provided.`, ``]),
119
- `--- JOURNEY DOCUMENT ---`,
120
- journey.rawMarkdown,
121
- `--- END ---`,
135
+ ...(evidence
136
+ ? [`--- THE JOURNEY RULES THIS CLAIM IS ABOUT ---`, evidence.rules, `--- END ---`]
137
+ : [`--- JOURNEY DOCUMENT ---`, journey.rawMarkdown, `--- END ---`]),
122
138
  ``,
123
139
  `One claim often covers many rows of the same test (one input per row). Judge`,
124
140
  `every row: when some rows fall under a different verdict than the rest (one`,
@@ -131,6 +147,37 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
131
147
  ` "exceptions": [{"tests": [<row numbers>], "verdict": "...", "reason": "...", "evidence": "...", "impact": "..."}]}`,
132
148
  ].join('\n');
133
149
  }
150
+ /**
151
+ * The code a claim is judged on, collected without the AI: each failing
152
+ * test's own lines, the application lines the claim cites (and the files
153
+ * holding the texts it quotes), and the journey rules it names.
154
+ */
155
+ export function reviewEvidence(group, journeyMarkdown, roots) {
156
+ const tests = group.tests.slice(0, 3).map((t) => {
157
+ try {
158
+ const lines = readFileSync(t.file, 'utf-8').split('\n');
159
+ const from = Math.max(1, t.line - 10);
160
+ // The test body: from just above its declaration to the next test.
161
+ let to = Math.min(lines.length, t.line + 60);
162
+ for (let i = t.line; i < to; i++)
163
+ if (/^\s*test(\.\w+)?\(/.test(lines[i]) && i > t.line) {
164
+ to = i;
165
+ break;
166
+ }
167
+ return `### ${relative(process.cwd(), t.file)}\n${lines.slice(from - 1, to).map((l, i) => `${from + i}: ${l}`).join('\n')}`;
168
+ }
169
+ catch {
170
+ return `### ${relative(process.cwd(), t.file)} (unreadable)`;
171
+ }
172
+ });
173
+ const memory = readRules(journeyMarkdown);
174
+ const ids = new Set(group.claim.match(/\bR\d+\b/g) ?? []);
175
+ const named = memory.rules.filter((r) => ids.has(r.id));
176
+ const rules = named.length
177
+ ? named.map((r) => `${r.id} ${r.section}\n${r.text}`).join('\n\n')
178
+ : memory.rules.length ? ruleIndex(memory) : journeyMarkdown;
179
+ return { tests: [...new Set(tests)].join('\n\n'), app: sourceExcerpts(group.claim, roots, 15_000), rules };
180
+ }
134
181
  /** How many rows of one claim the reviewer is shown and may single out. */
135
182
  const MAX_LISTED = 40;
136
183
  const asVerdict = (v) => (VERDICTS.includes(v) ? v : 'unproven');
@@ -298,8 +345,11 @@ checkpoint) {
298
345
  const config = loadConfig();
299
346
  const decisions = decisionsSource(config.decisions);
300
347
  // A linked list (an internal Notion page) needs the reference tools to be read.
301
- const tools = decisions.link ? withReferenceTools : withReadOnlyTools;
348
+ // Everything else is handed over as evidence, so the reviewer needs no tools.
349
+ const noTools = !decisions.link;
350
+ const tools = decisions.link ? withReferenceTools : withNoTools;
302
351
  const { program, args } = tools(splitCommand(config.aiCommand));
352
+ const roots = ['src', ...(config.relatedDirs ?? [])];
303
353
  const ask = io?.ask ?? ((prompt) => runWithStdin(program, args, prompt, config.aiTimeoutMs));
304
354
  const wait = io?.wait ?? (() => ((config.waitForLimit ?? true) ? waitForAi(config) : Promise.resolve(false)));
305
355
  const out = [];
@@ -317,7 +367,7 @@ checkpoint) {
317
367
  const siblings = passes.filter((p) => files.has(p.file));
318
368
  let raw = '';
319
369
  try {
320
- raw = await ask(buildDefectReviewPrompt(journey, g, siblings, decisions));
370
+ raw = await ask(buildDefectReviewPrompt(journey, g, siblings, decisions, noTools ? reviewEvidence(g, journey.rawMarkdown, roots) : undefined));
321
371
  noteAiSuccess();
322
372
  }
323
373
  catch (e) {
@@ -0,0 +1,140 @@
1
+ /**
2
+ * testforge — what changed in a journey doc, rule by rule.
3
+ *
4
+ * The journey's rules from the last run are kept in its memory. A new version
5
+ * of the doc is matched against them without the AI, so a doc that gained one
6
+ * feature or had one label renamed costs only that rule, never the whole doc.
7
+ * Rule ids stay stable across versions: a rule keeps its id while it exists,
8
+ * and a new rule gets the next free one, so the `// oracle: R12` comments in
9
+ * the tests keep pointing at the right rule.
10
+ */
11
+ import { existsSync, readFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ import { journeyDir } from '../paths.js';
14
+ export function memoryPath(journey) {
15
+ return join(journeyDir(journey), 'rules.json');
16
+ }
17
+ /** The journey as the last run saw it, or undefined on a first run. */
18
+ export function loadMemory(journey) {
19
+ const p = memoryPath(journey);
20
+ if (!existsSync(p))
21
+ return undefined;
22
+ try {
23
+ const m = JSON.parse(readFileSync(p, 'utf-8'));
24
+ return Array.isArray(m.rules) ? m : undefined;
25
+ }
26
+ catch {
27
+ return undefined;
28
+ }
29
+ }
30
+ const norm = (t) => t.replace(/\s+/g, ' ').trim();
31
+ const QUOTED = /"([^"\n]+)"|`([^`\n]+)`|'([^'\n]{2,})'/g;
32
+ const quotes = (t) => [...t.matchAll(QUOTED)].map((m) => m[1] ?? m[2] ?? m[3]);
33
+ const unquoted = (t) => norm(t.replace(QUOTED, '""'));
34
+ const words = (t) => new Set(norm(t).toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 2));
35
+ function similarity(a, b) {
36
+ const x = words(a);
37
+ const y = words(b);
38
+ if (!x.size || !y.size)
39
+ return 0;
40
+ let both = 0;
41
+ for (const w of x)
42
+ if (y.has(w))
43
+ both += 1;
44
+ return both / (x.size + y.size - both);
45
+ }
46
+ /**
47
+ * Give the new doc's rules the ids they had last time and say what happened
48
+ * to each. Exact text keeps its id; the same text with different quoted
49
+ * strings is a rename; a close match is a change; the rest is added or
50
+ * removed. Pure: no AI, no I/O.
51
+ */
52
+ export function alignRules(prev, next, retired = []) {
53
+ const diff = { same: [], changed: [], renamed: [], added: [], removed: [] };
54
+ const free = new Set(prev.map((r) => r.id));
55
+ const byText = new Map();
56
+ for (const r of prev)
57
+ byText.set(norm(r.text), [...(byText.get(norm(r.text)) ?? []), r]);
58
+ const id = new Map();
59
+ // 1. Unchanged wording keeps its id.
60
+ for (const r of next.rules) {
61
+ const match = (byText.get(norm(r.text)) ?? []).find((p) => free.has(p.id));
62
+ if (!match)
63
+ continue;
64
+ id.set(r, match.id);
65
+ free.delete(match.id);
66
+ diff.same.push(match.id);
67
+ }
68
+ // 2. Same wording around different quoted texts: a rename.
69
+ for (const r of next.rules) {
70
+ if (id.has(r))
71
+ continue;
72
+ const match = prev.find((p) => free.has(p.id) && p.tag === r.tag && unquoted(p.text) === unquoted(r.text));
73
+ if (!match)
74
+ continue;
75
+ const before = quotes(match.text);
76
+ const after = quotes(r.text);
77
+ id.set(r, match.id);
78
+ free.delete(match.id);
79
+ const pairs = before.map((b, i) => [b, after[i]]).filter(([b, a]) => a !== undefined && a !== b);
80
+ diff.renamed.push({ id: match.id, pairs: [...new Map(pairs.map((p) => [`${p[0]}\u0000${p[1]}`, p])).values()] });
81
+ }
82
+ // 2b. A wording renamed in one rule's quotes ("Assign" → "Choose") is the same
83
+ // rename in a rule that writes it without quotes: same id, a rename, no AI.
84
+ const known = [...new Map(diff.renamed.flatMap((r) => r.pairs).map((p) => [`${p[0]}\u0000${p[1]}`, p])).values()];
85
+ if (known.length) {
86
+ const apply = (t, pairs) => pairs.reduce((acc, [a, b]) => acc.split(a).join(b), t);
87
+ for (const r of next.rules) {
88
+ if (id.has(r))
89
+ continue;
90
+ const match = prev.find((p) => free.has(p.id) && p.tag === r.tag && norm(apply(p.text, known)) === norm(r.text));
91
+ if (!match)
92
+ continue;
93
+ id.set(r, match.id);
94
+ free.delete(match.id);
95
+ diff.renamed.push({ id: match.id, pairs: known.filter(([a]) => match.text.includes(a)) });
96
+ }
97
+ }
98
+ // 3. A close match, preferring the same section: the rule changed.
99
+ for (const r of next.rules) {
100
+ if (id.has(r))
101
+ continue;
102
+ let best;
103
+ let score = 0;
104
+ for (const p of prev) {
105
+ if (!free.has(p.id) || p.tag !== r.tag)
106
+ continue;
107
+ const s = similarity(p.text, r.text) + (p.section === r.section ? 0.1 : 0);
108
+ if (s > score) {
109
+ best = p;
110
+ score = s;
111
+ }
112
+ }
113
+ if (!best || score < 0.6)
114
+ continue;
115
+ id.set(r, best.id);
116
+ free.delete(best.id);
117
+ diff.changed.push(best.id);
118
+ }
119
+ // 4. Everything else is new, numbered after the highest id ever used.
120
+ // A removed rule's id is never given to a new one: old results still name it.
121
+ let top = Math.max(0, ...prev.map((r) => Number(r.id.slice(1)) || 0), ...retired.map((id) => Number(id.slice(1)) || 0));
122
+ for (const r of next.rules) {
123
+ if (id.has(r))
124
+ continue;
125
+ const fresh = `R${++top}`;
126
+ id.set(r, fresh);
127
+ diff.added.push(fresh);
128
+ }
129
+ diff.removed = [...free];
130
+ return { memory: { ...next, rules: next.rules.map((r) => ({ ...r, id: id.get(r) })) }, diff };
131
+ }
132
+ export function describeDiff(d) {
133
+ return [
134
+ `${d.same.length} same`,
135
+ d.renamed.length ? `${d.renamed.length} renamed` : '',
136
+ d.changed.length ? `${d.changed.length} changed` : '',
137
+ d.added.length ? `${d.added.length} added` : '',
138
+ d.removed.length ? `${d.removed.length} removed` : '',
139
+ ].filter(Boolean).join(', ');
140
+ }
@@ -11,11 +11,10 @@
11
11
  * auth (read-only tools), so it connects and seeds against the actual
12
12
  * schema — not a guess.
13
13
  */
14
- import { spawn } from 'node:child_process';
15
14
  import { existsSync, readFileSync } from 'node:fs';
16
15
  import { createHash } from 'node:crypto';
17
16
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
18
- import { cleanOutput } from './ai-generator.js';
17
+ import { cleanOutput, runWithStdin } from './ai-generator.js';
19
18
  import { runAiSync } from './ai-usage.js';
20
19
  const MAP_PATH = 'testforge.md';
21
20
  /**
@@ -44,9 +43,13 @@ export const HARNESS_CONTRACT = [
44
43
  ` - created(kind, id, cleanup?, role?): void record what the run made (H8).`,
45
44
  ` - the "run" role: token('run'), ensureToken('run'), login(page, 'run'),`,
46
45
  ` apiContext(request, 'run') — the account tests that CREATE data use (H8).`,
46
+ ` - distinctRunAccount(): boolean true only when the "run" role is a`,
47
+ ` different account from the fixture role (H8), so a test that needs two`,
48
+ ` different people at once can tell whether it has them.`,
47
49
  `Plus, for Playwright suites:`,
48
50
  ` - openStep(page, step, input?): Promise<true | string> open a later step`,
49
51
  ` of a multi-step flow with what the earlier steps would have left (H9).`,
52
+ ` - STEPS: object the step names openStep accepts (H9).`,
50
53
  ` - settledClick, fillFrameInputs, watchDialogs: re-exported unchanged from`,
51
54
  ` require('@activate-core/test-forge/reporters/ui-helpers.cjs') — never`,
52
55
  ` rewritten here.`,
@@ -171,7 +174,9 @@ export const HARNESS_CONTRACT = [
171
174
  ` signup — create a fresh account through the product's own sign-up, only`,
172
175
  ` when this environment can complete it (no code it cannot read and`,
173
176
  ` cannot bypass under H7); otherwise fall back to fixture and say so once.`,
174
- ` Mint and cache its token like any role.`,
177
+ ` Mint and cache its token like any role. distinctRunAccount() returns true`,
178
+ ` only in pool or signup mode, when the run account really differs from`,
179
+ ` the fixture one.`,
175
180
  `H9. STEPS OF A FLOW. A later step of a multi-step flow (onboarding, a`,
176
181
  ` booking or checkout wizard, a sign-up in stages) depends on what the`,
177
182
  ` earlier steps stored — a choice, an address, a cart, a plan — in URL`,
@@ -186,8 +191,14 @@ export const HARNESS_CONTRACT = [
186
191
  ` - It waits until the step is really shown and resolves true; when the`,
187
192
  ` product still sends the visitor elsewhere it resolves a string naming`,
188
193
  ` the step it landed on, so the test can skip with that reason.`,
189
- ` - Offer only the steps the flow has; when the project has no such flow,`,
190
- ` export openStep as a function resolving 'this product has no multi-step flow'.`,
194
+ ` - Offer EVERY step the flow has, up to and including its last page before`,
195
+ ` the result — the summary, review or checkout page where the visitor`,
196
+ ` confirms and pays — reaching a page that follows a choice (an operator,`,
197
+ ` a plan, a slot) by making that choice through the UI, as a visitor would.`,
198
+ ` Keep the names in one exported object STEPS (name → what identifies the`,
199
+ ` page), and look the step up in it: an unknown name resolves`,
200
+ ` 'unknown step "<name>" (steps: …)'. When the project has no such flow,`,
201
+ ` export STEPS = {} and openStep resolving 'this product has no multi-step flow'.`,
191
202
  ].join('\n');
192
203
  /**
193
204
  * Stamp written into every generated testkit: the contract it was built
@@ -295,35 +306,8 @@ export function generateHarnessAsync(harnessPath, framework) {
295
306
  const config = loadConfig();
296
307
  const prompt = buildHarnessPrompt(harnessPath, framework);
297
308
  const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
298
- const timeoutMs = config.aiTimeoutMs ?? 300_000;
299
- return new Promise((resolve, reject) => {
300
- const child = spawn(program, args, { stdio: ['pipe', 'pipe', 'pipe'] });
301
- let out = '';
302
- let err = '';
303
- let done = false;
304
- const finish = (fn) => {
305
- if (done)
306
- return;
307
- done = true;
308
- clearTimeout(timer);
309
- fn();
310
- };
311
- // Without this, a harness call that hangs (flaky network, a stalled AI
312
- // CLI) never closes and the whole run wedges on step 1 forever — kill it
313
- // and surface a resumable error so the caller can retry.
314
- const timer = setTimeout(() => {
315
- child.kill('SIGKILL');
316
- finish(() => reject(new Error(`harness gen timed out after ${timeoutMs}ms — the AI CLI did not respond`)));
317
- }, timeoutMs);
318
- child.stdout.setEncoding('utf-8');
319
- child.stdout.on('data', (d) => (out += d));
320
- child.stderr.on('data', (d) => (err += d));
321
- child.on('error', (e) => finish(() => reject(e)));
322
- child.on('close', (code) => finish(() => code === 0
323
- ? resolve(cleanOutput(out))
324
- : reject(new Error(`harness gen exited ${code}: ${err.slice(0, 300)}`))));
325
- child.stdin.end(prompt);
326
- });
309
+ // Same runner as every other async AI call: one timeout guard, one usage count, one trace.
310
+ return runWithStdin(program, args, prompt, config.aiTimeoutMs ?? 300_000).then(cleanOutput);
327
311
  }
328
312
  /**
329
313
  * A minimal Playwright config so a webapp/ops area that currently uses