@activate-core/test-forge 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +98 -34
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +51 -4
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +51 -24
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +68 -5
  10. package/dist/core/docs/review.js +81 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +56 -0
  13. package/dist/core/gates/coverage.js +12 -19
  14. package/dist/core/gates/validate.js +277 -4
  15. package/dist/core/generate/ai-generator.js +243 -96
  16. package/dist/core/generate/ai-session.js +8 -0
  17. package/dist/core/generate/ai-usage.js +150 -0
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/defect-review.js +213 -20
  20. package/dist/core/generate/doc-diff.js +139 -0
  21. package/dist/core/generate/harness.js +52 -32
  22. package/dist/core/generate/journey-memory.js +325 -0
  23. package/dist/core/generate/plan-cache.js +114 -7
  24. package/dist/core/generate/required-data.js +75 -0
  25. package/dist/core/generate/review-journey.js +104 -0
  26. package/dist/core/generate/run-generate.js +193 -80
  27. package/dist/core/generate/seed-generator.js +2 -2
  28. package/dist/core/generate/suite-planner.js +138 -25
  29. package/dist/core/generate/test-blocks.js +84 -0
  30. package/dist/core/generate/update-journey.js +115 -0
  31. package/dist/core/generate/verify.js +9 -4
  32. package/dist/core/paths.js +90 -13
  33. package/dist/core/project/config.js +25 -2
  34. package/dist/core/project/env-diff.js +122 -0
  35. package/dist/core/project/env-template.js +3 -2
  36. package/dist/core/project/init.js +2 -2
  37. package/dist/core/project/map.js +2 -2
  38. package/dist/core/run/ci.js +94 -0
  39. package/dist/core/run/clean.js +111 -0
  40. package/dist/core/run/readiness.js +119 -0
  41. package/dist/core/run/report.js +2 -0
  42. package/dist/core/run/run-tests.js +23 -1
  43. package/package.json +1 -1
  44. package/reporters/_shared.cjs +24 -3
  45. package/reporters/jest-reporter.cjs +3 -1
  46. package/reporters/playwright-reporter.cjs +3 -1
  47. package/reporters/run-data.cjs +66 -0
  48. package/reporters/ui-helpers.cjs +133 -0
  49. package/dist/core/generate/verify-journey.js +0 -174
@@ -0,0 +1,91 @@
1
+ /**
2
+ * testforge — do the tests and the shipped code use the same names?
3
+ *
4
+ * `testforge bind` reports, per journey, which test files are ready to run
5
+ * (the code has every name and page they use), which test a feature not built
6
+ * yet, and which use a name the code spells differently. `--accept "A=B"`
7
+ * settles a naming difference in the code's favour: the doc and the journey's
8
+ * tests take the code's wording, and the next `generate` records it in the
9
+ * journey's memory as a rename. No AI.
10
+ */
11
+ import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
12
+ import { basename, extname, join, relative } from 'node:path';
13
+ import { findMarkdownFiles } from '../find-files.js';
14
+ import { notAJourneyDoc, parseJourney } from '../docs/parser.js';
15
+ import { loadConfig, testDirForType } from '../project/config.js';
16
+ import { matchesJourney } from './plan-cache.js';
17
+ import { sourceText } from '../docs/excerpts.js';
18
+ import { checkReadiness, shownTexts, testFilesIn } from '../run/readiness.js';
19
+ function journeys(docsDir, only) {
20
+ const config = loadConfig();
21
+ const out = [];
22
+ for (const file of findMarkdownFiles(docsDir)) {
23
+ const markdown = readFileSync(join(docsDir, file), 'utf-8');
24
+ if (notAJourneyDoc(file, markdown))
25
+ continue;
26
+ const area = /[\\/]/.test(file) ? file.split(/[\\/]/)[0] : undefined;
27
+ const journey = parseJourney(markdown, area, basename(file, extname(file)));
28
+ if (only && !matchesJourney(file, journey.id, only))
29
+ continue;
30
+ const dir = join(testDirForType(config, journey.area), journey.id);
31
+ if (existsSync(dir))
32
+ out.push({ id: journey.id, doc: join(docsDir, file), dir });
33
+ }
34
+ return out;
35
+ }
36
+ /** The raw source under the project's roots, for the texts the code shows. */
37
+ function rawSource(roots) {
38
+ const parts = [];
39
+ const walk = (d) => {
40
+ for (const e of existsSync(d) ? readdirSync(d, { withFileTypes: true }) : []) {
41
+ const p = join(d, e.name);
42
+ if (e.isDirectory()) {
43
+ if (!/^(node_modules|\.next|dist|build|vendor|tests?)$/.test(e.name))
44
+ walk(p);
45
+ }
46
+ else if (/\.(tsx?|jsx?|vue|svelte|php)$/.test(e.name) && !/\.(test|spec)\./.test(e.name))
47
+ parts.push(readFileSync(p, 'utf-8'));
48
+ }
49
+ };
50
+ for (const r of roots)
51
+ walk(r);
52
+ return parts.join('\n');
53
+ }
54
+ export function sourceRoots() {
55
+ return ['src', ...(loadConfig().relatedDirs ?? [])];
56
+ }
57
+ export function bindJourneys(opts = {}) {
58
+ const roots = sourceRoots();
59
+ const haystack = sourceText(roots);
60
+ let texts;
61
+ const texter = () => (texts ??= shownTexts(rawSource(roots)));
62
+ return journeys(opts.docsDir ?? 'docs', opts.journey).map((j) => ({
63
+ journey: j.id,
64
+ files: checkReadiness(testFilesIn(j.dir), haystack, texter).map((r) => ({ ...r, file: relative(process.cwd(), r.file) })),
65
+ }));
66
+ }
67
+ /**
68
+ * Settle naming differences in the code's favour: each "doc wording=code
69
+ * wording" pair is replaced in the journey docs that use it and in their
70
+ * tests. Returns what changed.
71
+ */
72
+ export function acceptNames(pairs, opts = {}) {
73
+ const notes = [];
74
+ for (const j of journeys(opts.docsDir ?? 'docs', opts.journey)) {
75
+ const files = [j.doc, ...testFilesIn(j.dir)];
76
+ let touched = 0;
77
+ for (const f of files) {
78
+ const before = readFileSync(f, 'utf-8');
79
+ let after = before;
80
+ for (const [from, to] of pairs)
81
+ after = after.split(from).join(to);
82
+ if (after !== before) {
83
+ writeFileSync(f, after, 'utf-8');
84
+ touched += 1;
85
+ }
86
+ }
87
+ if (touched)
88
+ notes.push(`${j.id}: ${touched} file(s) updated (doc and tests)`);
89
+ }
90
+ return notes;
91
+ }
@@ -16,12 +16,16 @@
16
16
  * what survives is reported as a defect; everything else is said for what
17
17
  * it is.
18
18
  */
19
- import { existsSync, readFileSync } from 'node:fs';
20
- import { relative } from 'node:path';
21
- import { loadConfig, splitCommand, withReadOnlyTools, withReferenceTools } from '../project/config.js';
22
- import { runWithStdin } from './ai-generator.js';
19
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
20
+ import { dirname, relative } from 'node:path';
21
+ import { loadConfig, splitCommand, withNoTools, withReferenceTools } from '../project/config.js';
22
+ import { readRules, ruleIndex } from './journey-memory.js';
23
+ import { sourceExcerpts } from '../docs/excerpts.js';
24
+ import { runWithStdin, DEFECT_MARK, FINDING_MARK } from './ai-generator.js';
23
25
  import { aiUnavailableReason, noteAiFailure, noteAiSuccess, waitForAi } from './ai-session.js';
24
- const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'unproven'];
26
+ const IMPACTS = ['blocker', 'major', 'minor'];
27
+ const asImpact = (v) => (IMPACTS.includes(v) ? v : undefined);
28
+ const VERDICTS = ['defect', 'finding', 'pending-decision', 'test-fault', 'environment', 'unproven'];
25
29
  /**
26
30
  * Group failing tests by the claim marked above them. A parametrised test
27
31
  * shares one declaration — and so one claim — across all its rows, and one
@@ -47,7 +51,7 @@ export function decisionsSource(spec) {
47
51
  return { text: readFileSync(spec, 'utf-8') };
48
52
  return { link: spec };
49
53
  }
50
- export function buildDefectReviewPrompt(journey, group, passes, decisions) {
54
+ export function buildDefectReviewPrompt(journey, group, passes, decisions, evidence) {
51
55
  const files = [...new Set(group.tests.map((t) => relative(process.cwd(), t.file)))];
52
56
  const sample = group.tests.slice(0, 3);
53
57
  return [
@@ -57,6 +61,10 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
57
61
  ``,
58
62
  `THE CLAIM (written by the tool that generated and repaired the test):`,
59
63
  ` ${group.claim}`,
64
+ ...(group.claim.includes(NOT_CLAIMED)
65
+ ? [` The generator did not claim a defect here: the claim is the test's own oracle.`,
66
+ ` Judge it the same way, and say "test-fault" or "environment" when that is what it is.`]
67
+ : []),
60
68
  ``,
61
69
  `FAILING TESTS that carry it (${group.tests.length}), numbered:`,
62
70
  ...group.tests.slice(0, MAX_LISTED).map((t, i) => ` ${i + 1}. ${relative(process.cwd(), t.file)}:${t.line} ${t.title}`),
@@ -73,8 +81,22 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
73
81
  ``,
74
82
  ]
75
83
  : []),
76
- `Read the test file(s) (${files.join(', ')}), the application source the test`,
77
- `exercises, and the server code behind it. You have read-only tools.`,
84
+ ...(evidence
85
+ ? [
86
+ `You have no tools. The failing tests' code and the application code the`,
87
+ `claim points at are below. When they do not show the cause, the verdict`,
88
+ `cannot be "defect" or "finding": say "unproven".`,
89
+ ``,
90
+ `--- TEST CODE ---`,
91
+ evidence.tests,
92
+ `--- APPLICATION CODE ---`,
93
+ evidence.app || '(the claim names no application file that could be found)',
94
+ `--- END CODE ---`,
95
+ ]
96
+ : [
97
+ `Read the test file(s) (${files.join(', ')}), the application source the test`,
98
+ `exercises, and the server code behind it. You have read-only tools.`,
99
+ ]),
78
100
  ``,
79
101
  `Decide exactly one verdict:`,
80
102
  `- "defect": ALL of these hold — (a) the criterion is stated in the journey`,
@@ -89,16 +111,30 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
89
111
  `- "pending-decision": the behaviour is covered by an open decision or a known`,
90
112
  ` issue in the list below — whatever the tests say.`,
91
113
  `- "test-fault": the test, its stub, its precondition or its claim is wrong.`,
114
+ `- "environment": the failure comes from the environment the test ran in — a`,
115
+ ` missing variable or account, an unreachable or refusing outside service, seed`,
116
+ ` data that is not there, a rate limit, or data left behind by earlier runs —`,
117
+ ` not from the application or the test's logic.`,
92
118
  `- "unproven": none of the above can be shown.`,
93
119
  ``,
120
+ `For a "defect" or "finding", also grade its impact on the customer:`,
121
+ `- "blocker": no way through, or unsafe — security or data exposure, money taken`,
122
+ ` wrongly, or a core step (log in, sign up, book, pay) that cannot be completed`,
123
+ ` at all. A failure the test forced (a stubbed error, a cut connection, absurd`,
124
+ ` input) is never a blocker on its own.`,
125
+ `- "major": the task can be done but the customer is misled or left without a`,
126
+ ` reason — a missing or wrong message, wrong data shown, an error on input.`,
127
+ `- "minor": presentation, or another way works — keyboard, focus, Escape,`,
128
+ ` wording and labels, icons, layout.`,
129
+ ``,
94
130
  ...(decisions.text
95
131
  ? [`OPEN DECISIONS AND KNOWN ISSUES (not requirements yet):`, decisions.text, ``]
96
132
  : decisions.link
97
133
  ? [`OPEN DECISIONS AND KNOWN ISSUES (not requirements yet) — read it: ${decisions.link}`, ``]
98
134
  : [`No list of open decisions was provided.`, ``]),
99
- `--- JOURNEY DOCUMENT ---`,
100
- journey.rawMarkdown,
101
- `--- END ---`,
135
+ ...(evidence
136
+ ? [`--- THE JOURNEY RULES THIS CLAIM IS ABOUT ---`, evidence.rules, `--- END ---`]
137
+ : [`--- JOURNEY DOCUMENT ---`, journey.rawMarkdown, `--- END ---`]),
102
138
  ``,
103
139
  `One claim often covers many rows of the same test (one input per row). Judge`,
104
140
  `every row: when some rows fall under a different verdict than the rest (one`,
@@ -106,10 +142,42 @@ export function buildDefectReviewPrompt(journey, group, passes, decisions) {
106
142
  `that fits most rows and list the others as exceptions, by number.`,
107
143
  ``,
108
144
  `Reply with ONLY this JSON object, nothing before or after it:`,
109
- `{"verdict": "defect|finding|pending-decision|test-fault|unproven", "reason": "<one or two sentences>", "evidence": "<app file:line, or empty>",`,
110
- ` "exceptions": [{"tests": [<row numbers>], "verdict": "...", "reason": "...", "evidence": "..."}]}`,
145
+ `{"verdict": "defect|finding|pending-decision|test-fault|environment|unproven", "reason": "<one or two sentences>", "evidence": "<app file:line, or empty>",`,
146
+ ` "impact": "blocker|major|minor (defect or finding only, else empty)",`,
147
+ ` "exceptions": [{"tests": [<row numbers>], "verdict": "...", "reason": "...", "evidence": "...", "impact": "..."}]}`,
111
148
  ].join('\n');
112
149
  }
150
+ /**
151
+ * The code a claim is judged on, collected without the AI: each failing
152
+ * test's own lines, the application lines the claim cites (and the files
153
+ * holding the texts it quotes), and the journey rules it names.
154
+ */
155
+ export function reviewEvidence(group, journeyMarkdown, roots) {
156
+ const tests = group.tests.slice(0, 3).map((t) => {
157
+ try {
158
+ const lines = readFileSync(t.file, 'utf-8').split('\n');
159
+ const from = Math.max(1, t.line - 10);
160
+ // The test body: from just above its declaration to the next test.
161
+ let to = Math.min(lines.length, t.line + 60);
162
+ for (let i = t.line; i < to; i++)
163
+ if (/^\s*test(\.\w+)?\(/.test(lines[i]) && i > t.line) {
164
+ to = i;
165
+ break;
166
+ }
167
+ return `### ${relative(process.cwd(), t.file)}\n${lines.slice(from - 1, to).map((l, i) => `${from + i}: ${l}`).join('\n')}`;
168
+ }
169
+ catch {
170
+ return `### ${relative(process.cwd(), t.file)} (unreadable)`;
171
+ }
172
+ });
173
+ const memory = readRules(journeyMarkdown);
174
+ const ids = new Set(group.claim.match(/\bR\d+\b/g) ?? []);
175
+ const named = memory.rules.filter((r) => ids.has(r.id));
176
+ const rules = named.length
177
+ ? named.map((r) => `${r.id} ${r.section}\n${r.text}`).join('\n\n')
178
+ : memory.rules.length ? ruleIndex(memory) : journeyMarkdown;
179
+ return { tests: [...new Set(tests)].join('\n\n'), app: sourceExcerpts(group.claim, roots, 15_000), rules };
180
+ }
113
181
  /** How many rows of one claim the reviewer is shown and may single out. */
114
182
  const MAX_LISTED = 40;
115
183
  const asVerdict = (v) => (VERDICTS.includes(v) ? v : 'unproven');
@@ -126,14 +194,20 @@ export function parseReview(raw) {
126
194
  verdict: asVerdict(e?.verdict),
127
195
  reason: text(e?.reason) ?? '',
128
196
  evidence: text(e?.evidence),
197
+ ...(asImpact(e?.impact) ? { impact: asImpact(e?.impact) } : {}),
129
198
  }))
130
199
  .filter((e) => e.tests.length);
131
- return { verdict: asVerdict(o.verdict), reason: text(o.reason) ?? '', evidence: text(o.evidence), exceptions };
200
+ const impact = asImpact(o.impact);
201
+ return { verdict: asVerdict(o.verdict), reason: text(o.reason) ?? '', evidence: text(o.evidence), ...(impact ? { impact } : {}), exceptions };
132
202
  }
133
203
  catch {
134
204
  return { verdict: 'unproven', reason: 'the review could not be read', exceptions: [] };
135
205
  }
136
206
  }
207
+ /** A defect or finding always carries an impact; it defaults to major when the reviewer gave none. Other verdicts carry none. */
208
+ function withImpact(r, impact) {
209
+ return r.verdict === 'defect' || r.verdict === 'finding' ? { ...r, impact: impact ?? 'major' } : r;
210
+ }
137
211
  /** One review per verdict: the group's own, and one for each set of exceptions. */
138
212
  export function splitReview(group, parsed) {
139
213
  const taken = new Set();
@@ -142,14 +216,102 @@ export function splitReview(group, parsed) {
142
216
  const tests = e.tests.filter((n) => n <= group.tests.length && !taken.has(n));
143
217
  tests.forEach((n) => taken.add(n));
144
218
  if (tests.length) {
145
- out.push({ claim: group.claim, tests: tests.map((n) => group.tests[n - 1]), verdict: e.verdict, reason: e.reason, evidence: e.evidence });
219
+ out.push(withImpact({ claim: group.claim, tests: tests.map((n) => group.tests[n - 1]), verdict: e.verdict, reason: e.reason, evidence: e.evidence }, e.impact));
146
220
  }
147
221
  }
148
222
  const rest = group.tests.filter((_, i) => !taken.has(i + 1));
149
223
  if (rest.length)
150
- out.unshift({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence });
224
+ out.unshift(withImpact({ claim: group.claim, tests: rest, verdict: parsed.verdict, reason: parsed.reason, evidence: parsed.evidence }, parsed.impact));
151
225
  return out;
152
226
  }
227
+ /** Marks a claim built from a test's oracle because the generator claimed nothing. */
228
+ export const NOT_CLAIMED = '(not claimed as a defect by the generator)';
229
+ /**
230
+ * The rule a failing test checks, read from its first "// oracle:" comment,
231
+ * so a failure nobody claimed can still be reviewed against its tier. Falls
232
+ * back to the test's title.
233
+ */
234
+ export function oracleClaim(t, content) {
235
+ const lines = content.split('\n');
236
+ for (let i = Math.max(0, t.line - 1), seen = 0; i < lines.length && seen < 200; i++, seen++) {
237
+ if (seen > 0 && /^\s*(?:test|it)(?:\.\w+)?\s*\(/.test(lines[i]))
238
+ break;
239
+ const m = /\/\/\s*oracle:\s*(.+)$/.exec(lines[i]);
240
+ if (m)
241
+ return `${m[1].trim()} ${NOT_CLAIMED}`;
242
+ }
243
+ return `${t.title} ${NOT_CLAIMED}`;
244
+ }
245
+ /** A [spec] line that carries the client's words, quoted or pointing at the quote above it ("source as above"). */
246
+ export const HAS_SOURCE = /\((?:source(?::\s*(?:[A-Z][A-Z0-9]+-\d+:?\s*)?["“]|:?\s*(?:as|same)\b)|same source\b)/i;
247
+ /** Playwright's report of a locator that matched more than one element. */
248
+ const AMBIGUOUS = /strict mode violation|resolved to \d+ elements/i;
249
+ /** An application file:line, the only evidence a defect may rest on. */
250
+ const CODE_REF = /[\w@./-]+\.(?:tsx?|jsx?|mjs|cjs|php|py|rb|go|java|kt|cs|vue|svelte)(?::\d+)/i;
251
+ const alnum = (v) => v.toLowerCase().replace(/\\./g, '').replace(/[^a-z0-9]+/g, '');
252
+ /** The tier a claim is made against, read from its tag. */
253
+ export function claimTier(claim) {
254
+ return /\[(spec|baseline|observed)\]/i.exec(claim)?.[1]?.toLowerCase();
255
+ }
256
+ /** The journey line a [spec] claim quotes, if the doc has one. */
257
+ export function specLineFor(claim, journeyMarkdown) {
258
+ // Claims often shorten the rule with "..." anywhere in it: match on its longest unshortened piece.
259
+ const rule = claim.slice(claim.search(/\[spec\]/i) + '[spec]'.length).split(/\s[—-]\s/)[0];
260
+ const piece = rule.split(/\.\.\.|…/).map(alnum).sort((a, b) => b.length - a.length)[0] ?? '';
261
+ const key = piece.slice(0, 40);
262
+ if (key.length < 12)
263
+ return undefined;
264
+ return journeyMarkdown.split('\n').find((l) => /\[spec\]/i.test(l) && alnum(l).includes(key));
265
+ }
266
+ /**
267
+ * The reviewer's word is not the last one. A "defect" stands only when the
268
+ * claim is a [spec] rule the client wrote, the journey line carries its
269
+ * (source: "…") quote, and the reviewer names the application file:line.
270
+ * Without the code reference it is unproven; against a [baseline]/[observed]
271
+ * rule, or a [spec] line with no quoted source, it is at most a finding.
272
+ */
273
+ export function gradeReview(r, journeyMarkdown) {
274
+ // A locator that matched several elements never reached one behaviour: that is the test's fault.
275
+ if ((r.verdict === 'defect' || r.verdict === 'finding') && r.tests.some((t) => AMBIGUOUS.test(t.detail ?? ''))) {
276
+ return { ...r, verdict: 'test-fault', impact: undefined, reason: `${r.reason} [graded: the test's locator matched several elements (strict mode violation)]`.trim() };
277
+ }
278
+ if (r.verdict !== 'defect')
279
+ return r;
280
+ if (!r.evidence || !CODE_REF.test(r.evidence)) {
281
+ return { ...r, verdict: 'unproven', impact: undefined, reason: `${r.reason} [graded: no application file:line was given as evidence]`.trim() };
282
+ }
283
+ const tier = claimTier(r.claim);
284
+ if (tier !== 'spec') {
285
+ return { ...r, verdict: 'finding', reason: `${r.reason} [graded: the rule is ${tier ? `[${tier}]` : 'untagged'}, not a client requirement]`.trim() };
286
+ }
287
+ const line = specLineFor(r.claim, journeyMarkdown);
288
+ if (!line || !HAS_SOURCE.test(line)) {
289
+ return { ...r, verdict: 'finding', reason: `${r.reason} [graded: the journey doc has no quoted source for this [spec] rule]`.trim() };
290
+ }
291
+ return r;
292
+ }
293
+ /**
294
+ * A repair may claim a defect only against a [spec] rule the journey quotes
295
+ * with its source. Any other claim — [baseline], [observed], untagged, or a
296
+ * [spec] rule the doc does not quote — is rewritten to a finding mark: the
297
+ * test stays failing and protected from repair, but is never reported as a
298
+ * defect.
299
+ */
300
+ export function normaliseClaims(content, journeyMarkdown) {
301
+ let downgraded = 0;
302
+ const lines = content.split('\n').map((line) => {
303
+ const at = line.indexOf(DEFECT_MARK);
304
+ if (at === -1)
305
+ return line;
306
+ const claim = line.slice(at + DEFECT_MARK.length).trim();
307
+ const spec = claimTier(claim) === 'spec' ? specLineFor(claim, journeyMarkdown) : undefined;
308
+ if (spec && HAS_SOURCE.test(spec))
309
+ return line;
310
+ downgraded += 1;
311
+ return line.slice(0, at) + FINDING_MARK + line.slice(at + DEFECT_MARK.length);
312
+ });
313
+ return { content: lines.join('\n'), downgraded };
314
+ }
153
315
  /**
154
316
  * Review every claim group, a few at a time. Never throws.
155
317
  *
@@ -160,12 +322,34 @@ export function splitReview(group, parsed) {
160
322
  * reported 0 defects. So the group goes back on the queue, every worker
161
323
  * shares one wait, and the review is asked again once the AI answers.
162
324
  */
163
- export async function reviewDefects(journey, groups, passes, io) {
325
+ export async function reviewDefects(journey, groups, passes, io,
326
+ /** Where answered groups are kept, so an interrupted review asks only the rest again. */
327
+ checkpoint) {
328
+ const groupKey = (g) => `${g.claim}\u0000${g.tests.map((t) => `${t.file}:${t.line}:${t.title}`).join('|')}`;
329
+ let answered = {};
330
+ if (checkpoint) {
331
+ try {
332
+ answered = JSON.parse(readFileSync(checkpoint, 'utf-8'));
333
+ }
334
+ catch {
335
+ answered = {};
336
+ }
337
+ }
338
+ const keep = (g, rs) => {
339
+ if (!checkpoint)
340
+ return;
341
+ answered[groupKey(g)] = rs;
342
+ mkdirSync(dirname(checkpoint), { recursive: true });
343
+ writeFileSync(checkpoint, JSON.stringify(answered), 'utf-8');
344
+ };
164
345
  const config = loadConfig();
165
346
  const decisions = decisionsSource(config.decisions);
166
347
  // A linked list (an internal Notion page) needs the reference tools to be read.
167
- const tools = decisions.link ? withReferenceTools : withReadOnlyTools;
348
+ // Everything else is handed over as evidence, so the reviewer needs no tools.
349
+ const noTools = !decisions.link;
350
+ const tools = decisions.link ? withReferenceTools : withNoTools;
168
351
  const { program, args } = tools(splitCommand(config.aiCommand));
352
+ const roots = ['src', ...(config.relatedDirs ?? [])];
169
353
  const ask = io?.ask ?? ((prompt) => runWithStdin(program, args, prompt, config.aiTimeoutMs));
170
354
  const wait = io?.wait ?? (() => ((config.waitForLimit ?? true) ? waitForAi(config) : Promise.resolve(false)));
171
355
  const out = [];
@@ -174,11 +358,16 @@ export async function reviewDefects(journey, groups, passes, io) {
174
358
  let gaveUp = false; // one failed wait is enough; the rest are recorded as unproven
175
359
  const worker = async () => {
176
360
  for (let g = queue.shift(); g; g = queue.shift()) {
361
+ const earlier = answered[groupKey(g)];
362
+ if (earlier) {
363
+ out.push(...earlier);
364
+ continue;
365
+ }
177
366
  const files = new Set(g.tests.map((t) => t.file));
178
367
  const siblings = passes.filter((p) => files.has(p.file));
179
368
  let raw = '';
180
369
  try {
181
- raw = await ask(buildDefectReviewPrompt(journey, g, siblings, decisions));
370
+ raw = await ask(buildDefectReviewPrompt(journey, g, siblings, decisions, noTools ? reviewEvidence(g, journey.rawMarkdown, roots) : undefined));
182
371
  noteAiSuccess();
183
372
  }
184
373
  catch (e) {
@@ -193,7 +382,11 @@ export async function reviewDefects(journey, groups, passes, io) {
193
382
  }
194
383
  // an unanswered review proves nothing
195
384
  }
196
- out.push(...splitReview(g, parseReview(raw)));
385
+ const rs = splitReview(g, parseReview(raw)).map((r) => gradeReview(r, journey.rawMarkdown));
386
+ // An unanswered group (the AI never replied) is not kept: it is asked again next time.
387
+ if (raw)
388
+ keep(g, rs);
389
+ out.push(...rs);
197
390
  }
198
391
  };
199
392
  await Promise.all(Array.from({ length: Math.max(1, Math.min(config.aiConcurrency ?? 4, groups.length)) }, worker));
@@ -0,0 +1,139 @@
1
+ /**
2
+ * testforge — what changed in a journey doc, rule by rule.
3
+ *
4
+ * The journey's rules from the last run are kept in its memory. A new version
5
+ * of the doc is matched against them without the AI, so a doc that gained one
6
+ * feature or had one label renamed costs only that rule, never the whole doc.
7
+ * Rule ids stay stable across versions: a rule keeps its id while it exists,
8
+ * and a new rule gets the next free one, so the `// oracle: R12` comments in
9
+ * the tests keep pointing at the right rule.
10
+ */
11
+ import { existsSync, readFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ import { journeyDir } from '../paths.js';
14
+ export function memoryPath(journey) {
15
+ return join(journeyDir(journey), 'rules.json');
16
+ }
17
+ /** The journey as the last run saw it, or undefined on a first run. */
18
+ export function loadMemory(journey) {
19
+ const p = memoryPath(journey);
20
+ if (!existsSync(p))
21
+ return undefined;
22
+ try {
23
+ const m = JSON.parse(readFileSync(p, 'utf-8'));
24
+ return Array.isArray(m.rules) ? m : undefined;
25
+ }
26
+ catch {
27
+ return undefined;
28
+ }
29
+ }
30
+ const norm = (t) => t.replace(/\s+/g, ' ').trim();
31
+ const QUOTED = /"([^"\n]+)"|`([^`\n]+)`|'([^'\n]{2,})'/g;
32
+ const quotes = (t) => [...t.matchAll(QUOTED)].map((m) => m[1] ?? m[2] ?? m[3]);
33
+ const unquoted = (t) => norm(t.replace(QUOTED, '""'));
34
+ const words = (t) => new Set(norm(t).toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 2));
35
+ function similarity(a, b) {
36
+ const x = words(a);
37
+ const y = words(b);
38
+ if (!x.size || !y.size)
39
+ return 0;
40
+ let both = 0;
41
+ for (const w of x)
42
+ if (y.has(w))
43
+ both += 1;
44
+ return both / (x.size + y.size - both);
45
+ }
46
+ /**
47
+ * Give the new doc's rules the ids they had last time and say what happened
48
+ * to each. Exact text keeps its id; the same text with different quoted
49
+ * strings is a rename; a close match is a change; the rest is added or
50
+ * removed. Pure: no AI, no I/O.
51
+ */
52
+ export function alignRules(prev, next) {
53
+ const diff = { same: [], changed: [], renamed: [], added: [], removed: [] };
54
+ const free = new Set(prev.map((r) => r.id));
55
+ const byText = new Map();
56
+ for (const r of prev)
57
+ byText.set(norm(r.text), [...(byText.get(norm(r.text)) ?? []), r]);
58
+ const id = new Map();
59
+ // 1. Unchanged wording keeps its id.
60
+ for (const r of next.rules) {
61
+ const match = (byText.get(norm(r.text)) ?? []).find((p) => free.has(p.id));
62
+ if (!match)
63
+ continue;
64
+ id.set(r, match.id);
65
+ free.delete(match.id);
66
+ diff.same.push(match.id);
67
+ }
68
+ // 2. Same wording around different quoted texts: a rename.
69
+ for (const r of next.rules) {
70
+ if (id.has(r))
71
+ continue;
72
+ const match = prev.find((p) => free.has(p.id) && p.tag === r.tag && unquoted(p.text) === unquoted(r.text));
73
+ if (!match)
74
+ continue;
75
+ const before = quotes(match.text);
76
+ const after = quotes(r.text);
77
+ id.set(r, match.id);
78
+ free.delete(match.id);
79
+ const pairs = before.map((b, i) => [b, after[i]]).filter(([b, a]) => a !== undefined && a !== b);
80
+ diff.renamed.push({ id: match.id, pairs: [...new Map(pairs.map((p) => [`${p[0]}\u0000${p[1]}`, p])).values()] });
81
+ }
82
+ // 2b. A wording renamed in one rule's quotes ("Assign" → "Choose") is the same
83
+ // rename in a rule that writes it without quotes: same id, a rename, no AI.
84
+ const known = [...new Map(diff.renamed.flatMap((r) => r.pairs).map((p) => [`${p[0]}\u0000${p[1]}`, p])).values()];
85
+ if (known.length) {
86
+ const apply = (t, pairs) => pairs.reduce((acc, [a, b]) => acc.split(a).join(b), t);
87
+ for (const r of next.rules) {
88
+ if (id.has(r))
89
+ continue;
90
+ const match = prev.find((p) => free.has(p.id) && p.tag === r.tag && norm(apply(p.text, known)) === norm(r.text));
91
+ if (!match)
92
+ continue;
93
+ id.set(r, match.id);
94
+ free.delete(match.id);
95
+ diff.renamed.push({ id: match.id, pairs: known.filter(([a]) => match.text.includes(a)) });
96
+ }
97
+ }
98
+ // 3. A close match, preferring the same section: the rule changed.
99
+ for (const r of next.rules) {
100
+ if (id.has(r))
101
+ continue;
102
+ let best;
103
+ let score = 0;
104
+ for (const p of prev) {
105
+ if (!free.has(p.id) || p.tag !== r.tag)
106
+ continue;
107
+ const s = similarity(p.text, r.text) + (p.section === r.section ? 0.1 : 0);
108
+ if (s > score) {
109
+ best = p;
110
+ score = s;
111
+ }
112
+ }
113
+ if (!best || score < 0.6)
114
+ continue;
115
+ id.set(r, best.id);
116
+ free.delete(best.id);
117
+ diff.changed.push(best.id);
118
+ }
119
+ // 4. Everything else is new, numbered after the highest id ever used.
120
+ let top = Math.max(0, ...prev.map((r) => Number(r.id.slice(1)) || 0));
121
+ for (const r of next.rules) {
122
+ if (id.has(r))
123
+ continue;
124
+ const fresh = `R${++top}`;
125
+ id.set(r, fresh);
126
+ diff.added.push(fresh);
127
+ }
128
+ diff.removed = [...free];
129
+ return { memory: { ...next, rules: next.rules.map((r) => ({ ...r, id: id.get(r) })) }, diff };
130
+ }
131
+ export function describeDiff(d) {
132
+ return [
133
+ `${d.same.length} same`,
134
+ d.renamed.length ? `${d.renamed.length} renamed` : '',
135
+ d.changed.length ? `${d.changed.length} changed` : '',
136
+ d.added.length ? `${d.added.length} added` : '',
137
+ d.removed.length ? `${d.removed.length} removed` : '',
138
+ ].filter(Boolean).join(', ');
139
+ }