@activate-core/test-forge 2.4.2 → 2.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -55,7 +55,7 @@ export function buildDocCheckPrompt(doc, reference, excerpts, gaps, otherDocs =
55
55
  excerpts || '(none found)',
56
56
  `</source_excerpts>`,
57
57
  `<draft>`,
58
- doc,
58
+ withRuleIds(doc),
59
59
  `</draft>`,
60
60
  ].join('\n');
61
61
  }
@@ -72,6 +72,23 @@ export function parseFixes(out) {
72
72
  }
73
73
  const TAGGED = /\[(spec|baseline|observed)\]/i;
74
74
  const norm = (t) => t.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
75
+ /**
76
+ * The doc as the check sees it: every criterion starts with its id, also in a
77
+ * doc written before ids were part of the format, so a fix can name the rule
78
+ * it is about. The file itself is not changed.
79
+ */
80
+ export function withRuleIds(markdown) {
81
+ const memory = readRules(markdown);
82
+ const lines = markdown.split('\n');
83
+ const where = ruleLines(lines, memory);
84
+ for (const r of memory.rules) {
85
+ const i = where.get(r.id);
86
+ if (i === undefined || r.explicit)
87
+ continue;
88
+ lines[i] = lines[i].replace(/^(\s*(?:[-*]|\d+(?:\.\d+)*\.?)\s+)/, `$1${r.id} `);
89
+ }
90
+ return lines.join('\n');
91
+ }
75
92
  /** Line index of each rule's first line, by id. */
76
93
  function ruleLines(lines, memory) {
77
94
  const at = new Map();
@@ -122,11 +139,25 @@ export function applyFixes(markdown, fixes) {
122
139
  const where = ruleLines(lines, memory);
123
140
  const byRule = fixes.filter((f) => f.action !== 'add');
124
141
  const removed = new Set();
142
+ // A fix names its rule by id; one that quotes the rule's words instead is matched by them.
143
+ const resolve = (ref) => {
144
+ const t = ref.trim();
145
+ const id = /^R\d+$/i.test(t) ? t.toUpperCase() : (/^R\d+\b/i.exec(t)?.[0]?.toUpperCase());
146
+ if (id && ids.has(id))
147
+ return id;
148
+ const want = norm(t.replace(/^R\d+\s*/i, ''));
149
+ if (want.length < 15)
150
+ return undefined;
151
+ const hit = memory.rules.find((r) => r.tag && (norm(r.text.split('\n')[0]).startsWith(want) || want.startsWith(norm(r.text.split('\n')[0]))));
152
+ return hit?.id;
153
+ };
154
+ for (const f of byRule)
155
+ f.rule = resolve(f.rule ?? '') ?? f.rule;
125
156
  for (const f of byRule.sort((a, b) => (where.get(b.rule ?? '') ?? -1) - (where.get(a.rule ?? '') ?? -1))) {
126
- const id = (f.rule ?? '').trim().toUpperCase();
157
+ const id = (f.rule ?? '').trim();
127
158
  const i = where.get(id);
128
159
  if (!id || !ids.has(id) || i === undefined || removed.has(id)) {
129
- refused.push(`${f.action} ${id || '(no rule)'}: the doc has no such rule`);
160
+ refused.push(`${f.action} ${id.length > 40 ? `"${id.slice(0, 40)}…"` : id || '(no rule)'}: the doc has no such rule`);
130
161
  continue;
131
162
  }
132
163
  if (f.action === 'remove') {
@@ -223,10 +254,15 @@ export function withCoverage(markdown) {
223
254
  lines.splice(before >= 0 ? before : lines.length, 0, ...table);
224
255
  return lines.join('\n').replace(/\n{3,}/g, '\n\n');
225
256
  }
226
- /** The quoted words of a [spec] line's source, one entry per quote. */
257
+ /** Markdown links read as their text: "[a](a)" is "a" once. */
258
+ const flatLinks = (t) => t.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1');
259
+ /** The quoted words of a [spec] line's source, one entry per quote ("…" and \"…\" inside are kept). */
227
260
  function quotesOf(line) {
228
- const src = /\(source:[^"“]*["“]([\s\S]*?)["”]\s*\)/i.exec(line);
229
- return src ? [src[1]] : [];
261
+ const at = line.search(/\(source:/i);
262
+ if (at < 0)
263
+ return [];
264
+ const src = line.slice(at);
265
+ return [...src.matchAll(/["“]((?:\\.|[^"“”\\])*)["”]/g)].map((m) => flatLinks(m[1].replace(/\\"/g, '"')));
230
266
  }
231
267
  /**
232
268
  * A [spec] line whose quote is not in the reference is the drafter's reading,
@@ -234,7 +270,7 @@ function quotesOf(line) {
234
270
  * is checked piece by piece. Without a reference text nothing is checked.
235
271
  */
236
272
  export function verifySpecQuotes(markdown, reference) {
237
- const ref = ` ${norm(reference)} `;
273
+ const ref = ` ${norm(flatLinks(reference))} `;
238
274
  const retagged = [];
239
275
  if (!reference.trim())
240
276
  return { markdown, retagged };
@@ -250,7 +286,7 @@ export function verifySpecQuotes(markdown, reference) {
250
286
  const pieces = quotes.flatMap((q) => q.split(/…|\.\.\./)).map(norm).filter((p) => p.split(' ').length >= 3);
251
287
  if (pieces.every((p) => ref.includes(` ${p} `)))
252
288
  return line;
253
- retagged.push(/R\d+/.exec(line)?.[0] ?? line.trim().slice(0, 40));
289
+ retagged.push(/^\s*[-*]\s+(R\d+)\b/.exec(line)?.[1] ?? `"${line.replace(/^\s*[-*]\s+/, '').slice(0, 50)}…"`);
254
290
  return line.replace(/\[spec\]/gi, '[observed]');
255
291
  });
256
292
  return { markdown: out.join('\n'), retagged };
@@ -148,6 +148,8 @@ const regexWords = (src) => src.split('|').map((a) => a.replace(/\\[sbdw]/g, ' '
148
148
  */
149
149
  export function testProblems(content, opts = {}) {
150
150
  const out = [];
151
+ // Names the file sets once, outside its tests: a written-out text, or need('…').
152
+ const fileNames = new Set([...content.matchAll(/\b(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:need\(\s*['"`]|['"`])/g)].map((m) => m[1]));
151
153
  for (const block of findTestBlocks(content)) {
152
154
  const text = content.slice(block.start, block.end);
153
155
  const oracle = (/\/\/\s*oracle:\s*(.+)$/m.exec(text)?.[1] ?? '').toLowerCase();
@@ -194,7 +196,8 @@ export function testProblems(content, opts = {}) {
194
196
  continue;
195
197
  const v = /^([A-Za-z_$][\w$]*)\s*[,)]/.exec(rest)?.[1];
196
198
  // A name from a list written out in the test (a parametrised loop, a const) is fine.
197
- const writtenOut = v && (new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
199
+ // …or a name the file sets once at its top, written out or asked for with need().
200
+ const writtenOut = v && (fileNames.has(v) || new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
198
201
  new RegExp(`for\\s*\\(\\s*(?:const|let)\\s+(?:${v}|\\[[^\\]]*\\b${v}\\b[^\\]]*\\])\\s+of\\s+\\[\\s*['"\`\\[]`).test(code));
199
202
  if (/^[A-Za-z_$]/.test(rest) && !writtenOut) {
200
203
  add("finds an element by a name held in a variable: take the name the screen shows from the CODE FACTS or the doc, written out — a value the test made up (kit.unique, a stub's id) is never a control's name");
@@ -15,6 +15,18 @@ const review = (content, rules, steps, knownTexts) => ({
15
15
  cov: ruleCoverage(content, rules),
16
16
  problems: testProblems(content, { steps, knownTexts }),
17
17
  });
18
+ /** A problem's reason without its advice: the part before the first colon or dash. */
19
+ export function shortReason(why) {
20
+ return why.split(/:\s|\s—\s/)[0].trim().slice(0, 90);
21
+ }
22
+ /**
23
+ * The file without the names it set at its top that no test uses any more,
24
+ * once their tests were taken out: `const X = need('…')` left behind is a
25
+ * lint error, and asks the product for a name nothing needs.
26
+ */
27
+ export function withoutNames(content) {
28
+ return content.replace(/^[ \t]*(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])[^'"`\n]*\2\s*\);?[ \t]*\n/gm, (line, name) => new RegExp(`\\b${name}\\b`).test(content.replace(line, '')) ? line : '');
29
+ }
18
30
  /** The file without the tests that have a problem. */
19
31
  function withoutBlocks(content, problems) {
20
32
  const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
@@ -30,11 +42,13 @@ export async function completeFile(content, ctx) {
30
42
  const first = review(content, ctx.rules, ctx.steps, ctx.knownTexts);
31
43
  let calls = 0;
32
44
  let out = content;
33
- const dropped = new Set();
45
+ const dropped = new Map();
46
+ const drop = (ps) => { for (const p of ps)
47
+ if (!dropped.has(p.title))
48
+ dropped.set(p.title, p.reason); };
34
49
  if (first.cov.missing.length || first.problems.length) {
35
- out = withoutBlocks(content, first.problems);
36
- for (const p of first.problems)
37
- dropped.add(p.title);
50
+ out = withoutNames(withoutBlocks(content, first.problems));
51
+ drop(first.problems);
38
52
  const left = ruleCoverage(out, ctx.rules).missing;
39
53
  if (left.length) {
40
54
  const blocks = findTestBlocks(out);
@@ -57,9 +71,8 @@ export async function completeFile(content, ctx) {
57
71
  // What the second answer still gets wrong is removed, not sent again.
58
72
  const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts });
59
73
  if (second.length) {
60
- out = withoutBlocks(out, second);
61
- for (const p of second)
62
- dropped.add(p.title);
74
+ out = withoutNames(withoutBlocks(out, second));
75
+ drop(second);
63
76
  }
64
77
  }
65
78
  }
@@ -69,9 +82,16 @@ export async function completeFile(content, ctx) {
69
82
  out = out.replace(/^[ \t]*\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)\n/gm, (line, ids, why) => ids.split(/\s*,\s*/).every((id) => has.has(id)) || !acceptedReason(why) ? '' : line);
70
83
  const cov = ruleCoverage(out, ctx.rules);
71
84
  const tested = [...has].filter((id) => ctx.rules.includes(id));
72
- if (dropped.size)
73
- notes.push(`took out ${dropped.size} test(s) that could not check their rule`);
85
+ if (dropped.size) {
86
+ // The reasons, counted, so the next fix goes where most tests were lost.
87
+ const by = new Map();
88
+ for (const why of dropped.values()) {
89
+ const k = shortReason(why);
90
+ by.set(k, (by.get(k) ?? 0) + 1);
91
+ }
92
+ notes.push(`took out ${dropped.size} test(s) that could not check their rule (${[...by].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${n} × ${k}`).join('; ')})`);
93
+ }
74
94
  if (cov.missing.length)
75
95
  notes.push(`no test for ${cov.missing.join(', ')}`);
76
- return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped], calls, notes };
96
+ return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes };
77
97
  }
@@ -389,6 +389,7 @@ export function recordWritten(journey, files) {
389
389
  const detail = [
390
390
  ...[...why].map(([id, w]) => `- ${id}: ${w}`),
391
391
  ...[...missing].map((id) => `- ${id}: no test was written for it`),
392
+ ...(dropped ? ['', '### Tests taken out (title — why)', ...files.flatMap((f) => f.dropped.map((d) => `- ${f.slug}: ${d}`))] : []),
392
393
  ];
393
394
  writeFileSync(check, `${before.trimEnd()}\n\n## Written\n\n${lines.join('; ')}.\n${detail.length ? `\n${detail.join('\n')}\n` : ''}`, 'utf-8');
394
395
  }
@@ -763,7 +763,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
763
763
  if (asked.length) {
764
764
  const blank = recordInterface(Object.fromEntries(asked.map((n) => [n, undefined])));
765
765
  if (blank.length) {
766
- warnings.push(`${blank.length} name(s) the product has not given yet — fill them in ${INTERFACE_PATH}; until then \`testforge run\` holds those files back: ${blank.slice(0, 8).join(', ')}${blank.length > 8 ? ', …' : ''}`);
766
+ warnings.push(`${blank.length} name(s) the product has not given yet — \`testforge run\` looks for them in the code before each run, and their tests report "waiting for the product" until they are there (${INTERFACE_PATH}): ${blank.slice(0, 8).join(', ')}${blank.length > 8 ? ', …' : ''}`);
767
767
  }
768
768
  }
769
769
  const testFiles = generated.map((g) => ({
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.4.2",
3
+ "version": "2.4.4",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {