@activate-core/test-forge 2.4.3 → 2.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -451,8 +451,9 @@ words. Nothing found, or the same lines as last time: no AI call. New lines:
451
451
  one small call without tools picks each value, kept only if the code really
452
452
  has it, and printed with what was found. Tests, docs and plans are never
453
453
  touched. A test whose name is still blank is reported as "waiting for the
454
- product", not as a defect — also when the name is asked for at the top of the
455
- file, outside any test. `--no-scan` skips the look-up.
454
+ product", not as a defect. A name asked for once at the top of a file holds back
455
+ only the tests that use it (all of them when a shared hook uses it); the file's
456
+ other tests run. `--no-scan` skips the look-up.
456
457
 
457
458
  **Before the run, the testkit's packages.** A testkit lists what it needs
458
459
  (`REQUIRED_PACKAGES`, a database client say). When the project lacks one,
@@ -148,6 +148,8 @@ const regexWords = (src) => src.split('|').map((a) => a.replace(/\\[sbdw]/g, ' '
148
148
  */
149
149
  export function testProblems(content, opts = {}) {
150
150
  const out = [];
151
+ // Names the file sets once, outside its tests: a written-out text, or need('…').
152
+ const fileNames = new Set([...content.matchAll(/\b(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:need\(\s*['"`]|['"`])/g)].map((m) => m[1]));
151
153
  for (const block of findTestBlocks(content)) {
152
154
  const text = content.slice(block.start, block.end);
153
155
  const oracle = (/\/\/\s*oracle:\s*(.+)$/m.exec(text)?.[1] ?? '').toLowerCase();
@@ -194,7 +196,8 @@ export function testProblems(content, opts = {}) {
194
196
  continue;
195
197
  const v = /^([A-Za-z_$][\w$]*)\s*[,)]/.exec(rest)?.[1];
196
198
  // A name from a list written out in the test (a parametrised loop, a const) is fine.
197
- const writtenOut = v && (new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
199
+ // …or a name the file sets once at its top, written out or asked for with need().
200
+ const writtenOut = v && (fileNames.has(v) || new RegExp(`\\b(?:const|let)\\s+${v}\\s*=\\s*['"\`]`).test(code) ||
198
201
  new RegExp(`for\\s*\\(\\s*(?:const|let)\\s+(?:${v}|\\[[^\\]]*\\b${v}\\b[^\\]]*\\])\\s+of\\s+\\[\\s*['"\`\\[]`).test(code));
199
202
  if (/^[A-Za-z_$]/.test(rest) && !writtenOut) {
200
203
  add("finds an element by a name held in a variable: take the name the screen shows from the CODE FACTS or the doc, written out — a value the test made up (kit.unique, a stub's id) is never a control's name");
@@ -15,6 +15,18 @@ const review = (content, rules, steps, knownTexts) => ({
15
15
  cov: ruleCoverage(content, rules),
16
16
  problems: testProblems(content, { steps, knownTexts }),
17
17
  });
18
+ /** A problem's reason without its advice: the part before the first colon or dash. */
19
+ export function shortReason(why) {
20
+ return why.split(/:\s|\s—\s/)[0].trim().slice(0, 90);
21
+ }
22
+ /**
23
+ * The file without the names it set at its top that no test uses any more,
24
+ * once their tests were taken out: `const X = need('…')` left behind is a
25
+ * lint error, and asks the product for a name nothing needs.
26
+ */
27
+ export function withoutNames(content) {
28
+ return content.replace(/^[ \t]*(?:const|let)\s+([A-Za-z_$][\w$]*)\s*=\s*need\(\s*(['"`])[^'"`\n]*\2\s*\);?[ \t]*\n/gm, (line, name) => new RegExp(`\\b${name}\\b`).test(content.replace(line, '')) ? line : '');
29
+ }
18
30
  /** The file without the tests that have a problem. */
19
31
  function withoutBlocks(content, problems) {
20
32
  const blocks = [...new Map(problems.map((p) => [p.block.start, p.block])).values()];
@@ -30,11 +42,13 @@ export async function completeFile(content, ctx) {
30
42
  const first = review(content, ctx.rules, ctx.steps, ctx.knownTexts);
31
43
  let calls = 0;
32
44
  let out = content;
33
- const dropped = new Set();
45
+ const dropped = new Map();
46
+ const drop = (ps) => { for (const p of ps)
47
+ if (!dropped.has(p.title))
48
+ dropped.set(p.title, p.reason); };
34
49
  if (first.cov.missing.length || first.problems.length) {
35
- out = withoutBlocks(content, first.problems);
36
- for (const p of first.problems)
37
- dropped.add(p.title);
50
+ out = withoutNames(withoutBlocks(content, first.problems));
51
+ drop(first.problems);
38
52
  const left = ruleCoverage(out, ctx.rules).missing;
39
53
  if (left.length) {
40
54
  const blocks = findTestBlocks(out);
@@ -57,9 +71,8 @@ export async function completeFile(content, ctx) {
57
71
  // What the second answer still gets wrong is removed, not sent again.
58
72
  const second = testProblems(out, { steps: ctx.steps, knownTexts: ctx.knownTexts });
59
73
  if (second.length) {
60
- out = withoutBlocks(out, second);
61
- for (const p of second)
62
- dropped.add(p.title);
74
+ out = withoutNames(withoutBlocks(out, second));
75
+ drop(second);
63
76
  }
64
77
  }
65
78
  }
@@ -69,9 +82,16 @@ export async function completeFile(content, ctx) {
69
82
  out = out.replace(/^[ \t]*\/\/\s*cannot-test:\s*(R\d+(?:\s*,\s*R\d+)*)\s*[—–-]+\s*(.+)\n/gm, (line, ids, why) => ids.split(/\s*,\s*/).every((id) => has.has(id)) || !acceptedReason(why) ? '' : line);
70
83
  const cov = ruleCoverage(out, ctx.rules);
71
84
  const tested = [...has].filter((id) => ctx.rules.includes(id));
72
- if (dropped.size)
73
- notes.push(`took out ${dropped.size} test(s) that could not check their rule`);
85
+ if (dropped.size) {
86
+ // The reasons, counted, so the next fix goes where most tests were lost.
87
+ const by = new Map();
88
+ for (const why of dropped.values()) {
89
+ const k = shortReason(why);
90
+ by.set(k, (by.get(k) ?? 0) + 1);
91
+ }
92
+ notes.push(`took out ${dropped.size} test(s) that could not check their rule (${[...by].sort((a, b) => b[1] - a[1]).map(([k, n]) => `${n} × ${k}`).join('; ')})`);
93
+ }
74
94
  if (cov.missing.length)
75
95
  notes.push(`no test for ${cov.missing.join(', ')}`);
76
- return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped], calls, notes };
96
+ return { content: out, tested, cannotTest: cov.cannotTest, missing: cov.missing, dropped: [...dropped].map(([t, why]) => `${t} — ${why}`), calls, notes };
77
97
  }
@@ -389,6 +389,7 @@ export function recordWritten(journey, files) {
389
389
  const detail = [
390
390
  ...[...why].map(([id, w]) => `- ${id}: ${w}`),
391
391
  ...[...missing].map((id) => `- ${id}: no test was written for it`),
392
+ ...(dropped ? ['', '### Tests taken out (title — why)', ...files.flatMap((f) => f.dropped.map((d) => `- ${f.slug}: ${d}`))] : []),
392
393
  ];
393
394
  writeFileSync(check, `${before.trimEnd()}\n\n## Written\n\n${lines.join('; ')}.\n${detail.length ? `\n${detail.join('\n')}\n` : ''}`, 'utf-8');
394
395
  }
@@ -763,7 +763,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
763
763
  if (asked.length) {
764
764
  const blank = recordInterface(Object.fromEntries(asked.map((n) => [n, undefined])));
765
765
  if (blank.length) {
766
- warnings.push(`${blank.length} name(s) the product has not given yet — fill them in ${INTERFACE_PATH}; until then \`testforge run\` holds those files back: ${blank.slice(0, 8).join(', ')}${blank.length > 8 ? ', …' : ''}`);
766
+ warnings.push(`${blank.length} name(s) the product has not given yet — \`testforge run\` looks for them in the code before each run, and their tests report "waiting for the product" until they are there (${INTERFACE_PATH}): ${blank.slice(0, 8).join(', ')}${blank.length > 8 ? ', …' : ''}`);
767
767
  }
768
768
  }
769
769
  const testFiles = generated.map((g) => ({
@@ -21,7 +21,7 @@ import { readInterface, recordInterface } from '../generate/interface-file.js';
21
21
  const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
22
22
  const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
23
23
  const CODE = /\.(php|ts|tsx|js|jsx|mjs|cjs|py|rb|go|vue|svelte|prisma|sql)$/;
24
- const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls']);
24
+ const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls', 'item', 'items', 'menu', 'link', 'entry', 'icon', 'text', 'title', 'page']);
25
25
  /** Where a name of this kind lives, judged from how the name ends. */
26
26
  export function kindOf(name) {
27
27
  const n = name.trim().toLowerCase();
@@ -49,11 +49,14 @@ function fileMatches(kind, rel) {
49
49
  }
50
50
  /** What a line that DEFINES a name of each kind looks like. */
51
51
  const SHAPE = {
52
- route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]/,
52
+ // …or a URL the app keeps as a constant (faq: 'https://…').
53
+ route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]|\b\w+\s*:\s*['"`]https?:\/\//,
53
54
  command: /\$signature\s*=|\$name\s*=\s*['"]|->command\(\s*['"]|\.command\(\s*['"]|add_parser\(|@click\.command/,
54
55
  schema: /Schema::(?:create|table)\(|\$table->\w+\(\s*['"]|create_table|\bclass\s+\w+\s+extends\s+\w*Model\b|protected\s+\$(?:table|fillable)|\bmodel\s+\w+\s*\{|@(?:Entity|Column|Table)\b/,
55
56
  field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]/,
56
- label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]/,
57
+ // …or a menu entry's label (label: 'Orders').
58
+ // …or a JSX text child on a line of its own (<Button>\n Escalate\n</Button>).
59
+ label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]|\blabel\s*:\s*['"`]|^\s*[A-Z][\w'’ ]{1,40}\s*$/,
57
60
  };
58
61
  /** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
59
62
  function candidateLines(name, kind, files, root) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.4.3",
3
+ "version": "2.4.5",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -37,21 +37,89 @@ function values() {
37
37
 
38
38
  /**
39
39
  * While a test file loads (need() at its top, outside any test), a throw would
40
- * fail the whole file and none of its tests would be reported. Instead each of
41
- * its tests fails with the waiting error, from a beforeEach. Inside a running
42
- * test the caller throws as usual.
40
+ * fail the whole file and none of its tests would be reported. Instead the
41
+ * tests that use the name fail with the waiting error, from a beforeEach; the
42
+ * file's other tests run. Inside a running test the caller throws as usual.
43
43
  */
44
44
  function inRunningTest() {
45
45
  try { if (globalThis.expect && globalThis.expect.getState().currentTestName) return true; } catch { /* not Jest */ }
46
46
  try { require('@playwright/test').test.info(); return true; } catch { return false; } // throws outside a test
47
47
  }
48
48
 
49
- function failTestsOfThisFile(message) {
49
+ /** The test file that called need(): the first frame outside this module. */
50
+ function callerFile() {
51
+ const stack = String(new Error().stack || '').split('\n').slice(1);
52
+ for (const line of stack) {
53
+ const m = /\(?((?:file:\/\/)?\/[^():]+|[A-Za-z]:\\[^():]+):\d+:\d+\)?\s*$/.exec(line);
54
+ const file = m && m[1].replace(/^file:\/\//, '');
55
+ if (file && file !== __filename && !/node_modules[\\/](?:@playwright|playwright|jest|@jest)/.test(file)) return file;
56
+ }
57
+ return undefined;
58
+ }
59
+
60
+ const TEST_START = /^\s*(?:test|it)(?:\.(?:only|skip|fixme|fail|slow|concurrent))?\s*\(/;
61
+
62
+ /**
63
+ * Which tests of a file use the constant a blank name was assigned to, worked
64
+ * out once per file and name. undefined means "every test": the constant is
65
+ * used outside a test (a hook, a shared helper), or could not be found, so no
66
+ * test is let through that might depend on it.
67
+ */
68
+ function usersOf(file, name) {
69
+ let src;
70
+ try { src = fs.readFileSync(file, 'utf-8'); } catch { return undefined; }
71
+ const esc = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
72
+ const decl = new RegExp(`\\b(?:const|let|var)\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*need\\(\\s*(['"\`])${esc}\\2\\s*\\)`).exec(src);
73
+ if (!decl) return undefined;
74
+ const v = decl[1];
75
+ const lines = src.split('\n');
76
+ const declLine = src.slice(0, decl.index).split('\n').length - 1;
77
+ // Each test: from its test( line to the next test( line or the next describe.
78
+ const tests = [];
79
+ lines.forEach((l, i) => { if (TEST_START.test(l)) tests.push({ start: i }); });
80
+ tests.forEach((t, k) => {
81
+ let end = k + 1 < tests.length ? tests[k + 1].start : lines.length;
82
+ for (let i = t.start + 1; i < end; i++) if (/^\s*(?:test\.)?(?:describe(?:\.\w+)?|beforeEach|beforeAll|afterEach|afterAll)\s*\(/.test(lines[i])) { end = i; break; }
83
+ t.end = end;
84
+ t.text = lines.slice(t.start, end).join('\n');
85
+ t.title = (/\(\s*(['"`])((?:(?!\1).)*)\1/.exec(lines[t.start]) || [])[2];
86
+ });
87
+ const uses = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
88
+ const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
89
+ if (lines.some((l, i) => i !== declLine && !inTest(i) && uses.test(l))) return undefined;
90
+ return tests.filter((t) => uses.test(t.text));
91
+ }
92
+
93
+ function blockTestsThatUse(message, name) {
50
94
  if (inRunningTest()) return false;
51
- const hook = () => { throw new Error(message); };
95
+ const file = callerFile();
96
+ let users;
97
+ try { users = file ? usersOf(file, name) : undefined; } catch { users = undefined; }
98
+ // Does the running test use the name? Playwright says which line declared it;
99
+ // Jest gives its full name, matched against the titles written in the file.
100
+ const needs = (line, fullName) => {
101
+ if (!users) return true;
102
+ if (line) return users.some((t) => t.start + 1 === line);
103
+ if (fullName) {
104
+ const t = users.find((u) => u.title && !/\$\{/.test(u.title) && fullName.endsWith(u.title));
105
+ if (t) return true;
106
+ // A title built at run time cannot be matched: when any user has one, wait to be safe.
107
+ return users.some((u) => !u.title || /\$\{/.test(u.title));
108
+ }
109
+ return true;
110
+ };
52
111
  try {
53
- if (typeof globalThis.beforeEach === 'function') { globalThis.beforeEach(hook); return true; } // Jest
54
- require('@playwright/test').test.beforeEach(hook); // Playwright
112
+ if (typeof globalThis.beforeEach === 'function') { // Jest: a hook with no parameters
113
+ globalThis.beforeEach(function () {
114
+ let fullName;
115
+ try { fullName = globalThis.expect.getState().currentTestName; } catch { /* unknown */ }
116
+ if (needs(undefined, fullName)) throw new Error(message);
117
+ });
118
+ return true;
119
+ }
120
+ require('@playwright/test').test.beforeEach(async ({}, testInfo) => { // Playwright
121
+ if (needs(testInfo && testInfo.line, testInfo && testInfo.titlePath && testInfo.titlePath.join(' '))) throw new Error(message);
122
+ });
55
123
  return true;
56
124
  } catch {
57
125
  return false;
@@ -64,8 +132,8 @@ function need(name) {
64
132
  if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
65
133
  // The prefix lets the reporters file this as "waiting for the product", not as a defect.
66
134
  const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json): this test runs for real once it is there.`;
67
- if (failTestsOfThisFile(message)) return `<waiting: ${name}>`;
135
+ if (blockTestsThatUse(message, name)) return `<waiting: ${name}>`;
68
136
  throw new Error(message);
69
137
  }
70
138
 
71
- module.exports = { need };
139
+ module.exports = { need, _usersOf: usersOf };