@activate-core/test-forge 2.4.4 → 2.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -451,8 +451,9 @@ words. Nothing found, or the same lines as last time: no AI call. New lines:
451
451
  one small call without tools picks each value, kept only if the code really
452
452
  has it, and printed with what was found. Tests, docs and plans are never
453
453
  touched. A test whose name is still blank is reported as "waiting for the
454
- product", not as a defect — also when the name is asked for at the top of the
455
- file, outside any test. `--no-scan` skips the look-up.
454
+ product", not as a defect. A name asked for once at the top of a file holds back
455
+ only the tests that use it (all of them when a shared hook uses it); the file's
456
+ other tests run. `--no-scan` skips the look-up.
456
457
 
457
458
  **Before the run, the testkit's packages.** A testkit lists what it needs
458
459
  (`REQUIRED_PACKAGES`, a database client say). When the project lacks one,
@@ -37,7 +37,7 @@ const MAP_PATH = 'testforge.md';
37
37
  * that upgrades keeps serving the facts it cached under the old version —
38
38
  * so the new facts never reach a single prompt and the upgrade does nothing.
39
39
  */
40
- const FACTS_VERSION = 6;
40
+ const FACTS_VERSION = 7;
41
41
  /** Where a journey's grounded facts live, mirroring the plan cache layout. */
42
42
  function groundingPath(journey) {
43
43
  return join(journeyDir(journey), 'facts.json');
@@ -69,7 +69,11 @@ function buildGroundingPrompt(journey, map, srcDir, excerpts = '') {
69
69
  ` token still populates the request user through a pre-auth hook; and the`,
70
70
  ` field that ownership is scoped on.`,
71
71
  `SHAPES: for each route, the request payload, the response body as its`,
72
- ` serializer builds it, and every status code the handler can return.`,
72
+ ` serializer builds it, and every status code the handler can return. For a`,
73
+ ` response a screen shows as a list or a detail, also list every field the`,
74
+ ` screen reads to show an item (with its type), the values a list keeps or`,
75
+ ` drops it by (a status the list filters on), and which of those fields the`,
76
+ ` screen renders as text, so a test can stub one item and find it on screen.`,
73
77
  `DATA: the tables or collections read or written, the fields that matter, and`,
74
78
  ` their enum values. For each table a test must seed, copy from the`,
75
79
  ` migrations, schema dump or model declarations: every NOT NULL column without`,
@@ -49,7 +49,9 @@ export async function completeFile(content, ctx) {
49
49
  if (first.cov.missing.length || first.problems.length) {
50
50
  out = withoutNames(withoutBlocks(content, first.problems));
51
51
  drop(first.problems);
52
- const left = ruleCoverage(out, ctx.rules).missing;
52
+ // Only rules the doc has are written again: an id with no text makes the writer invent one.
53
+ const known = new Set(ctx.memory.rules.filter((r) => r.tag).map((r) => r.id));
54
+ const left = ruleCoverage(out, ctx.rules).missing.filter((id) => known.has(id));
53
55
  if (left.length) {
54
56
  const blocks = findTestBlocks(out);
55
57
  const head = out.slice(0, blocks[0] ? blocks[0].start : Math.min(out.length, 4000)).slice(0, 4000);
@@ -573,6 +573,24 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
573
573
  const planned = useAI
574
574
  ? loadOrPlan(journey, hash, memory, memory ? testedElsewhere(journey, memory, tested) : [])
575
575
  : [{ slug: journey.id, title: journey.name, covers: '' }];
576
+ // A plan kept from an earlier version of the doc may name rules the doc no
577
+ // longer has (reworded into new ids, or removed). A writer asked to test a
578
+ // rule it is given no text for invents one, so those ids are dropped here,
579
+ // and a file left with no rule is not written.
580
+ if (memory) {
581
+ const known = new Set(memory.rules.filter((r) => r.tag).map((r) => r.id));
582
+ for (let k = planned.length - 1; k >= 0; k--) {
583
+ const pf = planned[k];
584
+ if (!pf.rules?.length)
585
+ continue;
586
+ const gone = pf.rules.filter((id) => !known.has(id));
587
+ if (!gone.length)
588
+ continue;
589
+ pf.rules = pf.rules.filter((id) => known.has(id));
590
+ if (!pf.rules.length)
591
+ planned.splice(k, 1);
592
+ }
593
+ }
576
594
  // A plan-only run leaves the memory as the tests were last written, so the
577
595
  // next generate still sees what changed in the doc and updates those tests.
578
596
  if (memory && !opts.planOnly) {
@@ -277,14 +277,19 @@ export const WEB = `WEB (Playwright)
277
277
  - A state the test cannot create (a cancelled booking, an expired code, a
278
278
  banned account, an empty list) is stubbed with page.route(). Shape the stub
279
279
  like the response in <code_facts>, and put "(stubbed state)" in the test
280
- title. Match the stub by exact host, port, path prefix or a request
280
+ title. A stubbed item carries every field the screen reads to show it,
281
+ including the value the list filters on (its status), and the test finds it
282
+ on screen by a field the screen renders as text, never by one it does not
283
+ show. Match the stub by exact host, port, path prefix or a request
281
284
  predicate, never by a substring that also matches the app's own pages.
282
285
  - Waiting: prefer retrying assertions. Register waitForResponse or
283
286
  waitForRequest before the action that triggers it. After a login, sign-out
284
287
  or navigation, wait for the control you will use, then assert the URL the
285
288
  app chooses. A page that never renders is reported as that, not as the rule
286
289
  failing.
287
- - A missing element is a failure, never a skip.
290
+ - A missing element is a failure, never a skip. kit.find fails when the
291
+ element is not there, so to check that something is not shown use
292
+ page.getByRole(…) or page.getByText(…) with toHaveCount(0) or toBeHidden().
288
293
  - For API calls inside a UI test, call the request fixture directly
289
294
  (request.get, request.post). For a separate context with its own headers,
290
295
  use kit.apiContext(request, role). Never call request.newContext() or
@@ -16,12 +16,11 @@ import { dirname, join, relative } from 'node:path';
16
16
  import { STATE_DIR } from '../paths.js';
17
17
  import { loadConfig, splitCommand, withNoTools } from '../project/config.js';
18
18
  import { runAiSync } from '../generate/ai-usage.js';
19
- import { cleanOutput } from '../generate/ai-generator.js';
20
19
  import { readInterface, recordInterface } from '../generate/interface-file.js';
21
20
  const SCAN_PATH = join(STATE_DIR, 'interface-scan.json');
22
21
  const SKIP = /^(node_modules|vendor|storage|public|dist|build|coverage|\.next|\.git|tests?|__tests__|testforge|generated-tests)$/;
23
22
  const CODE = /\.(php|ts|tsx|js|jsx|mjs|cjs|py|rb|go|vue|svelte|prisma|sql)$/;
24
- const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls']);
23
+ const GENERIC = new Set(['the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'list', 'value', 'name', 'response', 'request', 'body', 'query', 'path', 'prefix', 'endpoint', 'route', 'command', 'table', 'column', 'field', 'fields', 'label', 'button', 'key', 'setting', 'param', 'parameter', 'parameters', 'header', 'status', 'url', 'urls', 'item', 'items', 'menu', 'link', 'entry', 'icon', 'text', 'title', 'page']);
25
24
  /** Where a name of this kind lives, judged from how the name ends. */
26
25
  export function kindOf(name) {
27
26
  const n = name.trim().toLowerCase();
@@ -49,11 +48,14 @@ function fileMatches(kind, rel) {
49
48
  }
50
49
  /** What a line that DEFINES a name of each kind looks like. */
51
50
  const SHAPE = {
52
- route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]/,
51
+ // …or a URL the app keeps as a constant (faq: 'https://…').
52
+ route: /Route::\w+\(|\brouter\.\w+\(|\bapp\.(?:get|post|put|patch|delete)\(|@(?:Get|Post|Put|Patch|Delete)\(|\bpath\(\s*['"]|\bre_path\(|^\s*['"`]\/[\w\-/{}:]+['"`]|\b\w+\s*:\s*['"`]https?:\/\//,
53
53
  command: /\$signature\s*=|\$name\s*=\s*['"]|->command\(\s*['"]|\.command\(\s*['"]|add_parser\(|@click\.command/,
54
54
  schema: /Schema::(?:create|table)\(|\$table->\w+\(\s*['"]|create_table|\bclass\s+\w+\s+extends\s+\w*Model\b|protected\s+\$(?:table|fillable)|\bmodel\s+\w+\s*\{|@(?:Entity|Column|Table)\b/,
55
55
  field: /['"][\w.-]+['"]\s*(?:=>|:)|^\s*[\w$]+\??\s*:\s*\S|->(?:input|get|query)\(\s*['"]/,
56
- label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]/,
56
+ // …or a menu entry's label (label: 'Orders').
57
+ // …or a JSX text child on a line of its own (<Button>\n Escalate\n</Button>).
58
+ label: />\s*[A-Za-z][^<>{}]{1,60}<|\b(?:aria-label|placeholder|title|label)\s*=\s*['"{]|\blabel\s*:\s*['"`]|^\s*[A-Z][\w'’ ]{1,40}\s*$|\/>\s*[A-Z][\w'’ ]{1,40}\s*$/,
57
59
  };
58
60
  /** The lines of a kind's files that share a word with the name, best first; the route prefixes ride along. */
59
61
  function candidateLines(name, kind, files, root) {
@@ -213,7 +215,10 @@ export function scanNames(blank, root = '.') {
213
215
  try {
214
216
  const { program, args } = withNoTools(splitCommand(loadConfig().aiCommand), 'Answer only with the JSON object asked for.');
215
217
  calls = 1;
216
- const raw = cleanOutput(runAiSync(program, args, { input: prompt, encoding: 'utf-8', maxBuffer: 4 * 1024 * 1024, timeout: 120_000 }));
218
+ // The answer is JSON, not code: read it as it is (cleanOutput keeps only code-like lines and drops it).
219
+ const raw = String(runAiSync(program, args, { input: prompt, encoding: 'utf-8', maxBuffer: 4 * 1024 * 1024, timeout: 120_000 }));
220
+ if (raw.indexOf('{') < 0)
221
+ throw new Error(`the answer was not JSON: ${raw.slice(0, 80)}`);
217
222
  const json = JSON.parse(raw.slice(raw.indexOf('{'), raw.lastIndexOf('}') + 1));
218
223
  const keep = {};
219
224
  for (const a of ask) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "2.4.4",
3
+ "version": "2.4.6",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -37,21 +37,108 @@ function values() {
37
37
 
38
38
  /**
39
39
  * While a test file loads (need() at its top, outside any test), a throw would
40
- * fail the whole file and none of its tests would be reported. Instead each of
41
- * its tests fails with the waiting error, from a beforeEach. Inside a running
42
- * test the caller throws as usual.
40
+ * fail the whole file and none of its tests would be reported. Instead the
41
+ * tests that use the name fail with the waiting error, from a beforeEach; the
42
+ * file's other tests run. Inside a running test the caller throws as usual.
43
43
  */
44
44
  function inRunningTest() {
45
45
  try { if (globalThis.expect && globalThis.expect.getState().currentTestName) return true; } catch { /* not Jest */ }
46
46
  try { require('@playwright/test').test.info(); return true; } catch { return false; } // throws outside a test
47
47
  }
48
48
 
49
- function failTestsOfThisFile(message) {
49
+ /** The test file that called need(): the first frame outside this module. */
50
+ function callerFile() {
51
+ const stack = String(new Error().stack || '').split('\n').slice(1);
52
+ for (const line of stack) {
53
+ const m = /\(?((?:file:\/\/)?\/[^():]+|[A-Za-z]:\\[^():]+):\d+:\d+\)?\s*$/.exec(line);
54
+ const file = m && m[1].replace(/^file:\/\//, '');
55
+ if (file && file !== __filename && !/node_modules[\\/](?:@playwright|playwright|jest|@jest)/.test(file)) return file;
56
+ }
57
+ return undefined;
58
+ }
59
+
60
+ // A test is declared with its title first; test.skip(condition, 'why') inside a test is not one.
61
+ /** Index of the ")" closing the "(" at `open`, skipping strings, templates and comments; -1 if none. */
62
+ function closingParen(src, open) {
63
+ let depth = 0;
64
+ for (let i = open; i < src.length; i++) {
65
+ const c = src[i];
66
+ if (c === '/' && src[i + 1] === '/') { i = src.indexOf('\n', i); if (i < 0) return -1; continue; }
67
+ if (c === '/' && src[i + 1] === '*') { i = src.indexOf('*/', i + 2); if (i < 0) return -1; i += 1; continue; }
68
+ if (c === '"' || c === "'" || c === '`') {
69
+ for (i += 1; i < src.length && src[i] !== c; i++) if (src[i] === '\\') i += 1;
70
+ continue;
71
+ }
72
+ if (c === '(') depth += 1;
73
+ else if (c === ')') { depth -= 1; if (depth === 0) return i; }
74
+ }
75
+ return -1;
76
+ }
77
+
78
+ const TEST_START = /^\s*(?:test|it)(?:\.(?:only|skip|fixme|fail|slow|concurrent))?\s*\(\s*['"`]/;
79
+
80
+ /**
81
+ * Which tests of a file use the constant a blank name was assigned to, worked
82
+ * out once per file and name. undefined means "every test": the constant is
83
+ * used outside a test (a hook, a shared helper), or could not be found, so no
84
+ * test is let through that might depend on it.
85
+ */
86
+ function usersOf(file, name) {
87
+ let src;
88
+ try { src = fs.readFileSync(file, 'utf-8'); } catch { return undefined; }
89
+ const esc = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
90
+ const decl = new RegExp(`\\b(?:const|let|var)\\s+([A-Za-z_$][\\w$]*)\\s*=\\s*need\\(\\s*(['"\`])${esc}\\2\\s*\\)`).exec(src);
91
+ if (!decl) return undefined;
92
+ const v = decl[1];
93
+ const lines = src.split('\n');
94
+ const declLine = src.slice(0, decl.index).split('\n').length - 1;
95
+ // Each test: from its test( line to the bracket that closes the call, so a
96
+ // helper written between two tests belongs to neither of them.
97
+ const starts = [];
98
+ let at = 0;
99
+ lines.forEach((l, i) => { if (TEST_START.test(l)) starts.push({ i, pos: at + l.indexOf('(') }); at += l.length + 1; });
100
+ const lineOf = (pos) => src.slice(0, pos).split('\n').length - 1;
101
+ const tests = starts.map(({ i, pos }) => {
102
+ const close = closingParen(src, pos);
103
+ const end = close < 0 ? lines.length : lineOf(close) + 1;
104
+ return { start: i, end, text: lines.slice(i, end).join('\n'), title: (/\(\s*(['"`])((?:(?!\1).)*)\1/.exec(lines[i]) || [])[2] };
105
+ });
106
+ const uses = new RegExp(`\\b${v.replace(/\$/g, '\\$')}\\b`);
107
+ const inTest = (i) => tests.some((t) => i >= t.start && i < t.end);
108
+ if (lines.some((l, i) => i !== declLine && !inTest(i) && uses.test(l))) return undefined;
109
+ return tests.filter((t) => uses.test(t.text));
110
+ }
111
+
112
+ function blockTestsThatUse(message, name) {
50
113
  if (inRunningTest()) return false;
51
- const hook = () => { throw new Error(message); };
114
+ const file = callerFile();
115
+ let users;
116
+ try { users = file ? usersOf(file, name) : undefined; } catch { users = undefined; }
117
+ // Does the running test use the name? Playwright says which line declared it;
118
+ // Jest gives its full name, matched against the titles written in the file.
119
+ const needs = (line, fullName) => {
120
+ if (!users) return true;
121
+ if (line) return users.some((t) => t.start + 1 === line);
122
+ if (fullName) {
123
+ const t = users.find((u) => u.title && !/\$\{/.test(u.title) && fullName.endsWith(u.title));
124
+ if (t) return true;
125
+ // A title built at run time cannot be matched: when any user has one, wait to be safe.
126
+ return users.some((u) => !u.title || /\$\{/.test(u.title));
127
+ }
128
+ return true;
129
+ };
52
130
  try {
53
- if (typeof globalThis.beforeEach === 'function') { globalThis.beforeEach(hook); return true; } // Jest
54
- require('@playwright/test').test.beforeEach(hook); // Playwright
131
+ if (typeof globalThis.beforeEach === 'function') { // Jest: a hook with no parameters
132
+ globalThis.beforeEach(function () {
133
+ let fullName;
134
+ try { fullName = globalThis.expect.getState().currentTestName; } catch { /* unknown */ }
135
+ if (needs(undefined, fullName)) throw new Error(message);
136
+ });
137
+ return true;
138
+ }
139
+ require('@playwright/test').test.beforeEach(async ({}, testInfo) => { // Playwright
140
+ if (needs(testInfo && testInfo.line, testInfo && testInfo.titlePath && testInfo.titlePath.join(' '))) throw new Error(message);
141
+ });
55
142
  return true;
56
143
  } catch {
57
144
  return false;
@@ -64,8 +151,8 @@ function need(name) {
64
151
  if (typeof v === 'string' ? v.trim() : v !== undefined && v !== null) return v;
65
152
  // The prefix lets the reporters file this as "waiting for the product", not as a defect.
66
153
  const message = `[testforge:waiting] "${name}" is not in the product yet (blank in testforge/interface.json): this test runs for real once it is there.`;
67
- if (failTestsOfThisFile(message)) return `<waiting: ${name}>`;
154
+ if (blockTestsThatUse(message, name)) return `<waiting: ${name}>`;
68
155
  throw new Error(message);
69
156
  }
70
157
 
71
- module.exports = { need };
158
+ module.exports = { need, _usersOf: usersOf };