@activate-core/test-forge 1.3.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +51 -28
  2. package/dist/cli/index.js +39 -33
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +49 -2
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +49 -22
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +60 -3
  10. package/dist/core/docs/review.js +38 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +3 -3
  13. package/dist/core/gates/coverage.js +1 -16
  14. package/dist/core/gates/validate.js +171 -7
  15. package/dist/core/generate/ai-generator.js +202 -104
  16. package/dist/core/generate/ai-usage.js +116 -5
  17. package/dist/core/generate/bind.js +91 -0
  18. package/dist/core/generate/defect-review.js +59 -9
  19. package/dist/core/generate/doc-diff.js +139 -0
  20. package/dist/core/generate/harness.js +3 -31
  21. package/dist/core/generate/journey-memory.js +325 -0
  22. package/dist/core/generate/plan-cache.js +114 -7
  23. package/dist/core/generate/review-journey.js +3 -3
  24. package/dist/core/generate/run-generate.js +169 -78
  25. package/dist/core/generate/suite-planner.js +128 -23
  26. package/dist/core/generate/test-blocks.js +84 -0
  27. package/dist/core/generate/update-journey.js +115 -0
  28. package/dist/core/paths.js +90 -13
  29. package/dist/core/project/config.js +25 -2
  30. package/dist/core/project/env-template.js +2 -1
  31. package/dist/core/run/ci.js +1 -1
  32. package/dist/core/run/clean.js +1 -1
  33. package/dist/core/run/readiness.js +119 -0
  34. package/dist/core/run/run-tests.js +23 -1
  35. package/package.json +1 -1
  36. package/reporters/_shared.cjs +2 -2
  37. package/reporters/run-data.cjs +1 -1
  38. package/reporters/ui-helpers.cjs +78 -1
  39. package/dist/core/generate/verify-journey.js +0 -215
@@ -0,0 +1,84 @@
1
+ /**
2
+ * testforge — single tests inside a generated file.
3
+ *
4
+ * Each generated test names the rule it checks in an oracle comment
5
+ * ("// oracle: R12 [spec] …"). That is how a change to one rule reaches only
6
+ * its tests: they are found, replaced, renamed or removed in place, and the
7
+ * rest of the file is never read by the AI or rewritten.
8
+ */
9
+ import { matchParen } from '../gates/validate.js';
10
+ const TEST_CALL = /(^|[^\w.])(?:test|it)(?:\.(?:only|fixme|skip|slow|fail))?\s*\(\s*(['"`])((?:(?!\2).)*)\2/gm;
11
+ const RULE_ID = /\boracle:\s*((?:R\d+[\s,/&]*(?:and\s+)?)+)/g;
12
+ /** Every test(…) call in the file, describe blocks and hooks excluded. */
13
+ export function findTestBlocks(content) {
14
+ const out = [];
15
+ for (const m of content.matchAll(TEST_CALL)) {
16
+ const callAt = m.index + m[1].length;
17
+ const open = content.indexOf('(', callAt);
18
+ const close = matchParen(content, open);
19
+ if (close === -1)
20
+ continue;
21
+ let end = close + 1;
22
+ if (content[end] === ';')
23
+ end += 1;
24
+ // Pull in the comment lines right above the call (oracle, defect marks).
25
+ let start = content.lastIndexOf('\n', callAt - 1) + 1;
26
+ for (;;) {
27
+ const prevEnd = start - 1;
28
+ if (prevEnd <= 0)
29
+ break;
30
+ const prevStart = content.lastIndexOf('\n', prevEnd - 1) + 1;
31
+ if (!/^\s*\/\//.test(content.slice(prevStart, prevEnd)))
32
+ break;
33
+ start = prevStart;
34
+ }
35
+ const text = content.slice(start, end);
36
+ const rules = [...text.matchAll(RULE_ID)].flatMap((r) => r[1].match(/R\d+/g) ?? []);
37
+ out.push({ start, end, title: m[3], rules: [...new Set(rules)] });
38
+ }
39
+ return out.filter((b) => !out.some((o) => o !== b && o.start <= b.start && o.end >= b.end && o.end - o.start > b.end - b.start));
40
+ }
41
+ /** Replace (or, with '', remove) the given blocks; later ones first so offsets stay valid. */
42
+ export function spliceBlocks(content, edits) {
43
+ let out = content;
44
+ for (const { block, text } of [...edits].sort((a, b) => b.block.start - a.block.start)) {
45
+ if (text) {
46
+ out = out.slice(0, block.start) + text + out.slice(block.end);
47
+ continue;
48
+ }
49
+ // A removed test takes its own line and the blank line that separated it.
50
+ out = out.slice(0, block.start).replace(/\n[ \t]*\n[ \t]*$/, '\n') + out.slice(block.end).replace(/^[ \t]*\n/, '');
51
+ }
52
+ return out;
53
+ }
54
+ /** Add tests inside the file's outermost describe, before it closes; at the end when there is none. */
55
+ export function appendTests(content, tests) {
56
+ const d = /\btest\.describe(?:\.\w+)?\s*\(/.exec(content);
57
+ if (d) {
58
+ const close = matchParen(content, content.indexOf('(', d.index));
59
+ if (close !== -1) {
60
+ // The describe's body closes with "}" just before ")".
61
+ const brace = content.lastIndexOf('}', close);
62
+ if (brace > d.index)
63
+ return `${content.slice(0, brace).trimEnd()}\n\n${indent(tests.trim(), ' ')}\n${content.slice(brace)}`;
64
+ }
65
+ }
66
+ return `${content.trimEnd()}\n\n${tests.trim()}\n`;
67
+ }
68
+ const indent = (t, pad) => t.split('\n').map((l) => (l ? pad + l : l)).join('\n');
69
+ /** Literal replacement of each old text with the new one, only inside the given blocks. */
70
+ export function renameInBlocks(content, blocks, pairs) {
71
+ let replaced = 0;
72
+ const edits = blocks.map((block) => {
73
+ let text = content.slice(block.start, block.end);
74
+ for (const [from, to] of pairs) {
75
+ const n = text.split(from).length - 1;
76
+ if (n) {
77
+ text = text.split(from).join(to);
78
+ replaced += n;
79
+ }
80
+ }
81
+ return { block, text };
82
+ });
83
+ return { content: spliceBlocks(content, edits), replaced };
84
+ }
@@ -0,0 +1,115 @@
1
+ /**
2
+ * testforge — bring an existing suite up to a changed journey doc, in place.
3
+ *
4
+ * The doc diff says which rules were renamed, reworded, added or removed. A
5
+ * rename is a find-and-replace inside that rule's tests; a removed rule's
6
+ * tests are deleted; a reworded or added rule gets its tests written by one
7
+ * small AI call that sees only those rules and the file's top. Nothing else in
8
+ * the file is sent or rewritten. Rules in a section no file covers yet become
9
+ * new files in the plan, which the normal per-file step writes.
10
+ */
11
+ import { existsSync, readFileSync, writeFileSync } from 'node:fs';
12
+ import { journeySlice, factsSlice } from './journey-memory.js';
13
+ import { appendTests, findTestBlocks, renameInBlocks, spliceBlocks } from './test-blocks.js';
14
+ import { writeTestsWithAI } from './ai-generator.js';
15
+ const blocksFor = (content, ids) => findTestBlocks(content).filter((b) => b.rules.some((r) => ids.includes(r)));
16
+ const slugOf = (section) => section.replace(/^§[\d.]+\s*/, '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').split('-').slice(0, 4).join('-') || 'new-rules';
17
+ export async function updateJourney(ctx) {
18
+ const { memory, diff } = ctx;
19
+ const notes = [];
20
+ const rule = new Map(memory.rules.map((r) => [r.id, r]));
21
+ const removed = new Set(diff.removed);
22
+ const testedId = (id) => {
23
+ const r = rule.get(id);
24
+ return !!r && !!r.tag && ctx.tested.includes(r.tag);
25
+ };
26
+ const plan = ctx.plan.map((f) => ({ ...f, rules: (f.rules ?? []).filter((id) => !removed.has(id)) }));
27
+ // Each added rule joins the file that already tests its section; a section
28
+ // no file covers becomes a new file the normal step writes.
29
+ const added = diff.added.filter(testedId);
30
+ const toFile = new Map();
31
+ const orphans = new Map();
32
+ for (const id of added) {
33
+ const section = rule.get(id).section;
34
+ const home = plan.find((f) => f.rules.some((r) => rule.get(r)?.section === section));
35
+ if (home)
36
+ toFile.set(home, [...(toFile.get(home) ?? []), id]);
37
+ else
38
+ orphans.set(section, [...(orphans.get(section) ?? []), id]);
39
+ }
40
+ for (const f of plan) {
41
+ const path = ctx.pathOf(f);
42
+ if (!existsSync(path))
43
+ continue;
44
+ const before = readFileSync(path, 'utf-8');
45
+ let content = before;
46
+ const mine = (ids) => ids.filter((id) => ctx.plan.find((p) => p.slug === f.slug)?.rules?.includes(id));
47
+ // Renames: literal replacement inside the renamed rule's tests only. No AI.
48
+ // A file written before tests named their rule gets the rename in every test.
49
+ const legacy = !findTestBlocks(content).some((b) => b.rules.length);
50
+ for (const r of diff.renamed.filter((x) => legacy || mine([x.id]).length)) {
51
+ const res = renameInBlocks(content, legacy ? findTestBlocks(content) : blocksFor(content, [r.id]), r.pairs);
52
+ content = res.content;
53
+ if (res.replaced)
54
+ notes.push(`${f.slug}: ${r.id} renamed ${r.pairs.map(([a, b]) => `"${a}" → "${b}"`).join(', ')} (${res.replaced} place(s))`);
55
+ }
56
+ // Removed rules: their tests go, unless a test also checks a rule that stays.
57
+ const gone = findTestBlocks(content).filter((b) => b.rules.length && b.rules.every((id) => removed.has(id)));
58
+ if (gone.length) {
59
+ content = spliceBlocks(content, gone.map((block) => ({ block, text: '' })));
60
+ notes.push(`${f.slug}: removed ${gone.length} test(s) of rules no longer in the doc`);
61
+ }
62
+ // Reworded rules and added rules: one small AI call for just those.
63
+ const changed = mine(diff.changed).filter(testedId);
64
+ const extra = toFile.get(f) ?? [];
65
+ const ids = [...changed, ...extra];
66
+ if (ids.length) {
67
+ const old = blocksFor(content, changed);
68
+ const firstTest = findTestBlocks(content)[0];
69
+ const head = content.slice(0, firstTest ? firstTest.start : Math.min(content.length, 4000)).slice(0, 4000);
70
+ const title = /test\.describe(?:\.\w+)?\(\s*(['"`])((?:(?!\1).)*)\1/.exec(content)?.[2] ?? f.title;
71
+ const slice = journeySlice(memory, ids);
72
+ const mem = { journey: slice.text, helpers: ctx.helpers, rules: ids };
73
+ const oldText = old.map((b) => content.slice(b.start, b.end)).join('\n\n');
74
+ const base = old.length ? spliceBlocks(content, old.map((block) => ({ block, text: '' }))) : content;
75
+ let refusal;
76
+ let next = null;
77
+ for (let attempt = 0; attempt < 2 && next === null; attempt++) {
78
+ let tests = '';
79
+ try {
80
+ tests = await writeTestsWithAI(ctx.journey, path, head, title, oldText, ctx.harnessPath, ctx.transport, factsSlice(ctx.facts, slice.controls), mem, refusal);
81
+ }
82
+ catch (e) {
83
+ refusal = undefined;
84
+ notes.push(`${f.slug}: the AI call for ${ids.join(', ')} failed — ${e.message.slice(0, 120)}`);
85
+ break;
86
+ }
87
+ const candidate = appendTests(base, tests);
88
+ refusal = findTestBlocks(tests).length ? ctx.check(candidate, path) ?? undefined : 'it contained no test(...) block';
89
+ if (!refusal)
90
+ next = candidate;
91
+ }
92
+ if (next !== null) {
93
+ content = next;
94
+ if (changed.length)
95
+ notes.push(`${f.slug}: rewrote the tests of ${changed.join(', ')} (rule changed)`);
96
+ if (extra.length)
97
+ notes.push(`${f.slug}: added tests for ${extra.join(', ')}`);
98
+ f.rules = [...new Set([...f.rules, ...extra])];
99
+ }
100
+ else if (refusal) {
101
+ notes.push(`${f.slug}: tests for ${ids.join(', ')} were refused (${refusal}) — the file keeps its old tests`);
102
+ }
103
+ }
104
+ if (content !== before)
105
+ writeFileSync(path, content, 'utf-8');
106
+ }
107
+ for (const [section, ids] of orphans) {
108
+ let slug = slugOf(section);
109
+ while (plan.some((f) => f.slug === slug))
110
+ slug = `${slug}-2`;
111
+ plan.push({ slug, title: `${ctx.journey.name} — ${section.replace(/^§[\d.]+\s*/, '')}`, covers: `The rules of ${section}.`, rules: ids });
112
+ notes.push(`${slug}: new file for ${ids.join(', ')} (new section ${section})`);
113
+ }
114
+ return { plan: plan.filter((f) => f.rules.length), notes };
115
+ }
@@ -1,5 +1,5 @@
1
- import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
2
- import { join } from 'node:path';
1
+ import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
2
+ import { dirname, join } from 'node:path';
3
3
  /**
4
4
  * Single source of truth for TestForge's per-project state directory and the
5
5
  * files/sub-dirs inside it. Everything the tool persists between runs lives
@@ -9,26 +9,103 @@ import { join } from 'node:path';
9
9
  * Rename in ONE place and every command — setup, discover, generate, update,
10
10
  * coverage — follows automatically, so no path can drift out of sync.
11
11
  */
12
- export const STATE_DIR = 'test-artifacts';
13
- /** Legacy state-dir name kept only so source-hash skip-lists still ignore a
14
- * pre-migration folder if one is lying around. Not written to. */
12
+ export const STATE_DIR = 'testforge';
13
+ /** Earlier state-dir names, still skipped by source hashing and moved by migrateState(). */
15
14
  export const LEGACY_STATE_DIR = '.testforge';
15
+ export const OLD_STATE_DIR = 'test-artifacts';
16
16
  export const CACHE_PATH = join(STATE_DIR, 'cache.json');
17
17
  export const SOURCE_PATH = join(STATE_DIR, 'source.json');
18
18
  export const COVERAGE_PATH = join(STATE_DIR, 'coverage.json');
19
- /** Per-journey generation plans, organised per area: <PLANS_DIR>/<area>/<id>.json */
20
- export const PLANS_DIR = join(STATE_DIR, 'test-plans');
19
+ /** Everything TestForge remembers about each journey: <MEMORY_DIR>/<area>/<id>/ */
20
+ export const MEMORY_DIR = join(STATE_DIR, 'memory');
21
+ /** One journey's folder: rules.json, plan.json, plan-check.md, facts.json. */
22
+ export function journeyDir(journey) {
23
+ return join(MEMORY_DIR, (journey.area || 'misc').replace(/[^a-z0-9-]/gi, '-'), journey.id);
24
+ }
25
+ /** The references read (Notion pages, ticket bundles) and the comparison with the map. */
26
+ export const REFERENCE_DIR = join(STATE_DIR, 'reference');
27
+ /** What one run leaves behind, kept on this machine only. */
28
+ export const RUNS_DIR = join(STATE_DIR, 'runs');
21
29
  /** Where Playwright writes its per-run output (failure reports, traces). */
22
- export const RESULTS_DIR = join(STATE_DIR, 'playwright-results');
30
+ export const RESULTS_DIR = join(RUNS_DIR, 'playwright-results');
31
+ /**
32
+ * Move a project's state from the old layout (test-artifacts/, testforge.diff.md)
33
+ * into testforge/. Nothing is rebuilt and nothing already in the new place is
34
+ * overwritten. Safe to call on every command.
35
+ */
36
+ export function migrateState() {
37
+ const moved = [];
38
+ const move = (from, to) => {
39
+ if (!existsSync(from) || existsSync(to))
40
+ return;
41
+ mkdirSync(dirname(to), { recursive: true });
42
+ renameSync(from, to);
43
+ moved.push(`${from} → ${to}`);
44
+ };
45
+ move('testforge.diff.md', join(REFERENCE_DIR, 'diff.md'));
46
+ const old = OLD_STATE_DIR;
47
+ if (!existsSync(old) || !statSync(old).isDirectory())
48
+ return moved;
49
+ for (const f of ['cache.json', 'source.json', 'coverage.json'])
50
+ move(join(old, f), join(STATE_DIR, f));
51
+ move(join(old, 'tickets-reference.md'), join(REFERENCE_DIR, 'tickets.md'));
52
+ for (const f of readdirSafe(join(old, 'reference')))
53
+ move(join(old, 'reference', f), join(REFERENCE_DIR, f));
54
+ for (const d of ['trace', 'playwright-results', 'verify-results', 'verify-progress', 'defect-review', 'run-data'])
55
+ move(join(old, d), join(RUNS_DIR, d));
56
+ // Per-journey files, from three folders into one per journey.
57
+ const perJourney = [
58
+ ['memory', (n) => (n.endsWith('.json') ? n.slice(0, -5) : undefined), 'rules.json'],
59
+ ['test-plans', (n) => (n.endsWith('.json') ? n.slice(0, -5) : undefined), 'plan.json'],
60
+ ['test-plans', (n) => (n.endsWith('.gate.md') ? n.slice(0, -8) : undefined), 'plan-check.md'],
61
+ ['grounding', (n) => (n.endsWith('.json') ? n.slice(0, -5) : undefined), 'facts.json'],
62
+ ];
63
+ for (const [dir, idOf, file] of perJourney) {
64
+ for (const area of readdirSafe(join(old, dir))) {
65
+ for (const n of readdirSafe(join(old, dir, area))) {
66
+ const id = idOf(n);
67
+ if (id)
68
+ move(join(old, dir, area, n), join(MEMORY_DIR, area, id, file));
69
+ }
70
+ }
71
+ }
72
+ removeIfEmpty(old);
73
+ return moved;
74
+ }
75
+ function readdirSafe(dir) {
76
+ try {
77
+ return readdirSync(dir);
78
+ }
79
+ catch {
80
+ return [];
81
+ }
82
+ }
83
+ /** Remove a directory tree that holds no files (only empty folders, or the old ignore file). */
84
+ function removeIfEmpty(dir) {
85
+ const files = (d) => readdirSafe(d).reduce((n, e) => {
86
+ const p = join(d, e);
87
+ return n + (statSync(p).isDirectory() ? files(p) : e === '.gitignore' ? 0 : 1);
88
+ }, 0);
89
+ if (existsSync(dir) && files(dir) === 0)
90
+ rmSync(dir, { recursive: true, force: true });
91
+ }
92
+ /** The ignore file TestForge used to write: everything local. Replaced when found. */
93
+ const OLD_IGNORE = '# testforge local state: regenerated as needed, so not committed.\n*\n!.gitignore\n!source.json\n';
94
+ const IGNORE = [
95
+ '# testforge state. Commit it: each journey\'s memory (rules, plan, code facts) and',
96
+ '# the references read cost AI to rebuild, and a teammate who pulls them starts',
97
+ '# from where you left off. Only runs/ (run output, AI logs) stays on this machine.',
98
+ 'runs/',
99
+ '',
100
+ ].join('\n');
23
101
  /**
24
- * Keep the state dir out of git without touching the project's own
25
- * .gitignore: caches, plans and run output are local and regenerated, only
26
- * the source-of-truth choice is worth sharing with the team.
102
+ * The state dir is shared with the team through git, except what one run
103
+ * leaves behind. Written once; the old "ignore everything" file is upgraded.
27
104
  */
28
105
  export function ensureStateDirIgnored() {
29
106
  const file = join(STATE_DIR, '.gitignore');
30
- if (existsSync(file))
107
+ if (existsSync(file) && readFileSync(file, 'utf-8') !== OLD_IGNORE)
31
108
  return;
32
109
  mkdirSync(STATE_DIR, { recursive: true });
33
- writeFileSync(file, '# testforge local state: regenerated as needed, so not committed.\n*\n!.gitignore\n!source.json\n', 'utf-8');
110
+ writeFileSync(file, IGNORE, 'utf-8');
34
111
  }
@@ -119,7 +119,7 @@ export function resolveFramework(config, area, type) {
119
119
  return isApi ? 'jest' : 'playwright';
120
120
  }
121
121
  /** Is this command Claude's CLI (so Claude-specific flags apply)? */
122
- function isClaude(program) {
122
+ export function isClaude(program) {
123
123
  const base = program.split(/[\\/]/).pop() ?? program;
124
124
  return base === 'claude';
125
125
  }
@@ -152,9 +152,32 @@ export function withReadOnlyTools(cmd, relatedDirs = loadConfig().relatedDirs ??
152
152
  return {
153
153
  program: cmd.program,
154
154
  // Space-separated allowlist; Write/Edit/Bash deliberately excluded.
155
- args: [...cmd.args, '--allowedTools', 'Read Grep Glob LS', ...addDirArgs(relatedDirs)],
155
+ args: [...cmd.args, '--allowedTools', 'Read Grep Glob LS', ...LEAN, ...addDirArgs(relatedDirs)],
156
156
  };
157
157
  }
158
+ /**
159
+ * What a TestForge call never needs from the user's own Claude setup: their
160
+ * settings, hooks, CLAUDE.md memory, MCP servers and slash commands. Each of
161
+ * those rides along on every turn; measured, they were ~10k of the ~30k tokens
162
+ * a turn carried before the task began. Not for a phase that reads Notion:
163
+ * the connectors come in with the settings.
164
+ */
165
+ const LEAN = ['--setting-sources', '', '--strict-mcp-config', '--disable-slash-commands'];
166
+ /** The whole system prompt of a no-tools call: the task itself comes on stdin. */
167
+ const WRITER_SYSTEM = 'You write automated test code for a QA tool. Follow the instructions in the message exactly and output only what it asks for.';
168
+ /**
169
+ * A call that has everything it needs in its prompt: no tools, no exploring,
170
+ * Claude Code's own system prompt replaced by one line. One turn in, one
171
+ * answer out. Measured: ~600 tokens of overhead per call instead of ~30k.
172
+ */
173
+ export function withNoTools(cmd, system = WRITER_SYSTEM) {
174
+ if (!isClaude(cmd.program))
175
+ return cmd;
176
+ // Low effort: the thinking is in the prompt's rules and the slice it carries;
177
+ // what is asked for is code from them, and thinking tokens are output tokens.
178
+ const effort = cmd.args.includes('--effort') ? [] : ['--effort', loadConfig().aiEffort ?? 'low'];
179
+ return { program: cmd.program, args: [...cmd.args, '--system-prompt', system, '--tools', '', ...effort, ...LEAN] };
180
+ }
158
181
  /**
159
182
  * Read-only tools a headless exploring phase needs pre-approved. The model
160
183
  * reaches for shell `grep`/`ls` as often as for the Grep tool; under
@@ -14,7 +14,7 @@
14
14
  import { readFileSync, writeFileSync, existsSync, readdirSync, } from 'node:fs';
15
15
  import { join } from 'node:path';
16
16
  import { findMarkdownFiles } from '../find-files.js';
17
- import { STATE_DIR, LEGACY_STATE_DIR } from '../paths.js';
17
+ import { STATE_DIR, LEGACY_STATE_DIR, OLD_STATE_DIR } from '../paths.js';
18
18
  import { loadConfig } from './config.js';
19
19
  /** Pull env var names from a chunk of text. */
20
20
  export function extractEnvVars(text) {
@@ -53,6 +53,7 @@ const SKIP_DIRS = new Set([
53
53
  'generated-tests',
54
54
  STATE_DIR,
55
55
  LEGACY_STATE_DIR,
56
+ OLD_STATE_DIR,
56
57
  ]);
57
58
  const SOURCE_EXT = /\.(c|m)?[jt]sx?$/i;
58
59
  /**
@@ -66,7 +66,7 @@ export function workflowYaml(o) {
66
66
  ` uses: actions/upload-artifact@v4`,
67
67
  ` with:`,
68
68
  ` name: testforge-results`,
69
- ` path: test-artifacts/`,
69
+ ` path: testforge/runs/`,
70
70
  ` retention-days: 14`,
71
71
  ``,
72
72
  ].join('\n');
@@ -49,7 +49,7 @@ export async function runClean(opts) {
49
49
  if (!opts.runId && opts.olderThanDays == null) {
50
50
  const runs = runData.listRuns(process.cwd());
51
51
  if (!runs.length)
52
- return 'No run has recorded any data yet (test-artifacts/run-data/ is empty).';
52
+ return 'No run has recorded any data yet (testforge/runs/run-data/ is empty).';
53
53
  return ['Runs with recorded data:', ...runs.map((r) => ` ${r.runId} ${r.at.slice(0, 16).replace('T', ' ')} ${r.count} record(s)`),
54
54
  '', 'Clean one with --run <id>, or every run older than N days with --older-than <N>. Add --yes to remove.'].join('\n');
55
55
  }
@@ -0,0 +1,119 @@
1
+ /**
2
+ * testforge — has the feature a test file checks been built yet?
3
+ *
4
+ * Tests are written from the doc, often before the code exists. Before a run,
5
+ * each file's names (the buttons, links, fields, headings, images it looks
6
+ * for) and the pages it opens are looked for in the app's source. A file whose
7
+ * names are all missing tests a feature not built yet, and is held back as
8
+ * pending rather than run to fail; a name missing with a close one in the code
9
+ * is a naming difference to settle. Once the code has them, the file runs. No AI.
10
+ */
11
+ import { existsSync, readdirSync, readFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ import { usedLabels } from '../gates/validate.js';
14
+ /** A name as the code would write it: no required-field star, no closing punctuation. */
15
+ const bare = (n) => n.replace(/\s*\*\s*$/, '').replace(/[\s.:!?…]+$/, '').replace(/\s+/g, ' ').trim();
16
+ /**
17
+ * What a test file needs the code to have: the names it locates elements by to
18
+ * act on them (role names, labels, placeholders, kit.find) and the pages it
19
+ * opens. Texts it only expects to see are left out: one the code lacks is a
20
+ * finding for the test to report, not a sign the feature is missing.
21
+ */
22
+ export function namesInTest(content) {
23
+ const names = new Set(usedLabels(content).map(bare).filter((n) => n.length >= 3));
24
+ const pages = new Set();
25
+ for (const m of content.matchAll(/\bgoto\(\s*[`'"](?:\$\{[^}]+\})?(\/[\w\-./]*)/g)) {
26
+ const p = m[1].replace(/\/+$/, '') || '/';
27
+ if (p !== '/')
28
+ pages.add(p);
29
+ }
30
+ return { names: [...names], pages: [...pages] };
31
+ }
32
+ /** The words that carry meaning: three letters or more. */
33
+ const words = (t) => new Set(t.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 3));
34
+ function closeness(a, b) {
35
+ const x = words(a);
36
+ const y = words(b);
37
+ let both = 0;
38
+ for (const w of x)
39
+ if (y.has(w))
40
+ both += 1;
41
+ return x.size + y.size ? both / (x.size + y.size - both) : 0;
42
+ }
43
+ /** Texts the code can show: JSX text and short string literals, for "did you mean". */
44
+ export function shownTexts(source) {
45
+ const out = new Set();
46
+ for (const m of source.matchAll(/>\s*([^<>{}\n]{2,60}?)\s*</g))
47
+ out.add(m[1].trim());
48
+ for (const m of source.matchAll(/(['"`])([^'"`\n]{2,60})\1/g))
49
+ if (/[a-z]/i.test(m[2]) && /\s|^[A-Z]/.test(m[2]))
50
+ out.add(m[2].trim());
51
+ return [...out];
52
+ }
53
+ /** The nearest text the code has to a name, when one is close enough to be the same thing renamed. */
54
+ export function nearestText(name, texts) {
55
+ let best = 0.3;
56
+ let nearest;
57
+ for (const t of texts) {
58
+ const c = closeness(name, t);
59
+ if (c > best) {
60
+ best = c;
61
+ nearest = t;
62
+ }
63
+ }
64
+ return nearest;
65
+ }
66
+ /** Does the app have this page: a route string in the source, or a Next.js / file-based route folder? */
67
+ function hasPage(path, haystack, appRoots) {
68
+ if (haystack.includes(`'${path}'`) || haystack.includes(`"${path}"`) || haystack.includes(`\`${path}`) || haystack.includes(`${path}/`))
69
+ return true;
70
+ const segs = path.replace(/^\//, '');
71
+ return appRoots.some((r) => ['app', 'pages', 'src/app', 'src/pages'].some((d) => existsSync(join(r, d, segs))));
72
+ }
73
+ /**
74
+ * Classify each test file: built (everything found), partial (some found),
75
+ * not-built (nothing found), renamed (a missing name has a close one in the code).
76
+ */
77
+ export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
78
+ let texts;
79
+ return files.map((file) => {
80
+ const { names, pages } = namesInTest(readFileSync(file, 'utf-8'));
81
+ const missing = [];
82
+ const renamed = [];
83
+ for (const n of names) {
84
+ if (haystack.includes(n.toLowerCase().replace(/\s+/g, ' ')))
85
+ continue;
86
+ texts ??= sourceTexts();
87
+ const near = nearestText(n, texts);
88
+ if (near)
89
+ renamed.push({ name: n, nearest: near });
90
+ else
91
+ missing.push(n);
92
+ }
93
+ for (const p of pages)
94
+ if (!hasPage(p, haystack, appRoots))
95
+ missing.push(p);
96
+ const total = names.length + pages.length;
97
+ const found = total - missing.length - renamed.length;
98
+ // Not built: the code has none of it. A different name or a partial match still runs, and fails if it must.
99
+ const state = !total || found === total ? 'built' : found === 0 && !renamed.length ? 'not-built' : renamed.length ? 'renamed' : 'partial';
100
+ return { file, state, missing, renamed };
101
+ });
102
+ }
103
+ /** The test files under a directory. */
104
+ export function testFilesIn(dir) {
105
+ const out = [];
106
+ const walk = (d) => {
107
+ for (const e of existsSync(d) ? readdirSync(d, { withFileTypes: true }) : []) {
108
+ const p = join(d, e.name);
109
+ if (e.isDirectory()) {
110
+ if (e.name !== 'helpers' && e.name !== 'node_modules')
111
+ walk(p);
112
+ }
113
+ else if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(e.name))
114
+ out.push(p);
115
+ }
116
+ };
117
+ walk(dir);
118
+ return out;
119
+ }
@@ -12,6 +12,8 @@
12
12
  * It runs from the area's nearest package root (where package.json /
13
13
  * playwright.config live), with config.envFile merged into the env.
14
14
  */
15
+ import { checkReadiness, testFilesIn } from './readiness.js';
16
+ import { sourceText } from '../docs/excerpts.js';
15
17
  import { spawnSync } from 'node:child_process';
16
18
  import { existsSync } from 'node:fs';
17
19
  import { join, dirname, resolve, relative } from 'node:path';
@@ -128,7 +130,27 @@ export function runTests(opts = {}) {
128
130
  const framework = resolveFramework(config, area, area);
129
131
  const root = packageRoot(testDir);
130
132
  const rel = relative(root, testDir) || '.';
131
- let cmd = runnerCommand(config, area, framework, root, rel, opts.journey);
133
+ // Tests written before their feature: hold back the files whose names and
134
+ // pages the code does not have yet, instead of running them to fail.
135
+ let targets;
136
+ if (framework === 'playwright' && !opts.all) {
137
+ const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
138
+ const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
139
+ const roots = ['src', ...(config.relatedDirs ?? [])];
140
+ const states = checkReadiness(files, sourceText(roots), () => [], [root]);
141
+ const held = states.filter((s) => s.state === 'not-built');
142
+ if (held.length) {
143
+ console.log(`\n${held.length} file(s) pending — the code has none of the elements or pages they use yet (\`testforge bind\` shows which; --all runs them anyway):`);
144
+ for (const h of held.slice(0, 15))
145
+ console.log(` not built ${relative(root, h.file)}: ${h.missing.slice(0, 3).map((m) => `"${m}"`).join(', ')}`);
146
+ targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
147
+ if (!targets.length) {
148
+ console.log(`[skip] ${area}: nothing to run yet`);
149
+ continue;
150
+ }
151
+ }
152
+ }
153
+ let cmd = runnerCommand(config, area, framework, root, rel, opts.journey, targets);
132
154
  if (opts.passThrough?.length)
133
155
  cmd += ` ${opts.passThrough.join(' ')}`;
134
156
  ranAnything = true;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "1.3.0",
3
+ "version": "2.0.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -374,12 +374,12 @@ function testRoot(root, category) {
374
374
  return d ? path.resolve(root, d) : undefined;
375
375
  }
376
376
 
377
- // The defect review's verdict for each failed test (test-artifacts/defect-review/*.json),
377
+ // The defect review's verdict for each failed test (testforge/runs/defect-review/*.json),
378
378
  // so the dashboard can say whether a failure is a product defect, a test fault or the
379
379
  // environment. Keyed by absolute file + test title; an absent or broken file means none.
380
380
  function reviewVerdicts(root) {
381
381
  const map = new Map();
382
- const dir = path.join(root || process.cwd(), 'test-artifacts', 'defect-review');
382
+ const dir = path.join(root || process.cwd(), 'testforge', 'runs', 'defect-review');
383
383
  let files = [];
384
384
  try { files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); } catch { /* no reviews yet */ }
385
385
  for (const f of files) {
@@ -8,7 +8,7 @@ const path = require('path');
8
8
  const crypto = require('crypto');
9
9
  const { findRoot } = require('./_shared.cjs');
10
10
 
11
- const dir = (root) => path.join(root || findRoot(process.cwd()), 'test-artifacts', 'run-data');
11
+ const dir = (root) => path.join(root || findRoot(process.cwd()), 'testforge', 'runs', 'run-data');
12
12
 
13
13
  /** The current run's id: <yyyymmdd>-<6 hex>, created once and kept in the env. */
14
14
  function runId() {