@activate-core/test-forge 1.2.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +67 -28
  2. package/dist/cli/index.js +98 -34
  3. package/dist/core/docs/cache.js +2 -1
  4. package/dist/core/docs/discover.js +51 -4
  5. package/dist/core/docs/doc-guidance.js +61 -23
  6. package/dist/core/docs/excerpts.js +245 -0
  7. package/dist/core/docs/grounding.js +51 -24
  8. package/dist/core/docs/reference-text.js +78 -0
  9. package/dist/core/docs/reference.js +68 -5
  10. package/dist/core/docs/review.js +81 -0
  11. package/dist/core/docs/source-of-truth.js +1 -1
  12. package/dist/core/docs/tickets.js +56 -0
  13. package/dist/core/gates/coverage.js +12 -19
  14. package/dist/core/gates/validate.js +277 -4
  15. package/dist/core/generate/ai-generator.js +243 -96
  16. package/dist/core/generate/ai-session.js +8 -0
  17. package/dist/core/generate/ai-usage.js +150 -0
  18. package/dist/core/generate/bind.js +91 -0
  19. package/dist/core/generate/defect-review.js +213 -20
  20. package/dist/core/generate/doc-diff.js +139 -0
  21. package/dist/core/generate/harness.js +52 -32
  22. package/dist/core/generate/journey-memory.js +325 -0
  23. package/dist/core/generate/plan-cache.js +114 -7
  24. package/dist/core/generate/required-data.js +75 -0
  25. package/dist/core/generate/review-journey.js +104 -0
  26. package/dist/core/generate/run-generate.js +193 -80
  27. package/dist/core/generate/seed-generator.js +2 -2
  28. package/dist/core/generate/suite-planner.js +138 -25
  29. package/dist/core/generate/test-blocks.js +84 -0
  30. package/dist/core/generate/update-journey.js +115 -0
  31. package/dist/core/generate/verify.js +9 -4
  32. package/dist/core/paths.js +90 -13
  33. package/dist/core/project/config.js +25 -2
  34. package/dist/core/project/env-diff.js +122 -0
  35. package/dist/core/project/env-template.js +3 -2
  36. package/dist/core/project/init.js +2 -2
  37. package/dist/core/project/map.js +2 -2
  38. package/dist/core/run/ci.js +94 -0
  39. package/dist/core/run/clean.js +111 -0
  40. package/dist/core/run/readiness.js +119 -0
  41. package/dist/core/run/report.js +2 -0
  42. package/dist/core/run/run-tests.js +23 -1
  43. package/package.json +1 -1
  44. package/reporters/_shared.cjs +24 -3
  45. package/reporters/jest-reporter.cjs +3 -1
  46. package/reporters/playwright-reporter.cjs +3 -1
  47. package/reporters/run-data.cjs +66 -0
  48. package/reporters/ui-helpers.cjs +133 -0
  49. package/dist/core/generate/verify-journey.js +0 -174
@@ -0,0 +1,122 @@
1
+ /**
2
+ * `testforge env --diff` — which variables the generated suite reads that the
3
+ * env file does not set, per journey, with the files that read each one.
4
+ *
5
+ * Only names are read from the env file, never values. A variable the code
6
+ * reads with a fallback (`process.env.X ?? 'default'`) is listed apart: the
7
+ * suite runs without it, on the default.
8
+ */
9
+ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
10
+ import { basename, join, relative } from 'node:path';
11
+ import { loadConfig } from './config.js';
12
+ import { extractEnvVars } from './env-template.js';
13
+ /** Variables a runner or the tool itself provides; never asked of a person. */
14
+ const PROVIDED = new Set(['CI', 'NODE_ENV', 'PWDEBUG', 'TESTFORGE_RUN_ID', 'TESTFORGE_CLEAN_FILE', 'HOME', 'PATH', 'TZ']);
15
+ /** Names set in an env file (blank values count as unset). */
16
+ export function envNames(text) {
17
+ const out = new Set();
18
+ for (const line of text.split('\n')) {
19
+ const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
20
+ if (m && m[2].trim().replace(/^(['"])\1$/, '') !== '')
21
+ out.add(m[1]);
22
+ }
23
+ return out;
24
+ }
25
+ const withDefault = (text, name) => new RegExp(`process\\.env(?:\\.${name}|\\[\\s*['"]${name}['"]\\s*\\])\\s*(?:\\?\\?|\\|\\|)`).test(text) ||
26
+ new RegExp(`env\\(\\s*['"]${name}['"]\\s*,\\s*[^)\\s]`).test(text);
27
+ function files(dir) {
28
+ const out = [];
29
+ const walk = (d) => {
30
+ for (const e of readdirSync(d, { withFileTypes: true })) {
31
+ const p = join(d, e.name);
32
+ if (e.isDirectory()) {
33
+ if (e.name !== 'node_modules' && !e.name.startsWith('.'))
34
+ walk(p);
35
+ }
36
+ else if (/\.(c|m)?[jt]sx?$/.test(e.name))
37
+ out.push(p);
38
+ }
39
+ };
40
+ if (existsSync(dir) && statSync(dir).isDirectory())
41
+ walk(dir);
42
+ return out;
43
+ }
44
+ /**
45
+ * For each journey folder under each area's test folder (and the shared
46
+ * helpers), the variables its files read that the env file does not set.
47
+ * `only` limits it to the named journey folders.
48
+ */
49
+ export function envDiff(root = '.', only) {
50
+ const config = loadConfig();
51
+ const envFile = config.envFile ?? '.env.testing';
52
+ const set = envNames(existsSync(join(root, envFile)) ? readFileSync(join(root, envFile), 'utf-8') : '');
53
+ const journeys = [];
54
+ // A variable the suite sets for itself at run time (the testkit's setup() exports
55
+ // tokens and ids — H4 — and runner configs set base URLs) is not asked of a person.
56
+ const assigned = new Set();
57
+ const dirs = Object.values(config.testOutput ?? {}).flatMap((d) => [join(root, d), join(root, d, '..')]);
58
+ for (const f of [...new Set(dirs.flatMap((d) => files(d).filter((x) => /helpers|config|setup/i.test(x))))]) {
59
+ const text = readFileSync(f, 'utf-8');
60
+ for (const m of text.matchAll(/process\.env(?:\.([A-Z0-9_]+)|\[\s*['"]([A-Z0-9_]+)['"]\s*\])\s*(?:=(?!=)|\?\?=|\|\|=)/g))
61
+ assigned.add(m[1] ?? m[2]);
62
+ // setup() often sets several at once: process.env[`TEST_${k}`] = … or Object.assign(process.env, {...}).
63
+ for (const m of text.matchAll(/Object\.assign\(\s*process\.env\s*,\s*\{([^}]*)\}/g))
64
+ for (const k of m[1].matchAll(/([A-Z][A-Z0-9_]+)\s*:/g))
65
+ assigned.add(k[1]);
66
+ // …or loops over an object of them: for (const [k, v] of Object.entries(publish)) … process.env[k] = …
67
+ for (const m of text.matchAll(/Object\.entries\(\s*(\w+)\s*\)[^;{]*\)?\s*[^;]*process\.env\[\s*\w+\s*\]\s*=(?!=)/g)) {
68
+ const obj = new RegExp(`(?:const|let|var)\\s+${m[1]}\\s*=\\s*\\{([\\s\\S]*?)\\n\\s*\\};`).exec(text);
69
+ if (obj)
70
+ for (const k of obj[1].matchAll(/^\s*([A-Z][A-Z0-9_]+)\s*:/gm))
71
+ assigned.add(k[1]);
72
+ }
73
+ }
74
+ for (const testDir of Object.values(config.testOutput ?? {})) {
75
+ const abs = join(root, testDir);
76
+ for (const f of files(abs)) {
77
+ const rel = relative(abs, f);
78
+ const first = rel.split(/[\\/]/)[0];
79
+ const journey = rel.includes('/') || rel.includes('\\') ? (first === 'helpers' ? 'shared (testkit and helpers)' : first) : basename(testDir);
80
+ if (only && !only.includes(first))
81
+ continue;
82
+ const text = readFileSync(f, 'utf-8');
83
+ for (const name of extractEnvVars(text)) {
84
+ if (set.has(name) || PROVIDED.has(name) || assigned.has(name))
85
+ continue;
86
+ let j = journeys.find((x) => x.journey === journey);
87
+ if (!j)
88
+ journeys.push((j = { journey, missing: [] }));
89
+ let v = j.missing.find((x) => x.name === name);
90
+ if (!v)
91
+ j.missing.push((v = { name, files: [], hasDefault: true }));
92
+ if (!v.files.includes(basename(f)))
93
+ v.files.push(basename(f));
94
+ v.hasDefault = v.hasDefault && withDefault(text, name);
95
+ }
96
+ }
97
+ }
98
+ for (const j of journeys)
99
+ j.missing.sort((a, b) => Number(a.hasDefault) - Number(b.hasDefault) || a.name.localeCompare(b.name));
100
+ journeys.sort((a, b) => a.journey.localeCompare(b.journey));
101
+ return { envFile, journeys };
102
+ }
103
+ export function describeEnvDiff(d) {
104
+ const needed = d.journeys.flatMap((j) => j.missing.filter((v) => !v.hasDefault));
105
+ const optional = d.journeys.flatMap((j) => j.missing.filter((v) => v.hasDefault));
106
+ if (!needed.length && !optional.length)
107
+ return `Every variable the suite reads is set in ${d.envFile}.`;
108
+ const lines = [`Not set in ${d.envFile} (names only; values are never read):`];
109
+ for (const j of d.journeys) {
110
+ const req = j.missing.filter((v) => !v.hasDefault);
111
+ if (!req.length)
112
+ continue;
113
+ lines.push(` ${j.journey}`);
114
+ for (const v of req)
115
+ lines.push(` ${v.name.padEnd(34)} used by ${v.files.slice(0, 3).join(', ')}${v.files.length > 3 ? ` +${v.files.length - 3}` : ''}`);
116
+ }
117
+ const uniq = (xs) => new Set(xs.map((v) => v.name)).size;
118
+ lines.push('', `${uniq(needed)} variable(s) to set. Tests that need one skip and name it until it is set.`);
119
+ if (optional.length)
120
+ lines.push(`${uniq(optional)} more have a default in the code and can stay unset: ${[...new Set(optional.map((v) => v.name))].slice(0, 12).join(', ')}${uniq(optional) > 12 ? ', …' : ''}`);
121
+ return lines.join('\n');
122
+ }
@@ -14,10 +14,10 @@
14
14
  import { readFileSync, writeFileSync, existsSync, readdirSync, } from 'node:fs';
15
15
  import { join } from 'node:path';
16
16
  import { findMarkdownFiles } from '../find-files.js';
17
- import { STATE_DIR, LEGACY_STATE_DIR } from '../paths.js';
17
+ import { STATE_DIR, LEGACY_STATE_DIR, OLD_STATE_DIR } from '../paths.js';
18
18
  import { loadConfig } from './config.js';
19
19
  /** Pull env var names from a chunk of text. */
20
- function extractEnvVars(text) {
20
+ export function extractEnvVars(text) {
21
21
  const vars = new Set();
22
22
  // process.env.XXX and process.env["XXX"] / process.env['XXX']
23
23
  for (const m of text.matchAll(/process\.env\.([A-Z0-9_]+)/gi)) {
@@ -53,6 +53,7 @@ const SKIP_DIRS = new Set([
53
53
  'generated-tests',
54
54
  STATE_DIR,
55
55
  LEGACY_STATE_DIR,
56
+ OLD_STATE_DIR,
56
57
  ]);
57
58
  const SOURCE_EXT = /\.(c|m)?[jt]sx?$/i;
58
59
  /**
@@ -10,8 +10,8 @@
10
10
  * Nothing is hardcoded for a specific framework or layout; the AI
11
11
  * infers it from what the project actually looks like.
12
12
  */
13
- import { execFileSync } from 'node:child_process';
14
13
  import { loadConfig, splitCommand, withEditTools } from './config.js';
14
+ import { runAiSync } from '../generate/ai-usage.js';
15
15
  /** Build the prompt that asks the AI to write testforge.config.json. */
16
16
  export function buildInitPrompt() {
17
17
  return [
@@ -71,7 +71,7 @@ export function buildInitPrompt() {
71
71
  export function runInit() {
72
72
  const prompt = buildInitPrompt();
73
73
  const { program, args } = withEditTools(splitCommand(loadConfig().aiCommand));
74
- const output = execFileSync(program, args, {
74
+ const output = runAiSync(program, args, {
75
75
  input: prompt,
76
76
  encoding: 'utf-8',
77
77
  maxBuffer: 10 * 1024 * 1024,
@@ -17,8 +17,8 @@
17
17
  * whatever the project actually uses. Secrets are never captured;
18
18
  * only env var names.
19
19
  */
20
- import { execFileSync } from 'node:child_process';
21
20
  import { loadConfig, splitCommand, withEditTools } from './config.js';
21
+ import { runAiSync } from '../generate/ai-usage.js';
22
22
  /** Build the prompt that asks the AI to produce the project map. */
23
23
  export function buildMapPrompt(srcDir, relatedDirs = []) {
24
24
  const related = relatedDirs.length
@@ -93,7 +93,7 @@ export function runMap(srcDir) {
93
93
  const config = loadConfig();
94
94
  const prompt = buildMapPrompt(srcDir, config.relatedDirs ?? []);
95
95
  const { program, args } = withEditTools(splitCommand(config.aiCommand), config.relatedDirs ?? []);
96
- const output = execFileSync(program, args, {
96
+ const output = runAiSync(program, args, {
97
97
  input: prompt,
98
98
  encoding: 'utf-8',
99
99
  maxBuffer: 10 * 1024 * 1024,
@@ -0,0 +1,94 @@
1
+ /**
2
+ * `testforge ci --github` — a GitHub Actions workflow that runs the generated
3
+ * tests against a deployed environment and publishes the result.
4
+ *
5
+ * Every value the suite needs comes from repository secrets of the same
6
+ * name: the names are read from the env file (names only, never values), so
7
+ * the workflow asks for exactly what a local run uses. Nothing here names a
8
+ * product or a stack beyond the runners TestForge generates for.
9
+ */
10
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
11
+ import { dirname, join } from 'node:path';
12
+ import { loadConfig, resolveFramework } from '../project/config.js';
13
+ import { envNames } from '../project/env-diff.js';
14
+ export const WORKFLOW_PATH = join('.github', 'workflows', 'testforge.yml');
15
+ /** --schedule values: a name or a five-field cron line. Unset or "none": manual only. */
16
+ export function scheduleCron(v) {
17
+ if (!v || v === 'none')
18
+ return undefined;
19
+ const named = { daily: '0 6 * * *', weekdays: '0 6 * * 1-5', weekly: '0 6 * * 1' };
20
+ if (named[v])
21
+ return named[v];
22
+ if (/^\S+\s+\S+\s+\S+\s+\S+\s+\S+$/.test(v.trim()))
23
+ return v.trim();
24
+ throw new Error(`--schedule takes none, daily, weekdays, weekly or a cron line like "0 6 * * 1": got "${v}"`);
25
+ }
26
+ export function workflowYaml(o) {
27
+ const env = [
28
+ ` TESTFORGE_KEY: \${{ secrets.TESTFORGE_KEY }}`,
29
+ ` TESTFORGE_DASHBOARD_URL: \${{ vars.TESTFORGE_DASHBOARD_URL }}`,
30
+ ` TESTFORGE_ENV: ${o.envName}`,
31
+ ` CI: 'true'`,
32
+ ...o.secrets.filter((n) => !/^(TESTFORGE_KEY|TESTFORGE_ENV|TESTFORGE_DASHBOARD_URL|CI)$/.test(n)).map((n) => ` ${n}: \${{ secrets.${n} }}`),
33
+ ];
34
+ return [
35
+ `# @generated by testforge ci --github`,
36
+ `# Runs the TestForge suite against a deployed environment and publishes the`,
37
+ `# result to the dashboard. Add each name under "env" as a repository secret`,
38
+ `# (Settings → Secrets and variables → Actions), and the dashboard address as`,
39
+ `# the repository variable TESTFORGE_DASHBOARD_URL.`,
40
+ `name: TestForge`,
41
+ ``,
42
+ `on:`,
43
+ ` workflow_dispatch: {} # Actions tab → TestForge → Run workflow`,
44
+ ...(o.schedule ? [` schedule:`, ` - cron: '${o.schedule}' # UTC`] : [` # No schedule: add one with testforge ci --github --schedule weekdays --force`]),
45
+ ``,
46
+ `concurrency:`,
47
+ ` group: testforge-\${{ github.ref }}`,
48
+ ` cancel-in-progress: false`,
49
+ ``,
50
+ `jobs:`,
51
+ ` test:`,
52
+ ` runs-on: ubuntu-latest`,
53
+ ` timeout-minutes: 180`,
54
+ ` env:`,
55
+ ...env,
56
+ ` steps:`,
57
+ ` - uses: actions/checkout@v4`,
58
+ ` - uses: actions/setup-node@v4`,
59
+ ` with:`,
60
+ ` node-version: ${o.node ?? 20}`,
61
+ ` cache: npm`,
62
+ ` - run: npm ci`,
63
+ ...(o.playwright ? [` - run: npx playwright install --with-deps chromium`] : []),
64
+ ` - run: npx testforge run`,
65
+ ` - if: always()`,
66
+ ` uses: actions/upload-artifact@v4`,
67
+ ` with:`,
68
+ ` name: testforge-results`,
69
+ ` path: testforge/runs/`,
70
+ ` retention-days: 14`,
71
+ ``,
72
+ ].join('\n');
73
+ }
74
+ /** Write the workflow for this project. Refuses to replace one without force. */
75
+ export function writeGithubWorkflow(opts = {}) {
76
+ const cron = scheduleCron(opts.schedule);
77
+ const root = opts.root ?? '.';
78
+ const path = join(root, WORKFLOW_PATH);
79
+ if (existsSync(path) && !opts.force)
80
+ return `${WORKFLOW_PATH} already exists — re-run with --force to replace it.`;
81
+ const config = loadConfig();
82
+ const envFile = config.envFile ?? '.env.testing';
83
+ const names = existsSync(join(root, envFile)) ? [...envNames(readFileSync(join(root, envFile), 'utf-8'))] : [];
84
+ const playwright = Object.keys(config.testOutput ?? {}).some((a) => resolveFramework(config, a, '') === 'playwright');
85
+ mkdirSync(dirname(path), { recursive: true });
86
+ writeFileSync(path, workflowYaml({ secrets: names.sort(), playwright, envName: opts.envName ?? 'staging', schedule: cron }), 'utf-8');
87
+ return [
88
+ `Wrote ${WORKFLOW_PATH}. ${cron ? `It runs on the schedule "${cron}" (UTC) and when started by hand.` : 'It runs only when started by hand (Actions tab → TestForge → Run workflow).'}`,
89
+ `Add these repository secrets (same names as ${envFile}, values for the deployed environment):`,
90
+ ` TESTFORGE_KEY${names.length ? `, ${names.join(', ')}` : ''}`,
91
+ `and the repository variable TESTFORGE_DASHBOARD_URL (the deployed dashboard).`,
92
+ `Point the base URL variables at the deployed app, not localhost.`,
93
+ ].join('\n');
94
+ }
@@ -0,0 +1,111 @@
1
+ /**
2
+ * `testforge clean` — remove what earlier runs created, through the product's
3
+ * own API, using the requests the testkit recorded (H8).
4
+ *
5
+ * Without --yes it only lists. With --yes it runs one throw-away test per
6
+ * testkit that signs in through that testkit and sends each recorded cleanup
7
+ * request; the result is reported per record and nothing is published.
8
+ * Gated like reset(): TESTFORGE_ALLOW_RESET must be "true".
9
+ */
10
+ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
11
+ import { createRequire } from 'node:module';
12
+ import { dirname, join, relative, resolve } from 'node:path';
13
+ import { spawnSync } from 'node:child_process';
14
+ import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
15
+ import { harnessInfo } from '../generate/harness.js';
16
+ import { packageRoot } from './run-tests.js';
17
+ const runData = createRequire(import.meta.url)('../../../reporters/run-data.cjs');
18
+ export function pickRecords(opts, now = Date.now()) {
19
+ const root = opts.root ?? process.cwd();
20
+ const runs = runData.listRuns(root);
21
+ const cutoff = opts.olderThanDays != null ? now - opts.olderThanDays * 86_400_000 : undefined;
22
+ const chosen = runs.filter((r) => (opts.runId ? r.runId === opts.runId : cutoff != null ? Date.parse(r.at) < cutoff : false));
23
+ return chosen.flatMap((r) => runData.readRun(r.runId, root));
24
+ }
25
+ const CLEAN_TEST = (kitRel) => [
26
+ `// @ts-nocheck`,
27
+ `// @generated by testforge clean — deleted after this run`,
28
+ `const kit = require('${kitRel}');`,
29
+ `const fs = require('fs');`,
30
+ `const t = typeof test !== 'undefined' ? test : require('@playwright/test').test;`,
31
+ `t('testforge clean', async () => {`,
32
+ ` const file = process.env.TESTFORGE_CLEAN_FILE;`,
33
+ ` const out = [];`,
34
+ ` for (const r of JSON.parse(fs.readFileSync(file, 'utf8'))) {`,
35
+ ` try {`,
36
+ ` const token = await kit.ensureToken(r.role || 'customer');`,
37
+ ` const res = await fetch(r.cleanup.url, { method: r.cleanup.method || 'DELETE', headers: { Authorization: 'Bearer ' + token, Accept: 'application/json' } });`,
38
+ ` out.push({ kind: r.kind, id: r.id, status: res.status });`,
39
+ ` } catch (e) {`,
40
+ ` out.push({ kind: r.kind, id: r.id, error: String((e && e.message) || e) });`,
41
+ ` }`,
42
+ ` }`,
43
+ ` fs.writeFileSync(file + '.out', JSON.stringify(out));`,
44
+ `});`,
45
+ ``,
46
+ ].join('\n');
47
+ export async function runClean(opts) {
48
+ const lines = [];
49
+ if (!opts.runId && opts.olderThanDays == null) {
50
+ const runs = runData.listRuns(process.cwd());
51
+ if (!runs.length)
52
+ return 'No run has recorded any data yet (testforge/runs/run-data/ is empty).';
53
+ return ['Runs with recorded data:', ...runs.map((r) => ` ${r.runId} ${r.at.slice(0, 16).replace('T', ' ')} ${r.count} record(s)`),
54
+ '', 'Clean one with --run <id>, or every run older than N days with --older-than <N>. Add --yes to remove.'].join('\n');
55
+ }
56
+ const recs = pickRecords(opts);
57
+ if (!recs.length)
58
+ return 'No recorded data matches.';
59
+ const removable = recs.filter((r) => r.cleanup?.url && r.kit);
60
+ const stuck = recs.filter((r) => !(r.cleanup?.url && r.kit));
61
+ lines.push(`${recs.length} record(s): ${removable.length} removable through the product's API, ${stuck.length} not.`);
62
+ for (const r of stuck)
63
+ lines.push(` keep ${r.kind} ${String(r.id)} — the product has no request that removes it`);
64
+ if (!opts.yes) {
65
+ for (const r of removable)
66
+ lines.push(` would ${r.cleanup.method || 'DELETE'} ${r.kind} ${String(r.id)}`);
67
+ lines.push('', 'Dry run. Add --yes to remove them.');
68
+ return lines.join('\n');
69
+ }
70
+ if (process.env.TESTFORGE_ALLOW_RESET !== 'true') {
71
+ lines.push('', 'Refused: set TESTFORGE_ALLOW_RESET=true (the same gate as the testkit reset) to remove test data.');
72
+ return lines.join('\n');
73
+ }
74
+ // One throw-away test per testkit, run by that area's own runner.
75
+ const config = loadConfig();
76
+ const byKit = new Map();
77
+ for (const r of removable)
78
+ byKit.set(resolve(r.kit), [...(byKit.get(resolve(r.kit)) ?? []), r]);
79
+ for (const [kitPath, rows] of byKit) {
80
+ const area = Object.keys(config.testOutput ?? {}).find((a) => {
81
+ const fw = resolveFramework(config, a, '') === 'jest' ? 'jest' : 'playwright';
82
+ return resolve(harnessInfo(testDirForType(config, a), fw, config.testExtension || '.test.js').path) === kitPath;
83
+ });
84
+ if (!area) {
85
+ lines.push(` skipped ${rows.length} record(s): ${kitPath} is not a testkit of this project`);
86
+ continue;
87
+ }
88
+ const areaDir = resolve(testDirForType(config, area));
89
+ const jest = resolveFramework(config, area, '') === 'jest';
90
+ const ext = (config.testExtension || '.test.js').replace(/^\.?(test|spec)/, '.test');
91
+ const file = join(areaDir, `tf-clean${ext.startsWith('.test') ? ext : '.test.js'}`);
92
+ const data = join(areaDir, '.tf-clean.json');
93
+ mkdirSync(dirname(file), { recursive: true });
94
+ let kitRel = relative(areaDir, kitPath).replace(/\.[cm]?[jt]s$/, '');
95
+ if (!kitRel.startsWith('.'))
96
+ kitRel = `./${kitRel}`;
97
+ writeFileSync(file, CLEAN_TEST(kitRel), 'utf-8');
98
+ writeFileSync(data, JSON.stringify(rows), 'utf-8');
99
+ const cwd = packageRoot(areaDir);
100
+ const args = jest ? ['jest', relative(cwd, file), '--runInBand', '--reporters=default'] : ['playwright', 'test', relative(cwd, file), '--reporter=list', '--workers=1'];
101
+ spawnSync('npx', args, { cwd, stdio: 'inherit', env: { ...process.env, TESTFORGE_CLEAN_FILE: data } });
102
+ const out = existsSync(`${data}.out`) ? JSON.parse(readFileSync(`${data}.out`, 'utf-8')) : [];
103
+ for (const o of out)
104
+ lines.push(` ${o.error ? 'failed ' : o.status < 300 ? 'removed' : `HTTP ${o.status}`} ${o.kind} ${String(o.id)}${o.error ? ` — ${o.error}` : ''}`);
105
+ if (!out.length)
106
+ lines.push(` ${area}: the clean test did not report back`);
107
+ for (const f of [file, data, `${data}.out`])
108
+ rmSync(f, { force: true });
109
+ }
110
+ return lines.join('\n');
111
+ }
@@ -0,0 +1,119 @@
1
+ /**
2
+ * testforge — has the feature a test file checks been built yet?
3
+ *
4
+ * Tests are written from the doc, often before the code exists. Before a run,
5
+ * each file's names (the buttons, links, fields, headings, images it looks
6
+ * for) and the pages it opens are looked for in the app's source. A file whose
7
+ * names are all missing tests a feature not built yet, and is held back as
8
+ * pending rather than run to fail; a name missing with a close one in the code
9
+ * is a naming difference to settle. Once the code has them, the file runs. No AI.
10
+ */
11
+ import { existsSync, readdirSync, readFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ import { usedLabels } from '../gates/validate.js';
14
+ /** A name as the code would write it: no required-field star, no closing punctuation. */
15
+ const bare = (n) => n.replace(/\s*\*\s*$/, '').replace(/[\s.:!?…]+$/, '').replace(/\s+/g, ' ').trim();
16
+ /**
17
+ * What a test file needs the code to have: the names it locates elements by to
18
+ * act on them (role names, labels, placeholders, kit.find) and the pages it
19
+ * opens. Texts it only expects to see are left out: one the code lacks is a
20
+ * finding for the test to report, not a sign the feature is missing.
21
+ */
22
+ export function namesInTest(content) {
23
+ const names = new Set(usedLabels(content).map(bare).filter((n) => n.length >= 3));
24
+ const pages = new Set();
25
+ for (const m of content.matchAll(/\bgoto\(\s*[`'"](?:\$\{[^}]+\})?(\/[\w\-./]*)/g)) {
26
+ const p = m[1].replace(/\/+$/, '') || '/';
27
+ if (p !== '/')
28
+ pages.add(p);
29
+ }
30
+ return { names: [...names], pages: [...pages] };
31
+ }
32
+ /** The words that carry meaning: three letters or more. */
33
+ const words = (t) => new Set(t.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 3));
34
+ function closeness(a, b) {
35
+ const x = words(a);
36
+ const y = words(b);
37
+ let both = 0;
38
+ for (const w of x)
39
+ if (y.has(w))
40
+ both += 1;
41
+ return x.size + y.size ? both / (x.size + y.size - both) : 0;
42
+ }
43
+ /** Texts the code can show: JSX text and short string literals, for "did you mean". */
44
+ export function shownTexts(source) {
45
+ const out = new Set();
46
+ for (const m of source.matchAll(/>\s*([^<>{}\n]{2,60}?)\s*</g))
47
+ out.add(m[1].trim());
48
+ for (const m of source.matchAll(/(['"`])([^'"`\n]{2,60})\1/g))
49
+ if (/[a-z]/i.test(m[2]) && /\s|^[A-Z]/.test(m[2]))
50
+ out.add(m[2].trim());
51
+ return [...out];
52
+ }
53
+ /** The nearest text the code has to a name, when one is close enough to be the same thing renamed. */
54
+ export function nearestText(name, texts) {
55
+ let best = 0.3;
56
+ let nearest;
57
+ for (const t of texts) {
58
+ const c = closeness(name, t);
59
+ if (c > best) {
60
+ best = c;
61
+ nearest = t;
62
+ }
63
+ }
64
+ return nearest;
65
+ }
66
+ /** Does the app have this page: a route string in the source, or a Next.js / file-based route folder? */
67
+ function hasPage(path, haystack, appRoots) {
68
+ if (haystack.includes(`'${path}'`) || haystack.includes(`"${path}"`) || haystack.includes(`\`${path}`) || haystack.includes(`${path}/`))
69
+ return true;
70
+ const segs = path.replace(/^\//, '');
71
+ return appRoots.some((r) => ['app', 'pages', 'src/app', 'src/pages'].some((d) => existsSync(join(r, d, segs))));
72
+ }
73
+ /**
74
+ * Classify each test file: built (everything found), partial (some found),
75
+ * not-built (nothing found), renamed (a missing name has a close one in the code).
76
+ */
77
+ export function checkReadiness(files, haystack, sourceTexts, appRoots = ['.']) {
78
+ let texts;
79
+ return files.map((file) => {
80
+ const { names, pages } = namesInTest(readFileSync(file, 'utf-8'));
81
+ const missing = [];
82
+ const renamed = [];
83
+ for (const n of names) {
84
+ if (haystack.includes(n.toLowerCase().replace(/\s+/g, ' ')))
85
+ continue;
86
+ texts ??= sourceTexts();
87
+ const near = nearestText(n, texts);
88
+ if (near)
89
+ renamed.push({ name: n, nearest: near });
90
+ else
91
+ missing.push(n);
92
+ }
93
+ for (const p of pages)
94
+ if (!hasPage(p, haystack, appRoots))
95
+ missing.push(p);
96
+ const total = names.length + pages.length;
97
+ const found = total - missing.length - renamed.length;
98
+ // Not built: the code has none of it. A different name or a partial match still runs, and fails if it must.
99
+ const state = !total || found === total ? 'built' : found === 0 && !renamed.length ? 'not-built' : renamed.length ? 'renamed' : 'partial';
100
+ return { file, state, missing, renamed };
101
+ });
102
+ }
103
+ /** The test files under a directory. */
104
+ export function testFilesIn(dir) {
105
+ const out = [];
106
+ const walk = (d) => {
107
+ for (const e of existsSync(d) ? readdirSync(d, { withFileTypes: true }) : []) {
108
+ const p = join(d, e.name);
109
+ if (e.isDirectory()) {
110
+ if (e.name !== 'helpers' && e.name !== 'node_modules')
111
+ walk(p);
112
+ }
113
+ else if (/\.(test|spec)\.[cm]?[jt]sx?$/.test(e.name))
114
+ out.push(p);
115
+ }
116
+ };
117
+ walk(dir);
118
+ return out;
119
+ }
@@ -67,6 +67,7 @@ export function mapPlaywright(json, category = 'web') {
67
67
  const suites = {};
68
68
  // Spec files are relative to the config's testDir; the report records the root it used.
69
69
  const rootDir = (json.config && json.config.rootDir) || process.cwd();
70
+ const verdictOf = shared.reviewVerdicts(shared.findRoot(process.cwd()));
70
71
  const walk = (suite) => {
71
72
  for (const spec of suite.specs || []) {
72
73
  const name = shared.suiteName(spec.file || suite.file || 'suite');
@@ -95,6 +96,7 @@ export function mapPlaywright(json, category = 'web') {
95
96
  error: status === 'failed' ? (strip(raw || '').split('\n').find((l) => l.trim()) || '').slice(0, 200) : undefined,
96
97
  sig: status === 'failed' ? shared.sig(raw) : undefined,
97
98
  note: status === 'skipped' ? (pend.find((a) => a.description) || {}).description || 'Skipped — not run in this execution' : undefined,
99
+ ...(status === 'failed' ? verdictOf(loc, spec.title) : {}),
98
100
  });
99
101
  }
100
102
  }
@@ -12,6 +12,8 @@
12
12
  * It runs from the area's nearest package root (where package.json /
13
13
  * playwright.config live), with config.envFile merged into the env.
14
14
  */
15
+ import { checkReadiness, testFilesIn } from './readiness.js';
16
+ import { sourceText } from '../docs/excerpts.js';
15
17
  import { spawnSync } from 'node:child_process';
16
18
  import { existsSync } from 'node:fs';
17
19
  import { join, dirname, resolve, relative } from 'node:path';
@@ -128,7 +130,27 @@ export function runTests(opts = {}) {
128
130
  const framework = resolveFramework(config, area, area);
129
131
  const root = packageRoot(testDir);
130
132
  const rel = relative(root, testDir) || '.';
131
- let cmd = runnerCommand(config, area, framework, root, rel, opts.journey);
133
+ // Tests written before their feature: hold back the files whose names and
134
+ // pages the code does not have yet, instead of running them to fail.
135
+ let targets;
136
+ if (framework === 'playwright' && !opts.all) {
137
+ const scope = opts.journey && existsSync(join(testDir, opts.journey)) ? join(testDir, opts.journey) : testDir;
138
+ const files = testFilesIn(scope).filter((f) => !opts.journey || f.includes(opts.journey));
139
+ const roots = ['src', ...(config.relatedDirs ?? [])];
140
+ const states = checkReadiness(files, sourceText(roots), () => [], [root]);
141
+ const held = states.filter((s) => s.state === 'not-built');
142
+ if (held.length) {
143
+ console.log(`\n${held.length} file(s) pending — the code has none of the elements or pages they use yet (\`testforge bind\` shows which; --all runs them anyway):`);
144
+ for (const h of held.slice(0, 15))
145
+ console.log(` not built ${relative(root, h.file)}: ${h.missing.slice(0, 3).map((m) => `"${m}"`).join(', ')}`);
146
+ targets = states.filter((s) => !held.includes(s)).map((s) => s.file);
147
+ if (!targets.length) {
148
+ console.log(`[skip] ${area}: nothing to run yet`);
149
+ continue;
150
+ }
151
+ }
152
+ }
153
+ let cmd = runnerCommand(config, area, framework, root, rel, opts.journey, targets);
132
154
  if (opts.passThrough?.length)
133
155
  cmd += ` ${opts.passThrough.join(' ')}`;
134
156
  ranAnything = true;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@activate-core/test-forge",
3
- "version": "1.2.2",
3
+ "version": "2.0.0",
4
4
  "description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -332,7 +332,8 @@ async function publish(category, suites, durationSec) {
332
332
  const root = findRoot(process.cwd());
333
333
  loadEnv(root);
334
334
  const cfg = loadConfig(root);
335
- const project = cfg.project, dashboardUrl = cfg.dashboardUrl, key = process.env.TESTFORGE_KEY;
335
+ // In CI the deployed dashboard comes from the environment; locally, from the config.
336
+ const project = cfg.project, dashboardUrl = process.env.TESTFORGE_DASHBOARD_URL || cfg.dashboardUrl, key = process.env.TESTFORGE_KEY;
336
337
  if (!project || !dashboardUrl || !key) {
337
338
  console.warn('[testforge] skipped publish — need "project" + "dashboardUrl" in testforge.config.json and TESTFORGE_KEY in .testforge.env');
338
339
  return;
@@ -343,7 +344,7 @@ async function publish(category, suites, durationSec) {
343
344
  const res = await fetch(dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
344
345
  method: 'POST',
345
346
  headers: { Authorization: 'Bearer ' + key, 'Content-Type': 'application/json' },
346
- body: JSON.stringify({ project, env, branch, sha, ciBuild, durationSec, suites }),
347
+ body: JSON.stringify({ project, env, branch, sha, ciBuild, durationSec, suites, runId: process.env.TESTFORGE_RUN_ID || undefined }),
347
348
  });
348
349
  if (res.ok) console.log(`[testforge] ✓ published ${category} run (env: ${env}${branch ? ` · ${branch}` : ''}) → ${dashboardUrl}`);
349
350
  else console.warn(`[testforge] publish failed (${res.status}): ${(await res.text()).slice(0, 200)}`);
@@ -373,4 +374,24 @@ function testRoot(root, category) {
373
374
  return d ? path.resolve(root, d) : undefined;
374
375
  }
375
376
 
376
- module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot };
377
+ // The defect review's verdict for each failed test (testforge/runs/defect-review/*.json),
378
+ // so the dashboard can say whether a failure is a product defect, a test fault or the
379
+ // environment. Keyed by absolute file + test title; an absent or broken file means none.
380
+ function reviewVerdicts(root) {
381
+ const map = new Map();
382
+ const dir = path.join(root || process.cwd(), 'testforge', 'runs', 'defect-review');
383
+ let files = [];
384
+ try { files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); } catch { /* no reviews yet */ }
385
+ for (const f of files) {
386
+ try {
387
+ for (const r of JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) || []) {
388
+ for (const t of r.tests || []) {
389
+ map.set(`${path.resolve(String(t.file))}|${t.title}`, { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) });
390
+ }
391
+ }
392
+ } catch { /* a half-written review is ignored */ }
393
+ }
394
+ return (file, title) => (file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined);
395
+ }
396
+
397
+ module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts };
@@ -1,12 +1,13 @@
1
1
  // TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
2
2
  // Add to jest.config.js:
3
3
  // reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
4
- const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot } = require('./_shared.cjs');
4
+ const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
5
5
 
6
6
  module.exports = class TestForgeJestReporter {
7
7
  constructor(_globalConfig, options) {
8
8
  this.category = (options && options.category) || 'api';
9
9
  this.testRoot = testRoot(findRoot(process.cwd()), this.category);
10
+ this.verdictOf = reviewVerdicts(findRoot(process.cwd()));
10
11
  }
11
12
  async onRunComplete(_contexts, results) {
12
13
  const suites = {};
@@ -27,6 +28,7 @@ module.exports = class TestForgeJestReporter {
27
28
  checks: testChecks(f.testFilePath, a.location && a.location.line),
28
29
  ...(status === 'failed' ? expectation(raw) : {}),
29
30
  detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
31
+ ...(status === 'failed' ? this.verdictOf(f.testFilePath, a.fullName || a.title) : {}),
30
32
  });
31
33
  }
32
34
  }