@activate-core/test-forge 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,10 +14,10 @@
14
14
  * Secrets are never embedded: the connection comes from the env var
15
15
  * named in the map (e.g. process.env.DB_URL), not a literal string.
16
16
  */
17
- import { execFileSync } from 'node:child_process';
18
17
  import { readFileSync, existsSync } from 'node:fs';
19
18
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
20
19
  import { cleanOutput } from './ai-generator.js';
20
+ import { runAiSync } from './ai-usage.js';
21
21
  const MAP_PATH = 'testforge.md';
22
22
  /** Build the seed-generation prompt for one journey. */
23
23
  export function buildSeedPrompt(journeyDoc, seedPath) {
@@ -96,7 +96,7 @@ export function generateSeed(journeyDoc, seedPath) {
96
96
  // Allow read-only exploration so the seed matches the REAL schema
97
97
  // (models/collections/enums) read from the code; Write stays denied.
98
98
  const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
99
- const output = execFileSync(program, args, {
99
+ const output = runAiSync(program, args, {
100
100
  input: prompt,
101
101
  encoding: 'utf-8',
102
102
  maxBuffer: 10 * 1024 * 1024,
@@ -14,11 +14,11 @@
14
14
  * The planner returns structured data (a list of files), NOT code — the
15
15
  * per-file generator turns each entry into an actual focused test.
16
16
  */
17
- import { execFileSync } from 'node:child_process';
18
17
  import { readFileSync, existsSync, readdirSync } from 'node:fs';
19
18
  import { join } from 'node:path';
20
19
  import { loadConfig, splitCommand, withReadOnlyTools } from '../project/config.js';
21
20
  import { PLANS_DIR } from '../paths.js';
21
+ import { runAiSync } from './ai-usage.js';
22
22
  const MAP_PATH = 'testforge.md';
23
23
  /**
24
24
  * The other journeys in the same area — their names, and the files already
@@ -89,6 +89,14 @@ export function buildPlanPrompt(journey, map, siblings = '') {
89
89
  ` "covers" ("opening the login form from X and landing on X after`,
90
90
  ` login — the form's own validation is owned by <journey>").`,
91
91
  `- Within this journey, no two files test the same behaviour.`,
92
+ `- ONE COMPONENT, ONE TEST OF IT. A behaviour that comes from one shared`,
93
+ ` component or layer — a pop-up closing with Escape, x or the overlay; cards`,
94
+ ` or tiles built the same way being reachable by keyboard; a wording rule`,
95
+ ` applied across pages; the message a failed request shows — is planned as`,
96
+ ` ONE file, in the journey that owns the component, checking one`,
97
+ ` representative screen per distinct implementation (read the code: same`,
98
+ ` component means same implementation). Other files do not repeat it per`,
99
+ ` screen.`,
92
100
  ``,
93
101
  ...(siblings
94
102
  ? [`OTHER JOURNEYS IN THIS AREA (what they own or will own):`, siblings, ``]
@@ -152,7 +160,7 @@ export function planSuite(journey) {
152
160
  const { program, args } = withReadOnlyTools(splitCommand(loadConfig().aiCommand));
153
161
  let raw = '';
154
162
  try {
155
- raw = execFileSync(program, args, {
163
+ raw = runAiSync(program, args, {
156
164
  input: prompt,
157
165
  encoding: 'utf-8',
158
166
  maxBuffer: 10 * 1024 * 1024,
@@ -18,11 +18,13 @@
18
18
  * A guard refuses any repair that deletes a test, adds a skip or loses an
19
19
  * assertion, so no repair can turn a caught defect into a silent one.
20
20
  */
21
- import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
21
+ import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync } from 'node:fs';
22
+ import { createHash } from 'node:crypto';
22
23
  import { basename, join, resolve } from 'node:path';
23
24
  import { STATE_DIR } from '../paths.js';
24
25
  import { DEFECT_MARK } from './ai-generator.js';
25
- import { groupClaims, reviewDefects } from './defect-review.js';
26
+ import { groupClaims, normaliseClaims, reviewDefects } from './defect-review.js';
27
+ import { saveVerifyResults } from './review-journey.js';
26
28
  import { verifyFile, verifyRun, claimedDefects, repairGuard, environmentSkip, } from './verify.js';
27
29
  const describe = (label, issues) => issues.length
28
30
  ? [`${label}:`, ...issues.map((i) => `- line ${i.line}: ${i.title}\n${i.detail.replace(/^/gm, ' ')}`)]
@@ -32,6 +34,20 @@ const label = (t) => `${basename(t.file)}:${t.line} ${t.title}`;
32
34
  const claimOf = (t) => claimedDefects(readFileSync(t.file, 'utf-8')).get(t.line);
33
35
  const unclaimed = (issues) => issues.filter((t) => !claimOf(t));
34
36
  const usable = (o) => o.ran && !o.infra;
37
+ const PROGRESS_DIR = join(STATE_DIR, 'verify-progress');
38
+ const fileHash = (f) => (existsSync(f) ? createHash('sha1').update(readFileSync(f)).digest('hex') : '');
39
+ function loadProgress(journeyId) {
40
+ try {
41
+ return JSON.parse(readFileSync(join(PROGRESS_DIR, `${journeyId}.json`), 'utf-8'));
42
+ }
43
+ catch {
44
+ return {};
45
+ }
46
+ }
47
+ function saveProgress(journeyId, p) {
48
+ mkdirSync(PROGRESS_DIR, { recursive: true });
49
+ writeFileSync(join(PROGRESS_DIR, `${journeyId}.json`), JSON.stringify(p), 'utf-8');
50
+ }
35
51
  export async function verifyJourney(ctx) {
36
52
  const { config, journey, targets } = ctx;
37
53
  const warnings = [];
@@ -40,9 +56,13 @@ export async function verifyJourney(ctx) {
40
56
  const repairOnce = async (target, open, suspect, note = '') => {
41
57
  const current = readFileSync(target, 'utf-8');
42
58
  const failure = [note, ...describe('FAILED', open), ...describe('SKIPPED', suspect)].filter(Boolean).join('\n');
43
- const next = await ctx.repair(target, current, failure);
44
- if (next === null)
59
+ const repaired = await ctx.repair(target, current, failure);
60
+ if (repaired === null)
45
61
  return false;
62
+ // Only a quoted [spec] rule may be claimed as a defect; anything else becomes a finding mark.
63
+ const { content: next, downgraded } = normaliseClaims(repaired, journey.rawMarkdown);
64
+ if (downgraded)
65
+ console.log(` ${basename(target)}: ${downgraded} defect claim(s) against a non-[spec] rule kept as finding(s)`);
46
66
  const refused = repairGuard(current, next);
47
67
  if (refused) {
48
68
  warnings.push(`verify: the repair of ${basename(target)} was refused — ${refused}`);
@@ -52,10 +72,24 @@ export async function verifyJourney(ctx) {
52
72
  return true;
53
73
  };
54
74
  // 1. Each file on its own: what is wrong inside the file.
75
+ // A file that finished this step before (same content since) is not run
76
+ // again: an interrupted verify — a usage limit, a closed laptop — resumes
77
+ // at the first file it had not finished.
55
78
  let spent = 0;
56
79
  const soloSkips = new Set();
57
80
  const lastSolo = new Map();
81
+ const progress = loadProgress(journey.id);
58
82
  for (const target of targets) {
83
+ const done = progress[resolve(target)];
84
+ if (done && done.hash === fileHash(target)) {
85
+ console.log(` ${basename(target)}: verified earlier, unchanged — kept`);
86
+ if (!usable(done.outcome))
87
+ notRun.push(`${basename(target)}: ${done.outcome.infra ?? done.outcome.reason}`);
88
+ lastSolo.set(target, done.outcome);
89
+ for (const k of done.outcome.skips)
90
+ soloSkips.add(key(k));
91
+ continue;
92
+ }
59
93
  const run = async () => {
60
94
  const started = Date.now();
61
95
  const o = await verifyFile(config, journey.area, target);
@@ -95,6 +129,8 @@ export async function verifyJourney(ctx) {
95
129
  lastSolo.set(target, outcome);
96
130
  for (const k of outcome.skips)
97
131
  soloSkips.add(key(k));
132
+ progress[resolve(target)] = { hash: fileHash(target), outcome };
133
+ saveProgress(journey.id, progress);
98
134
  }
99
135
  // 2. The whole journey in ONE run. Whatever fails only here is repaired with
100
136
  // that context, once.
@@ -130,12 +166,14 @@ export async function verifyJourney(ctx) {
130
166
  // reported as a defect on the repair's word alone.
131
167
  const finals = together && usable(together) ? [together] : [...lastSolo.values()].filter(usable);
132
168
  const failures = finals.flatMap((o) => o.failures);
169
+ // Kept so `testforge review` can judge these failures again without re-running verify.
170
+ saveVerifyResults(journey.id, failures, finals.flatMap((o) => o.passes));
133
171
  const unresolved = unclaimed(failures).map(label);
134
172
  let reviewed = [];
135
173
  const groups = groupClaims(failures, claimOf);
136
174
  if (groups.length) {
137
175
  console.log(` reviewing ${groups.length} claimed defect(s) independently…`);
138
- const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes));
176
+ const reviews = await reviewDefects(journey, groups, finals.flatMap((o) => o.passes), undefined, join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`));
139
177
  reviewed = reviews.filter((r) => r.verdict === 'defect' || r.verdict === 'finding' || r.verdict === 'pending-decision');
140
178
  // A rejected claim is the test's fault until shown otherwise: one guarded
141
179
  // repair with the reviewer's reason; whatever still fails is unverified and
@@ -162,7 +200,10 @@ export async function verifyJourney(ctx) {
162
200
  }
163
201
  mkdirSync(join(STATE_DIR, 'defect-review'), { recursive: true });
164
202
  writeFileSync(join(STATE_DIR, 'defect-review', `${journey.id}.json`), JSON.stringify(reviews, null, 2), 'utf-8');
203
+ rmSync(join(STATE_DIR, 'defect-review', `${journey.id}.partial.json`), { force: true });
165
204
  }
205
+ // The checkpoints only serve an interrupted verify; a finished one starts fresh next time.
206
+ rmSync(join(PROGRESS_DIR, `${journey.id}.json`), { force: true });
166
207
  if (notRun.length) {
167
208
  warnings.push(`verify: ${notRun.length} file(s) could not be run — ${notRun.slice(0, 5).join('; ')}`);
168
209
  }
@@ -19,7 +19,7 @@ import { createRequire } from 'node:module';
19
19
  import { join, relative, dirname, resolve } from 'node:path';
20
20
  import { loadEnvFile, testDirForType, resolveFramework, } from '../project/config.js';
21
21
  import { runnerCommand, packageRoot } from '../run/run-tests.js';
22
- import { DEFECT_MARK } from './ai-generator.js';
22
+ import { DEFECT_MARK, FINDING_MARK } from './ai-generator.js';
23
23
  /** How much runner output to carry into a repair prompt. */
24
24
  const MAX_OUTPUT = 12_000;
25
25
  /**
@@ -304,9 +304,11 @@ export function claimedDefects(content) {
304
304
  if (!/\b(?:test|it)(?:\.\w+)?\s*\(/.test(lines[i]))
305
305
  continue;
306
306
  for (let j = i - 1; j >= 0 && lines[j].trim().startsWith('//'); j--) {
307
- const at = lines[j].indexOf(DEFECT_MARK);
308
- if (at !== -1)
309
- claims.set(i + 1, lines[j].slice(at + DEFECT_MARK.length).trim());
307
+ for (const mark of [DEFECT_MARK, FINDING_MARK]) {
308
+ const at = lines[j].indexOf(mark);
309
+ if (at !== -1)
310
+ claims.set(i + 1, lines[j].slice(at + mark.length).trim());
311
+ }
310
312
  }
311
313
  }
312
314
  return claims;
@@ -328,5 +330,8 @@ export function repairGuard(before, after) {
328
330
  const expects = /\bexpect(?:\.soft)?\s*\(/g;
329
331
  if (count(after, expects) < count(before, expects))
330
332
  return 'it removed an assertion';
333
+ const sleeps = /\bwaitForTimeout\s*\(/g;
334
+ if (count(after, sleeps) > count(before, sleeps))
335
+ return 'it added a fixed wait (waitForTimeout)';
331
336
  return null;
332
337
  }
@@ -0,0 +1,122 @@
1
+ /**
2
+ * `testforge env --diff` — which variables the generated suite reads that the
3
+ * env file does not set, per journey, with the files that read each one.
4
+ *
5
+ * Only names are read from the env file, never values. A variable the code
6
+ * reads with a fallback (`process.env.X ?? 'default'`) is listed apart: the
7
+ * suite runs without it, on the default.
8
+ */
9
+ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
10
+ import { basename, join, relative } from 'node:path';
11
+ import { loadConfig } from './config.js';
12
+ import { extractEnvVars } from './env-template.js';
13
+ /** Variables a runner or the tool itself provides; never asked of a person. */
14
+ const PROVIDED = new Set(['CI', 'NODE_ENV', 'PWDEBUG', 'TESTFORGE_RUN_ID', 'TESTFORGE_CLEAN_FILE', 'HOME', 'PATH', 'TZ']);
15
+ /** Names set in an env file (blank values count as unset). */
16
+ export function envNames(text) {
17
+ const out = new Set();
18
+ for (const line of text.split('\n')) {
19
+ const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
20
+ if (m && m[2].trim().replace(/^(['"])\1$/, '') !== '')
21
+ out.add(m[1]);
22
+ }
23
+ return out;
24
+ }
25
+ const withDefault = (text, name) => new RegExp(`process\\.env(?:\\.${name}|\\[\\s*['"]${name}['"]\\s*\\])\\s*(?:\\?\\?|\\|\\|)`).test(text) ||
26
+ new RegExp(`env\\(\\s*['"]${name}['"]\\s*,\\s*[^)\\s]`).test(text);
27
+ function files(dir) {
28
+ const out = [];
29
+ const walk = (d) => {
30
+ for (const e of readdirSync(d, { withFileTypes: true })) {
31
+ const p = join(d, e.name);
32
+ if (e.isDirectory()) {
33
+ if (e.name !== 'node_modules' && !e.name.startsWith('.'))
34
+ walk(p);
35
+ }
36
+ else if (/\.(c|m)?[jt]sx?$/.test(e.name))
37
+ out.push(p);
38
+ }
39
+ };
40
+ if (existsSync(dir) && statSync(dir).isDirectory())
41
+ walk(dir);
42
+ return out;
43
+ }
44
+ /**
45
+ * For each journey folder under each area's test folder (and the shared
46
+ * helpers), the variables its files read that the env file does not set.
47
+ * `only` limits it to the named journey folders.
48
+ */
49
+ export function envDiff(root = '.', only) {
50
+ const config = loadConfig();
51
+ const envFile = config.envFile ?? '.env.testing';
52
+ const set = envNames(existsSync(join(root, envFile)) ? readFileSync(join(root, envFile), 'utf-8') : '');
53
+ const journeys = [];
54
+ // A variable the suite sets for itself at run time (the testkit's setup() exports
55
+ // tokens and ids — H4 — and runner configs set base URLs) is not asked of a person.
56
+ const assigned = new Set();
57
+ const dirs = Object.values(config.testOutput ?? {}).flatMap((d) => [join(root, d), join(root, d, '..')]);
58
+ for (const f of [...new Set(dirs.flatMap((d) => files(d).filter((x) => /helpers|config|setup/i.test(x))))]) {
59
+ const text = readFileSync(f, 'utf-8');
60
+ for (const m of text.matchAll(/process\.env(?:\.([A-Z0-9_]+)|\[\s*['"]([A-Z0-9_]+)['"]\s*\])\s*(?:=(?!=)|\?\?=|\|\|=)/g))
61
+ assigned.add(m[1] ?? m[2]);
62
+ // setup() often sets several at once: process.env[`TEST_${k}`] = … or Object.assign(process.env, {...}).
63
+ for (const m of text.matchAll(/Object\.assign\(\s*process\.env\s*,\s*\{([^}]*)\}/g))
64
+ for (const k of m[1].matchAll(/([A-Z][A-Z0-9_]+)\s*:/g))
65
+ assigned.add(k[1]);
66
+ // …or loops over an object of them: for (const [k, v] of Object.entries(publish)) … process.env[k] = …
67
+ for (const m of text.matchAll(/Object\.entries\(\s*(\w+)\s*\)[^;{]*\)?\s*[^;]*process\.env\[\s*\w+\s*\]\s*=(?!=)/g)) {
68
+ const obj = new RegExp(`(?:const|let|var)\\s+${m[1]}\\s*=\\s*\\{([\\s\\S]*?)\\n\\s*\\};`).exec(text);
69
+ if (obj)
70
+ for (const k of obj[1].matchAll(/^\s*([A-Z][A-Z0-9_]+)\s*:/gm))
71
+ assigned.add(k[1]);
72
+ }
73
+ }
74
+ for (const testDir of Object.values(config.testOutput ?? {})) {
75
+ const abs = join(root, testDir);
76
+ for (const f of files(abs)) {
77
+ const rel = relative(abs, f);
78
+ const first = rel.split(/[\\/]/)[0];
79
+ const journey = rel.includes('/') || rel.includes('\\') ? (first === 'helpers' ? 'shared (testkit and helpers)' : first) : basename(testDir);
80
+ if (only && !only.includes(first))
81
+ continue;
82
+ const text = readFileSync(f, 'utf-8');
83
+ for (const name of extractEnvVars(text)) {
84
+ if (set.has(name) || PROVIDED.has(name) || assigned.has(name))
85
+ continue;
86
+ let j = journeys.find((x) => x.journey === journey);
87
+ if (!j)
88
+ journeys.push((j = { journey, missing: [] }));
89
+ let v = j.missing.find((x) => x.name === name);
90
+ if (!v)
91
+ j.missing.push((v = { name, files: [], hasDefault: true }));
92
+ if (!v.files.includes(basename(f)))
93
+ v.files.push(basename(f));
94
+ v.hasDefault = v.hasDefault && withDefault(text, name);
95
+ }
96
+ }
97
+ }
98
+ for (const j of journeys)
99
+ j.missing.sort((a, b) => Number(a.hasDefault) - Number(b.hasDefault) || a.name.localeCompare(b.name));
100
+ journeys.sort((a, b) => a.journey.localeCompare(b.journey));
101
+ return { envFile, journeys };
102
+ }
103
+ export function describeEnvDiff(d) {
104
+ const needed = d.journeys.flatMap((j) => j.missing.filter((v) => !v.hasDefault));
105
+ const optional = d.journeys.flatMap((j) => j.missing.filter((v) => v.hasDefault));
106
+ if (!needed.length && !optional.length)
107
+ return `Every variable the suite reads is set in ${d.envFile}.`;
108
+ const lines = [`Not set in ${d.envFile} (names only; values are never read):`];
109
+ for (const j of d.journeys) {
110
+ const req = j.missing.filter((v) => !v.hasDefault);
111
+ if (!req.length)
112
+ continue;
113
+ lines.push(` ${j.journey}`);
114
+ for (const v of req)
115
+ lines.push(` ${v.name.padEnd(34)} used by ${v.files.slice(0, 3).join(', ')}${v.files.length > 3 ? ` +${v.files.length - 3}` : ''}`);
116
+ }
117
+ const uniq = (xs) => new Set(xs.map((v) => v.name)).size;
118
+ lines.push('', `${uniq(needed)} variable(s) to set. Tests that need one skip and name it until it is set.`);
119
+ if (optional.length)
120
+ lines.push(`${uniq(optional)} more have a default in the code and can stay unset: ${[...new Set(optional.map((v) => v.name))].slice(0, 12).join(', ')}${uniq(optional) > 12 ? ', …' : ''}`);
121
+ return lines.join('\n');
122
+ }
@@ -17,7 +17,7 @@ import { findMarkdownFiles } from '../find-files.js';
17
17
  import { STATE_DIR, LEGACY_STATE_DIR } from '../paths.js';
18
18
  import { loadConfig } from './config.js';
19
19
  /** Pull env var names from a chunk of text. */
20
- function extractEnvVars(text) {
20
+ export function extractEnvVars(text) {
21
21
  const vars = new Set();
22
22
  // process.env.XXX and process.env["XXX"] / process.env['XXX']
23
23
  for (const m of text.matchAll(/process\.env\.([A-Z0-9_]+)/gi)) {
@@ -10,8 +10,8 @@
10
10
  * Nothing is hardcoded for a specific framework or layout; the AI
11
11
  * infers it from what the project actually looks like.
12
12
  */
13
- import { execFileSync } from 'node:child_process';
14
13
  import { loadConfig, splitCommand, withEditTools } from './config.js';
14
+ import { runAiSync } from '../generate/ai-usage.js';
15
15
  /** Build the prompt that asks the AI to write testforge.config.json. */
16
16
  export function buildInitPrompt() {
17
17
  return [
@@ -71,7 +71,7 @@ export function buildInitPrompt() {
71
71
  export function runInit() {
72
72
  const prompt = buildInitPrompt();
73
73
  const { program, args } = withEditTools(splitCommand(loadConfig().aiCommand));
74
- const output = execFileSync(program, args, {
74
+ const output = runAiSync(program, args, {
75
75
  input: prompt,
76
76
  encoding: 'utf-8',
77
77
  maxBuffer: 10 * 1024 * 1024,
@@ -17,8 +17,8 @@
17
17
  * whatever the project actually uses. Secrets are never captured;
18
18
  * only env var names.
19
19
  */
20
- import { execFileSync } from 'node:child_process';
21
20
  import { loadConfig, splitCommand, withEditTools } from './config.js';
21
+ import { runAiSync } from '../generate/ai-usage.js';
22
22
  /** Build the prompt that asks the AI to produce the project map. */
23
23
  export function buildMapPrompt(srcDir, relatedDirs = []) {
24
24
  const related = relatedDirs.length
@@ -93,7 +93,7 @@ export function runMap(srcDir) {
93
93
  const config = loadConfig();
94
94
  const prompt = buildMapPrompt(srcDir, config.relatedDirs ?? []);
95
95
  const { program, args } = withEditTools(splitCommand(config.aiCommand), config.relatedDirs ?? []);
96
- const output = execFileSync(program, args, {
96
+ const output = runAiSync(program, args, {
97
97
  input: prompt,
98
98
  encoding: 'utf-8',
99
99
  maxBuffer: 10 * 1024 * 1024,
@@ -0,0 +1,94 @@
1
+ /**
2
+ * `testforge ci --github` — a GitHub Actions workflow that runs the generated
3
+ * tests against a deployed environment and publishes the result.
4
+ *
5
+ * Every value the suite needs comes from repository secrets of the same
6
+ * name: the names are read from the env file (names only, never values), so
7
+ * the workflow asks for exactly what a local run uses. Nothing here names a
8
+ * product or a stack beyond the runners TestForge generates for.
9
+ */
10
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
11
+ import { dirname, join } from 'node:path';
12
+ import { loadConfig, resolveFramework } from '../project/config.js';
13
+ import { envNames } from '../project/env-diff.js';
14
+ export const WORKFLOW_PATH = join('.github', 'workflows', 'testforge.yml');
15
+ /** --schedule values: a name or a five-field cron line. Unset or "none": manual only. */
16
+ export function scheduleCron(v) {
17
+ if (!v || v === 'none')
18
+ return undefined;
19
+ const named = { daily: '0 6 * * *', weekdays: '0 6 * * 1-5', weekly: '0 6 * * 1' };
20
+ if (named[v])
21
+ return named[v];
22
+ if (/^\S+\s+\S+\s+\S+\s+\S+\s+\S+$/.test(v.trim()))
23
+ return v.trim();
24
+ throw new Error(`--schedule takes none, daily, weekdays, weekly or a cron line like "0 6 * * 1": got "${v}"`);
25
+ }
26
+ export function workflowYaml(o) {
27
+ const env = [
28
+ ` TESTFORGE_KEY: \${{ secrets.TESTFORGE_KEY }}`,
29
+ ` TESTFORGE_DASHBOARD_URL: \${{ vars.TESTFORGE_DASHBOARD_URL }}`,
30
+ ` TESTFORGE_ENV: ${o.envName}`,
31
+ ` CI: 'true'`,
32
+ ...o.secrets.filter((n) => !/^(TESTFORGE_KEY|TESTFORGE_ENV|TESTFORGE_DASHBOARD_URL|CI)$/.test(n)).map((n) => ` ${n}: \${{ secrets.${n} }}`),
33
+ ];
34
+ return [
35
+ `# @generated by testforge ci --github`,
36
+ `# Runs the TestForge suite against a deployed environment and publishes the`,
37
+ `# result to the dashboard. Add each name under "env" as a repository secret`,
38
+ `# (Settings → Secrets and variables → Actions), and the dashboard address as`,
39
+ `# the repository variable TESTFORGE_DASHBOARD_URL.`,
40
+ `name: TestForge`,
41
+ ``,
42
+ `on:`,
43
+ ` workflow_dispatch: {} # Actions tab → TestForge → Run workflow`,
44
+ ...(o.schedule ? [` schedule:`, ` - cron: '${o.schedule}' # UTC`] : [` # No schedule: add one with testforge ci --github --schedule weekdays --force`]),
45
+ ``,
46
+ `concurrency:`,
47
+ ` group: testforge-\${{ github.ref }}`,
48
+ ` cancel-in-progress: false`,
49
+ ``,
50
+ `jobs:`,
51
+ ` test:`,
52
+ ` runs-on: ubuntu-latest`,
53
+ ` timeout-minutes: 180`,
54
+ ` env:`,
55
+ ...env,
56
+ ` steps:`,
57
+ ` - uses: actions/checkout@v4`,
58
+ ` - uses: actions/setup-node@v4`,
59
+ ` with:`,
60
+ ` node-version: ${o.node ?? 20}`,
61
+ ` cache: npm`,
62
+ ` - run: npm ci`,
63
+ ...(o.playwright ? [` - run: npx playwright install --with-deps chromium`] : []),
64
+ ` - run: npx testforge run`,
65
+ ` - if: always()`,
66
+ ` uses: actions/upload-artifact@v4`,
67
+ ` with:`,
68
+ ` name: testforge-results`,
69
+ ` path: test-artifacts/`,
70
+ ` retention-days: 14`,
71
+ ``,
72
+ ].join('\n');
73
+ }
74
+ /** Write the workflow for this project. Refuses to replace one without force. */
75
+ export function writeGithubWorkflow(opts = {}) {
76
+ const cron = scheduleCron(opts.schedule);
77
+ const root = opts.root ?? '.';
78
+ const path = join(root, WORKFLOW_PATH);
79
+ if (existsSync(path) && !opts.force)
80
+ return `${WORKFLOW_PATH} already exists — re-run with --force to replace it.`;
81
+ const config = loadConfig();
82
+ const envFile = config.envFile ?? '.env.testing';
83
+ const names = existsSync(join(root, envFile)) ? [...envNames(readFileSync(join(root, envFile), 'utf-8'))] : [];
84
+ const playwright = Object.keys(config.testOutput ?? {}).some((a) => resolveFramework(config, a, '') === 'playwright');
85
+ mkdirSync(dirname(path), { recursive: true });
86
+ writeFileSync(path, workflowYaml({ secrets: names.sort(), playwright, envName: opts.envName ?? 'staging', schedule: cron }), 'utf-8');
87
+ return [
88
+ `Wrote ${WORKFLOW_PATH}. ${cron ? `It runs on the schedule "${cron}" (UTC) and when started by hand.` : 'It runs only when started by hand (Actions tab → TestForge → Run workflow).'}`,
89
+ `Add these repository secrets (same names as ${envFile}, values for the deployed environment):`,
90
+ ` TESTFORGE_KEY${names.length ? `, ${names.join(', ')}` : ''}`,
91
+ `and the repository variable TESTFORGE_DASHBOARD_URL (the deployed dashboard).`,
92
+ `Point the base URL variables at the deployed app, not localhost.`,
93
+ ].join('\n');
94
+ }
@@ -0,0 +1,111 @@
1
+ /**
2
+ * `testforge clean` — remove what earlier runs created, through the product's
3
+ * own API, using the requests the testkit recorded (H8).
4
+ *
5
+ * Without --yes it only lists. With --yes it runs one throw-away test per
6
+ * testkit that signs in through that testkit and sends each recorded cleanup
7
+ * request; the result is reported per record and nothing is published.
8
+ * Gated like reset(): TESTFORGE_ALLOW_RESET must be "true".
9
+ */
10
+ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
11
+ import { createRequire } from 'node:module';
12
+ import { dirname, join, relative, resolve } from 'node:path';
13
+ import { spawnSync } from 'node:child_process';
14
+ import { loadConfig, testDirForType, resolveFramework } from '../project/config.js';
15
+ import { harnessInfo } from '../generate/harness.js';
16
+ import { packageRoot } from './run-tests.js';
17
+ const runData = createRequire(import.meta.url)('../../../reporters/run-data.cjs');
18
+ export function pickRecords(opts, now = Date.now()) {
19
+ const root = opts.root ?? process.cwd();
20
+ const runs = runData.listRuns(root);
21
+ const cutoff = opts.olderThanDays != null ? now - opts.olderThanDays * 86_400_000 : undefined;
22
+ const chosen = runs.filter((r) => (opts.runId ? r.runId === opts.runId : cutoff != null ? Date.parse(r.at) < cutoff : false));
23
+ return chosen.flatMap((r) => runData.readRun(r.runId, root));
24
+ }
25
+ const CLEAN_TEST = (kitRel) => [
26
+ `// @ts-nocheck`,
27
+ `// @generated by testforge clean — deleted after this run`,
28
+ `const kit = require('${kitRel}');`,
29
+ `const fs = require('fs');`,
30
+ `const t = typeof test !== 'undefined' ? test : require('@playwright/test').test;`,
31
+ `t('testforge clean', async () => {`,
32
+ ` const file = process.env.TESTFORGE_CLEAN_FILE;`,
33
+ ` const out = [];`,
34
+ ` for (const r of JSON.parse(fs.readFileSync(file, 'utf8'))) {`,
35
+ ` try {`,
36
+ ` const token = await kit.ensureToken(r.role || 'customer');`,
37
+ ` const res = await fetch(r.cleanup.url, { method: r.cleanup.method || 'DELETE', headers: { Authorization: 'Bearer ' + token, Accept: 'application/json' } });`,
38
+ ` out.push({ kind: r.kind, id: r.id, status: res.status });`,
39
+ ` } catch (e) {`,
40
+ ` out.push({ kind: r.kind, id: r.id, error: String((e && e.message) || e) });`,
41
+ ` }`,
42
+ ` }`,
43
+ ` fs.writeFileSync(file + '.out', JSON.stringify(out));`,
44
+ `});`,
45
+ ``,
46
+ ].join('\n');
47
+ export async function runClean(opts) {
48
+ const lines = [];
49
+ if (!opts.runId && opts.olderThanDays == null) {
50
+ const runs = runData.listRuns(process.cwd());
51
+ if (!runs.length)
52
+ return 'No run has recorded any data yet (test-artifacts/run-data/ is empty).';
53
+ return ['Runs with recorded data:', ...runs.map((r) => ` ${r.runId} ${r.at.slice(0, 16).replace('T', ' ')} ${r.count} record(s)`),
54
+ '', 'Clean one with --run <id>, or every run older than N days with --older-than <N>. Add --yes to remove.'].join('\n');
55
+ }
56
+ const recs = pickRecords(opts);
57
+ if (!recs.length)
58
+ return 'No recorded data matches.';
59
+ const removable = recs.filter((r) => r.cleanup?.url && r.kit);
60
+ const stuck = recs.filter((r) => !(r.cleanup?.url && r.kit));
61
+ lines.push(`${recs.length} record(s): ${removable.length} removable through the product's API, ${stuck.length} not.`);
62
+ for (const r of stuck)
63
+ lines.push(` keep ${r.kind} ${String(r.id)} — the product has no request that removes it`);
64
+ if (!opts.yes) {
65
+ for (const r of removable)
66
+ lines.push(` would ${r.cleanup.method || 'DELETE'} ${r.kind} ${String(r.id)}`);
67
+ lines.push('', 'Dry run. Add --yes to remove them.');
68
+ return lines.join('\n');
69
+ }
70
+ if (process.env.TESTFORGE_ALLOW_RESET !== 'true') {
71
+ lines.push('', 'Refused: set TESTFORGE_ALLOW_RESET=true (the same gate as the testkit reset) to remove test data.');
72
+ return lines.join('\n');
73
+ }
74
+ // One throw-away test per testkit, run by that area's own runner.
75
+ const config = loadConfig();
76
+ const byKit = new Map();
77
+ for (const r of removable)
78
+ byKit.set(resolve(r.kit), [...(byKit.get(resolve(r.kit)) ?? []), r]);
79
+ for (const [kitPath, rows] of byKit) {
80
+ const area = Object.keys(config.testOutput ?? {}).find((a) => {
81
+ const fw = resolveFramework(config, a, '') === 'jest' ? 'jest' : 'playwright';
82
+ return resolve(harnessInfo(testDirForType(config, a), fw, config.testExtension || '.test.js').path) === kitPath;
83
+ });
84
+ if (!area) {
85
+ lines.push(` skipped ${rows.length} record(s): ${kitPath} is not a testkit of this project`);
86
+ continue;
87
+ }
88
+ const areaDir = resolve(testDirForType(config, area));
89
+ const jest = resolveFramework(config, area, '') === 'jest';
90
+ const ext = (config.testExtension || '.test.js').replace(/^\.?(test|spec)/, '.test');
91
+ const file = join(areaDir, `tf-clean${ext.startsWith('.test') ? ext : '.test.js'}`);
92
+ const data = join(areaDir, '.tf-clean.json');
93
+ mkdirSync(dirname(file), { recursive: true });
94
+ let kitRel = relative(areaDir, kitPath).replace(/\.[cm]?[jt]s$/, '');
95
+ if (!kitRel.startsWith('.'))
96
+ kitRel = `./${kitRel}`;
97
+ writeFileSync(file, CLEAN_TEST(kitRel), 'utf-8');
98
+ writeFileSync(data, JSON.stringify(rows), 'utf-8');
99
+ const cwd = packageRoot(areaDir);
100
+ const args = jest ? ['jest', relative(cwd, file), '--runInBand', '--reporters=default'] : ['playwright', 'test', relative(cwd, file), '--reporter=list', '--workers=1'];
101
+ spawnSync('npx', args, { cwd, stdio: 'inherit', env: { ...process.env, TESTFORGE_CLEAN_FILE: data } });
102
+ const out = existsSync(`${data}.out`) ? JSON.parse(readFileSync(`${data}.out`, 'utf-8')) : [];
103
+ for (const o of out)
104
+ lines.push(` ${o.error ? 'failed ' : o.status < 300 ? 'removed' : `HTTP ${o.status}`} ${o.kind} ${String(o.id)}${o.error ? ` — ${o.error}` : ''}`);
105
+ if (!out.length)
106
+ lines.push(` ${area}: the clean test did not report back`);
107
+ for (const f of [file, data, `${data}.out`])
108
+ rmSync(f, { force: true });
109
+ }
110
+ return lines.join('\n');
111
+ }