@activate-core/test-forge 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +47 -0
- package/dist/core/cache.js +3 -1
- package/dist/core/env-template.js +9 -1
- package/dist/core/harness.js +122 -5
- package/dist/core/init.js +7 -1
- package/dist/core/paths.js +19 -0
- package/dist/core/report.js +269 -0
- package/dist/core/run-generate.js +63 -36
- package/dist/core/run-tests.js +1 -1
- package/dist/core/setup.js +3 -2
- package/dist/core/source-of-truth.js +2 -2
- package/dist/core/update.js +1 -1
- package/package.json +2 -1
- package/reporters/_shared.cjs +109 -0
- package/reporters/jest-reporter.cjs +30 -0
- package/reporters/playwright-reporter.cjs +30 -0
package/dist/cli/index.js
CHANGED
|
@@ -24,6 +24,7 @@ import { runSetup } from '../core/setup.js';
|
|
|
24
24
|
import { runUpdate } from '../core/update.js';
|
|
25
25
|
import { runEnvTemplate } from '../core/env-template.js';
|
|
26
26
|
import { loadConfig } from '../core/config.js';
|
|
27
|
+
import { publishReport, loadReportConfig, loadDotenv, scanInventory, publishInventory } from '../core/report.js';
|
|
27
28
|
import { dirname } from 'node:path';
|
|
28
29
|
const command = process.argv[2];
|
|
29
30
|
const useBasic = process.argv.includes('--basic');
|
|
@@ -78,6 +79,7 @@ function printUsage() {
|
|
|
78
79
|
console.log(' testforge update Re-check and update only what changed');
|
|
79
80
|
console.log(' testforge env Write .env.testing.example from docs');
|
|
80
81
|
console.log(' testforge run [--area a] [--journey j] Run the generated tests (Jest/Playwright)');
|
|
82
|
+
console.log(' testforge run --env-file <file> Run against another server (e.g. .env.dev / .env.staging)');
|
|
81
83
|
}
|
|
82
84
|
async function main() {
|
|
83
85
|
if (command === 'update') {
|
|
@@ -114,6 +116,7 @@ async function main() {
|
|
|
114
116
|
const code = runTests({
|
|
115
117
|
area: flagValue('--area'),
|
|
116
118
|
journey: flagValue('--journey'),
|
|
119
|
+
envFile: flagValue('--env-file'),
|
|
117
120
|
});
|
|
118
121
|
process.exitCode = code;
|
|
119
122
|
}
|
|
@@ -165,6 +168,50 @@ async function main() {
|
|
|
165
168
|
runEnvTemplate('docs', '.env.testing.example', '.', generatedScanDirs());
|
|
166
169
|
}
|
|
167
170
|
}
|
|
171
|
+
else if (command === 'report') {
|
|
172
|
+
// Publish a test run to the TestForge dashboard. Public by design — the only
|
|
173
|
+
// secret is the per-project API key (TESTFORGE_KEY), issued by the dashboard.
|
|
174
|
+
loadDotenv(); // pick up TESTFORGE_KEY from a gitignored .testforge.env if present
|
|
175
|
+
const cfg = loadReportConfig();
|
|
176
|
+
const resultsPath = flagValue('--results');
|
|
177
|
+
const project = flagValue('--project') || cfg.project;
|
|
178
|
+
const dashboardUrl = flagValue('--dashboard') || cfg.dashboardUrl || process.env.TESTFORGE_DASHBOARD_URL;
|
|
179
|
+
const apiKey = process.env.TESTFORGE_KEY;
|
|
180
|
+
const category = flagValue('--category') || 'api';
|
|
181
|
+
// --inventory: register the FULL generated test inventory (all suites) so the
|
|
182
|
+
// dashboard shows the total test-case count before/independent of any run.
|
|
183
|
+
if (process.argv.includes('--inventory')) {
|
|
184
|
+
if (!dashboardUrl || !apiKey) {
|
|
185
|
+
console.error('Usage: testforge report --inventory [--dashboard <url>] (TESTFORGE_KEY env required)');
|
|
186
|
+
process.exit(1);
|
|
187
|
+
}
|
|
188
|
+
console.log('Counting test cases (playwright --list expands parameterized tests; may take a moment)...');
|
|
189
|
+
const suites = scanInventory(process.cwd(), cfg.testOutput || {}, cfg.testExtension || '.test.js', cfg.framework || {});
|
|
190
|
+
const cases = suites.reduce((a, s) => a + s.total, 0);
|
|
191
|
+
const ir = await publishInventory({ suites, dashboardUrl, apiKey });
|
|
192
|
+
if (ir.status >= 200 && ir.status < 300)
|
|
193
|
+
console.log(`✓ Registered inventory: ${suites.length} suites, ${cases} test cases`);
|
|
194
|
+
else {
|
|
195
|
+
console.error(`✗ Inventory publish failed (${ir.status}): ${ir.text}`);
|
|
196
|
+
process.exitCode = 1;
|
|
197
|
+
}
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
if (!resultsPath || !project || !dashboardUrl || !apiKey) {
|
|
201
|
+
console.error('Usage: testforge report --results <jest-json> [--project <slug>] [--dashboard <url>] [--category api|web]');
|
|
202
|
+
console.error(' project / dashboard may also come from testforge.config.json.');
|
|
203
|
+
console.error(' TESTFORGE_KEY must be set in the environment (the key the dashboard gave you).');
|
|
204
|
+
process.exit(1);
|
|
205
|
+
}
|
|
206
|
+
const r = await publishReport({ resultsPath, project, dashboardUrl, apiKey, category });
|
|
207
|
+
if (r.status >= 200 && r.status < 300) {
|
|
208
|
+
console.log(`✓ Published "${project}" run to ${dashboardUrl}`);
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
console.error(`✗ Publish failed (${r.status}): ${r.text}`);
|
|
212
|
+
process.exitCode = 1;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
168
215
|
else {
|
|
169
216
|
printUsage();
|
|
170
217
|
}
|
package/dist/core/cache.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import { createHash } from 'node:crypto';
|
|
13
13
|
import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync, } from 'node:fs';
|
|
14
14
|
import { dirname, join, extname } from 'node:path';
|
|
15
|
+
import { STATE_DIR, LEGACY_STATE_DIR } from './paths.js';
|
|
15
16
|
/** Make a short fingerprint of a file's content. */
|
|
16
17
|
export function hashContent(content) {
|
|
17
18
|
return createHash('sha256').update(content).digest('hex');
|
|
@@ -63,7 +64,8 @@ const SKIP_DIRS = new Set([
|
|
|
63
64
|
'coverage',
|
|
64
65
|
'.next',
|
|
65
66
|
'docs',
|
|
66
|
-
|
|
67
|
+
STATE_DIR,
|
|
68
|
+
LEGACY_STATE_DIR,
|
|
67
69
|
'generated-tests',
|
|
68
70
|
]);
|
|
69
71
|
export function hashProjectSource(rootDir) {
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
import { readFileSync, writeFileSync, existsSync, readdirSync, } from 'node:fs';
|
|
15
15
|
import { join } from 'node:path';
|
|
16
16
|
import { findMarkdownFiles } from './find-files.js';
|
|
17
|
+
import { STATE_DIR, LEGACY_STATE_DIR } from './paths.js';
|
|
17
18
|
/** Pull env var names from a chunk of text. */
|
|
18
19
|
function extractEnvVars(text) {
|
|
19
20
|
const vars = new Set();
|
|
@@ -43,7 +44,8 @@ const SKIP_DIRS = new Set([
|
|
|
43
44
|
'coverage',
|
|
44
45
|
'docs',
|
|
45
46
|
'generated-tests',
|
|
46
|
-
|
|
47
|
+
STATE_DIR,
|
|
48
|
+
LEGACY_STATE_DIR,
|
|
47
49
|
]);
|
|
48
50
|
const SOURCE_EXT = /\.(c|m)?[jt]sx?$/i;
|
|
49
51
|
/**
|
|
@@ -256,6 +258,12 @@ export function runEnvTemplate(docsDir, outPath, srcDir = '.', testDirs = []) {
|
|
|
256
258
|
'# Only variables for the features you actually run are required.',
|
|
257
259
|
'# Keep .env.testing OUT of version control.',
|
|
258
260
|
'',
|
|
261
|
+
`# ${'='.repeat(40)}`,
|
|
262
|
+
'# TESTFORGE DASHBOARD (auto-publish each run)',
|
|
263
|
+
`# ${'='.repeat(40)}`,
|
|
264
|
+
'TESTFORGE_KEY= # from the TestForge dashboard (Integration tab) — the per-project key',
|
|
265
|
+
'TESTFORGE_ENV=local # environment youre testing = the DB/target you connect to (local/dev/stage/production). Set it in the SAME env-file that points tests at that DB. Not from git branch.',
|
|
266
|
+
'',
|
|
259
267
|
];
|
|
260
268
|
if (suiteVars.length > 0) {
|
|
261
269
|
lines.push(`# ${'='.repeat(40)}`);
|
package/dist/core/harness.js
CHANGED
|
@@ -234,19 +234,36 @@ export function generateHarness(harnessPath, framework) {
|
|
|
234
234
|
}
|
|
235
235
|
/** Async twin, so a harness can generate alongside other work. */
|
|
236
236
|
export function generateHarnessAsync(harnessPath, framework) {
|
|
237
|
+
const config = loadConfig();
|
|
237
238
|
const prompt = buildHarnessPrompt(harnessPath, framework);
|
|
238
|
-
const { program, args } = withReadOnlyTools(splitCommand(
|
|
239
|
+
const { program, args } = withReadOnlyTools(splitCommand(config.aiCommand));
|
|
240
|
+
const timeoutMs = config.aiTimeoutMs ?? 300_000;
|
|
239
241
|
return new Promise((resolve, reject) => {
|
|
240
242
|
const child = spawn(program, args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
|
241
243
|
let out = '';
|
|
242
244
|
let err = '';
|
|
245
|
+
let done = false;
|
|
246
|
+
const finish = (fn) => {
|
|
247
|
+
if (done)
|
|
248
|
+
return;
|
|
249
|
+
done = true;
|
|
250
|
+
clearTimeout(timer);
|
|
251
|
+
fn();
|
|
252
|
+
};
|
|
253
|
+
// Without this, a harness call that hangs (flaky network, a stalled AI
|
|
254
|
+
// CLI) never closes and the whole run wedges on step 1 forever — kill it
|
|
255
|
+
// and surface a resumable error so the caller can retry.
|
|
256
|
+
const timer = setTimeout(() => {
|
|
257
|
+
child.kill('SIGKILL');
|
|
258
|
+
finish(() => reject(new Error(`harness gen timed out after ${timeoutMs}ms — the AI CLI did not respond`)));
|
|
259
|
+
}, timeoutMs);
|
|
243
260
|
child.stdout.setEncoding('utf-8');
|
|
244
261
|
child.stdout.on('data', (d) => (out += d));
|
|
245
262
|
child.stderr.on('data', (d) => (err += d));
|
|
246
|
-
child.on('error', reject);
|
|
247
|
-
child.on('close', (code) => code === 0
|
|
263
|
+
child.on('error', (e) => finish(() => reject(e)));
|
|
264
|
+
child.on('close', (code) => finish(() => code === 0
|
|
248
265
|
? resolve(cleanOutput(out))
|
|
249
|
-
: reject(new Error(`harness gen exited ${code}: ${err.slice(0, 300)}`)));
|
|
266
|
+
: reject(new Error(`harness gen exited ${code}: ${err.slice(0, 300)}`))));
|
|
250
267
|
child.stdin.end(prompt);
|
|
251
268
|
});
|
|
252
269
|
}
|
|
@@ -256,7 +273,7 @@ export function generateHarnessAsync(harnessPath, framework) {
|
|
|
256
273
|
* dir, treats *.test.js as specs, and runs the testkit's setup() once via
|
|
257
274
|
* globalSetup (mints/populates tokens before any worker starts).
|
|
258
275
|
*/
|
|
259
|
-
export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage) {
|
|
276
|
+
export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage, category = 'web') {
|
|
260
277
|
return [
|
|
261
278
|
`// @generated by testforge`,
|
|
262
279
|
`// Minimal Playwright config so testforge's generated specs run.`,
|
|
@@ -268,14 +285,114 @@ export function playwrightConfig(testDirRelToPackage, globalSetupRelToPackage) {
|
|
|
268
285
|
` testMatch: /.*\\.(test|spec)\\.[jt]s$/,`,
|
|
269
286
|
` globalSetup: require.resolve('${globalSetupRelToPackage}'),`,
|
|
270
287
|
` timeout: 60000,`,
|
|
288
|
+
` // Console line output + Allure results (see ALLURE.md to view the report).`,
|
|
289
|
+
` // Last reporter auto-publishes the run to the TestForge dashboard (needs`,
|
|
290
|
+
` // TESTFORGE_KEY in env + project/dashboardUrl in testforge.config.json).`,
|
|
291
|
+
` reporter: [['line'], ['allure-playwright'], [require.resolve('@activate-core/test-forge/reporters/playwright-reporter.cjs'), { category: ${JSON.stringify(category)} }]],`,
|
|
271
292
|
` use: {`,
|
|
272
293
|
` baseURL: process.env.BASE_URL || 'http://localhost:3000',`,
|
|
273
294
|
` trace: 'on-first-retry',`,
|
|
295
|
+
` screenshot: 'only-on-failure',`,
|
|
274
296
|
` },`,
|
|
275
297
|
`});`,
|
|
276
298
|
``,
|
|
277
299
|
].join('\n');
|
|
278
300
|
}
|
|
301
|
+
/**
|
|
302
|
+
* Jest config for a backend (jest) area — self-contained so `npm test` / a bare
|
|
303
|
+
* `npx jest` runs with no shell setup, and emits Allure results.
|
|
304
|
+
* - testEnvironment 'allure-jest/node': captures each test into allure-results/
|
|
305
|
+
* - setupFiles: loads .env.testing + NODE_ENV + the reset opt-in before any
|
|
306
|
+
* test file imports the app
|
|
307
|
+
* - serial (maxWorkers 1): suites share one test DB and reset between tests
|
|
308
|
+
*/
|
|
309
|
+
export function jestConfig(testDirRelToPackage, setupRelToPackage, category = 'api') {
|
|
310
|
+
return [
|
|
311
|
+
`// @generated by testforge`,
|
|
312
|
+
`// Install once: npm i -D jest allure-jest allure-commandline (see ALLURE.md)`,
|
|
313
|
+
`module.exports = {`,
|
|
314
|
+
` testEnvironment: 'allure-jest/node',`,
|
|
315
|
+
` setupFiles: ['<rootDir>/${setupRelToPackage}'],`,
|
|
316
|
+
` testMatch: ['<rootDir>/${testDirRelToPackage}/**/*.test.js'],`,
|
|
317
|
+
` // Serial: the suites share ONE test DB and reset it between tests.`,
|
|
318
|
+
` maxWorkers: 1,`,
|
|
319
|
+
` forceExit: true,`,
|
|
320
|
+
` // 'default' keeps console output; the second reporter auto-publishes the`,
|
|
321
|
+
` // run to the TestForge dashboard (needs TESTFORGE_KEY in env +`,
|
|
322
|
+
` // project/dashboardUrl in testforge.config.json).`,
|
|
323
|
+
` reporters: ['default', [require.resolve('@activate-core/test-forge/reporters/jest-reporter.cjs'), { category: ${JSON.stringify(category)} }]],`,
|
|
324
|
+
`};`,
|
|
325
|
+
``,
|
|
326
|
+
].join('\n');
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Jest setup file (runs before each test file): loads the env so the suite is
|
|
330
|
+
* self-contained. Mirrors what a Playwright globalSetup + config does for web.
|
|
331
|
+
*/
|
|
332
|
+
export function jestSetupContent() {
|
|
333
|
+
return [
|
|
334
|
+
`// @generated by testforge`,
|
|
335
|
+
`'use strict';`,
|
|
336
|
+
`// Auto-loads test env so \`npm test\` / \`npx jest\` "just works" (no shell setup):`,
|
|
337
|
+
`// .env.testing, NODE_ENV=testing, and opt-in to the destructive test reset.`,
|
|
338
|
+
`const fs = require('fs');`,
|
|
339
|
+
`const path = require('path');`,
|
|
340
|
+
`let dotenv; try { dotenv = require('dotenv'); } catch (_) { dotenv = null; }`,
|
|
341
|
+
``,
|
|
342
|
+
`if (!process.env.NODE_ENV || process.env.NODE_ENV === 'test') {`,
|
|
343
|
+
` process.env.NODE_ENV = 'testing';`,
|
|
344
|
+
`}`,
|
|
345
|
+
`const envFile = path.resolve(__dirname, '../.env.testing');`,
|
|
346
|
+
`if (dotenv && fs.existsSync(envFile)) dotenv.config({ path: envFile });`,
|
|
347
|
+
`if (process.env.TESTFORGE_ALLOW_RESET === undefined) {`,
|
|
348
|
+
` process.env.TESTFORGE_ALLOW_RESET = 'true';`,
|
|
349
|
+
`}`,
|
|
350
|
+
``,
|
|
351
|
+
].join('\n');
|
|
352
|
+
}
|
|
353
|
+
/** Allure failure categories — buckets that show up in the report's "Categories"
|
|
354
|
+
* tab. Copied into allure-results/ so the report groups failures meaningfully. */
|
|
355
|
+
export function allureCategories() {
|
|
356
|
+
return JSON.stringify([
|
|
357
|
+
{ name: 'Product defect', matchedStatuses: ['failed'], messageRegex: '.*expect.*|.*assert.*' },
|
|
358
|
+
{ name: 'Test / harness issue', matchedStatuses: ['broken'] },
|
|
359
|
+
{ name: 'Flaky', matchedStatuses: ['failed', 'broken'], flaky: true },
|
|
360
|
+
], null, 2);
|
|
361
|
+
}
|
|
362
|
+
/** ALLURE.md — how to install the reporter deps and view the report. */
|
|
363
|
+
export function allureDoc() {
|
|
364
|
+
return [
|
|
365
|
+
`# Allure reporting (generated by testforge)`,
|
|
366
|
+
``,
|
|
367
|
+
`testforge wires the suites to emit **Allure** results. To view the report:`,
|
|
368
|
+
``,
|
|
369
|
+
`## 1. Install once (per package)`,
|
|
370
|
+
'```bash',
|
|
371
|
+
`# web (Playwright): npm i -D allure-playwright allure-commandline`,
|
|
372
|
+
`# api (Jest): npm i -D allure-jest allure-commandline`,
|
|
373
|
+
'```',
|
|
374
|
+
``,
|
|
375
|
+
`## 2. Run the tests`,
|
|
376
|
+
`Running the suite writes raw results into \`allure-results/\`.`,
|
|
377
|
+
``,
|
|
378
|
+
`## 3. Generate + open the report`,
|
|
379
|
+
'```bash',
|
|
380
|
+
`npx allure generate allure-results --clean -o allure-report`,
|
|
381
|
+
`npx allure open allure-report`,
|
|
382
|
+
'```',
|
|
383
|
+
`Or add scripts to package.json:`,
|
|
384
|
+
'```json',
|
|
385
|
+
`"report": "allure generate allure-results --clean -o allure-report && allure open allure-report"`,
|
|
386
|
+
'```',
|
|
387
|
+
``,
|
|
388
|
+
`## Categories`,
|
|
389
|
+
`\`allure-categories.json\` groups failures (Product defect / Test-harness / Flaky).`,
|
|
390
|
+
`Copy it to \`allure-results/categories.json\` before generating to activate it.`,
|
|
391
|
+
``,
|
|
392
|
+
`> \`allure-results/\`, \`allure-report/\` are build output — add them to .gitignore.`,
|
|
393
|
+
``,
|
|
394
|
+
].join('\n');
|
|
395
|
+
}
|
|
279
396
|
/**
|
|
280
397
|
* Playwright globalSetup file: runs the testkit's one-call setup() once
|
|
281
398
|
* before the suite (connect + seed + mint/populate tokens). Sits next to
|
package/dist/core/init.js
CHANGED
|
@@ -44,6 +44,10 @@ export function buildInitPrompt() {
|
|
|
44
44
|
` tests use, or the project's language (TypeScript vs JavaScript).`,
|
|
45
45
|
`- aiCommand: keep it as "claude -p" unless there is a clear reason`,
|
|
46
46
|
` to change it.`,
|
|
47
|
+
`- project: a short slug for this repo (kebab-case, e.g. the repo folder`,
|
|
48
|
+
` name lower-cased). Identifies the project on the TestForge dashboard.`,
|
|
49
|
+
`- dashboardUrl: leave as an empty string "". The developer fills in the`,
|
|
50
|
+
` TestForge dashboard URL when they connect the project.`,
|
|
47
51
|
``,
|
|
48
52
|
`Write valid JSON only, in this shape:`,
|
|
49
53
|
``,
|
|
@@ -52,7 +56,9 @@ export function buildInitPrompt() {
|
|
|
52
56
|
` "testOutput": { "<area>": "<path>" },`,
|
|
53
57
|
` "framework": { "<area>": "<jest|playwright|...>" },`,
|
|
54
58
|
` "seedOutput": "<path>",`,
|
|
55
|
-
` "testExtension": ".test.ts"
|
|
59
|
+
` "testExtension": ".test.ts",`,
|
|
60
|
+
` "project": "<repo-slug>",`,
|
|
61
|
+
` "dashboardUrl": ""`,
|
|
56
62
|
`}`,
|
|
57
63
|
``,
|
|
58
64
|
`Base every path on real evidence in the project (existing test`,
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
/**
|
|
3
|
+
* Single source of truth for TestForge's per-project state directory and the
|
|
4
|
+
* files/sub-dirs inside it. Everything the tool persists between runs lives
|
|
5
|
+
* here (like `.git` for git): the doc-hash cache, the source-of-truth choice,
|
|
6
|
+
* the coverage manifest, and the per-journey generation plans.
|
|
7
|
+
*
|
|
8
|
+
* Rename in ONE place and every command — setup, discover, generate, update,
|
|
9
|
+
* coverage — follows automatically, so no path can drift out of sync.
|
|
10
|
+
*/
|
|
11
|
+
export const STATE_DIR = 'test-artifacts';
|
|
12
|
+
/** Legacy state-dir name kept only so source-hash skip-lists still ignore a
|
|
13
|
+
* pre-migration folder if one is lying around. Not written to. */
|
|
14
|
+
export const LEGACY_STATE_DIR = '.testforge';
|
|
15
|
+
export const CACHE_PATH = join(STATE_DIR, 'cache.json');
|
|
16
|
+
export const SOURCE_PATH = join(STATE_DIR, 'source.json');
|
|
17
|
+
export const COVERAGE_PATH = join(STATE_DIR, 'coverage.json');
|
|
18
|
+
/** Per-journey generation plans, organised per area: <PLANS_DIR>/<area>/<id>.json */
|
|
19
|
+
export const PLANS_DIR = join(STATE_DIR, 'test-plans');
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* report — TestForge's built-in, public "publish results to the dashboard" step.
|
|
3
|
+
*
|
|
4
|
+
* The tool itself needs no auth to run this; authentication is the per-project
|
|
5
|
+
* API key (issued by the dashboard) supplied via the TESTFORGE_KEY env var.
|
|
6
|
+
* Project + dashboard URL come from flags or a committed testforge.config.json.
|
|
7
|
+
*/
|
|
8
|
+
import fs from 'node:fs';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import os from 'node:os';
|
|
11
|
+
import { execSync } from 'node:child_process';
|
|
12
|
+
/**
|
|
13
|
+
* Load TESTFORGE_KEY (and any other vars) from a gitignored `.testforge.env`
|
|
14
|
+
* (then `.env`) at the project root, so a developer can keep the key in a file
|
|
15
|
+
* instead of exporting it every time. Never overrides vars already in the
|
|
16
|
+
* environment (a real `export` still wins). Add `.testforge.env` to .gitignore.
|
|
17
|
+
*/
|
|
18
|
+
export function loadDotenv(cwd = process.cwd()) {
|
|
19
|
+
for (const file of ['.testforge.env', '.env']) {
|
|
20
|
+
const p = path.join(cwd, file);
|
|
21
|
+
if (!fs.existsSync(p))
|
|
22
|
+
continue;
|
|
23
|
+
for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
|
|
24
|
+
const s = line.trim();
|
|
25
|
+
if (!s || s.startsWith('#'))
|
|
26
|
+
continue;
|
|
27
|
+
const i = s.indexOf('=');
|
|
28
|
+
if (i === -1)
|
|
29
|
+
continue;
|
|
30
|
+
const k = s.slice(0, i).trim();
|
|
31
|
+
let v = s.slice(i + 1).trim();
|
|
32
|
+
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))
|
|
33
|
+
v = v.slice(1, -1);
|
|
34
|
+
if (k && process.env[k] === undefined)
|
|
35
|
+
process.env[k] = v;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** Read testforge.config.json (project, dashboardUrl) from the project root. */
|
|
40
|
+
export function loadReportConfig(cwd = process.cwd()) {
|
|
41
|
+
const p = path.join(cwd, 'testforge.config.json');
|
|
42
|
+
if (fs.existsSync(p)) {
|
|
43
|
+
try {
|
|
44
|
+
return JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
/* ignore malformed config */
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return {};
|
|
51
|
+
}
|
|
52
|
+
const strip = (s) => (s || '').replace(/\x1b\[[0-9;]*m/g, '');
|
|
53
|
+
/** Bucket a failure message into a root-cause signature the dashboard clusters on. */
|
|
54
|
+
function signature(msg) {
|
|
55
|
+
const m = strip(msg || '').replace(/\n/g, ' ');
|
|
56
|
+
if (/403|whoami|invalid_grant|refresh token/i.test(m))
|
|
57
|
+
return 'IM auth (403)';
|
|
58
|
+
if (/\$gt|\$exists|injection/i.test(m))
|
|
59
|
+
return 'NoSQL injection accepted';
|
|
60
|
+
if (/timeout|timed out|ETIMEDOUT/i.test(m))
|
|
61
|
+
return 'Timeout';
|
|
62
|
+
if (/\b500\b|Internal Server/i.test(m))
|
|
63
|
+
return 'Server 500';
|
|
64
|
+
if (/expect|toBe|toEqual|Received|Expected/i.test(m))
|
|
65
|
+
return 'Assertion mismatch';
|
|
66
|
+
return 'Uncategorised';
|
|
67
|
+
}
|
|
68
|
+
/** Map a Jest `--json` report to the dashboard run contract. */
|
|
69
|
+
export function mapJest(json, category = 'api') {
|
|
70
|
+
const suites = {};
|
|
71
|
+
for (const f of json.testResults || []) {
|
|
72
|
+
const name = String(f.name || f.testFilePath || 'suite')
|
|
73
|
+
.split('/').pop()
|
|
74
|
+
.replace(/\.(test|spec)\.[jt]s$/, '');
|
|
75
|
+
const su = (suites[name] ||= { name, category, tests: [] });
|
|
76
|
+
for (const a of f.assertionResults || []) {
|
|
77
|
+
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
|
|
78
|
+
const raw = a.failureMessages && a.failureMessages[0];
|
|
79
|
+
const err = strip(raw || '').split('\n').find((l) => l.trim()) || '';
|
|
80
|
+
su.tests.push({
|
|
81
|
+
name: a.title,
|
|
82
|
+
status,
|
|
83
|
+
error: status === 'failed' ? err.slice(0, 200) : undefined,
|
|
84
|
+
sig: status === 'failed' ? signature(raw) : undefined,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const durationSec = Math.round((json.testResults || []).reduce((a, f) => a + ((f.endTime || 0) - (f.startTime || 0)), 0) / 1000);
|
|
89
|
+
return { suites: Object.values(suites), durationSec };
|
|
90
|
+
}
|
|
91
|
+
export async function publishReport(opts) {
|
|
92
|
+
const json = JSON.parse(fs.readFileSync(opts.resultsPath, 'utf8'));
|
|
93
|
+
const { suites, durationSec } = mapJest(json, opts.category || 'api');
|
|
94
|
+
const body = { project: opts.project, env: opts.env || 'dev', durationSec, suites };
|
|
95
|
+
const res = await fetch(opts.dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
|
|
96
|
+
method: 'POST',
|
|
97
|
+
headers: { Authorization: `Bearer ${opts.apiKey}`, 'Content-Type': 'application/json' },
|
|
98
|
+
body: JSON.stringify(body),
|
|
99
|
+
});
|
|
100
|
+
return { status: res.status, text: await res.text() };
|
|
101
|
+
}
|
|
102
|
+
function walk(dir, ext, out) {
|
|
103
|
+
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
104
|
+
const full = path.join(dir, e.name);
|
|
105
|
+
if (e.isDirectory())
|
|
106
|
+
walk(full, ext, out);
|
|
107
|
+
else if (e.name.endsWith(ext))
|
|
108
|
+
out.push(full);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const suiteName = (file) => path.basename(file).replace(/\.(test|spec)\.[jt]sx?$/, '');
|
|
112
|
+
const staticCount = (file) => (fs.readFileSync(file, 'utf8').match(/\b(it|test)\s*\(/g) || []).length;
|
|
113
|
+
/** Static fallback: count `it(`/`test(` occurrences per file (undercounts parameterized tests). */
|
|
114
|
+
function jestStatic(dir, ext, category, out) {
|
|
115
|
+
const files = [];
|
|
116
|
+
walk(dir, ext, files);
|
|
117
|
+
for (const f of files)
|
|
118
|
+
out.push({ name: suiteName(f), category, total: staticCount(f) });
|
|
119
|
+
}
|
|
120
|
+
// setupFilesAfterEnv shim: runs after jest installs the test globals, replaces
|
|
121
|
+
// every test BODY and before/after hook with a no-op. Collection still runs
|
|
122
|
+
// (describe bodies, for-loops, .each all execute → real count), but nothing
|
|
123
|
+
// slow does — no DB, no assertions. Applied via a temp config that extends the
|
|
124
|
+
// project's own jest config, so env/mocks/moduleNameMapper/transforms are live.
|
|
125
|
+
const COLLECT_PATCH = `
|
|
126
|
+
const noop = () => {};
|
|
127
|
+
['it','test','fit','xit','xtest'].forEach((name) => {
|
|
128
|
+
const orig = global[name];
|
|
129
|
+
if (typeof orig !== 'function') return;
|
|
130
|
+
const wrap = (t,_fn,to) => orig(t, noop, to);
|
|
131
|
+
if (orig.each) wrap.each = (...a) => { const r = orig.each(...a); return (t,_fn,to) => r(t, noop, to); };
|
|
132
|
+
wrap.only = wrap; wrap.concurrent = wrap; wrap.failing = wrap;
|
|
133
|
+
wrap.skip = orig.skip ? ((t,_fn,to) => orig.skip(t, noop, to)) : wrap;
|
|
134
|
+
wrap.todo = orig.todo ? orig.todo.bind(orig) : ((t) => orig(t, noop));
|
|
135
|
+
global[name] = wrap;
|
|
136
|
+
});
|
|
137
|
+
global.beforeAll = noop; global.beforeEach = noop; global.afterAll = noop; global.afterEach = noop;
|
|
138
|
+
`;
|
|
139
|
+
const COLLECT_CONFIG = `
|
|
140
|
+
const fs = require('fs'), path = require('path');
|
|
141
|
+
let base = {};
|
|
142
|
+
for (const f of ['jest.config.js','jest.config.cjs','jest.config.json']) {
|
|
143
|
+
if (fs.existsSync(path.join(__dirname, f))) { base = require('./' + f); break; }
|
|
144
|
+
}
|
|
145
|
+
if (!Object.keys(base).length && fs.existsSync(path.join(__dirname,'package.json'))) base = require('./package.json').jest || {};
|
|
146
|
+
module.exports = {
|
|
147
|
+
...base,
|
|
148
|
+
testEnvironment: 'node', // avoid allure-jest side effects during a count-only run
|
|
149
|
+
setupFilesAfterEnv: [...(base.setupFilesAfterEnv || []), path.resolve(__dirname, '.tf-collect-patch.cjs')],
|
|
150
|
+
};
|
|
151
|
+
`;
|
|
152
|
+
/**
|
|
153
|
+
* Accurate for Jest: run the project's OWN jest with a temp config that no-ops
|
|
154
|
+
* test bodies + hooks. Files load exactly as under jest (setupFiles load
|
|
155
|
+
* .env.testing, jest.mock + transforms apply), so parameterized tests expand and
|
|
156
|
+
* nothing throws at import — but no bodies/DB run, so it's far faster than a real
|
|
157
|
+
* run. `numTotalTests` / per-file assertionResults give the true counts.
|
|
158
|
+
*/
|
|
159
|
+
function jestCount(dir, ext, category, out) {
|
|
160
|
+
const pkgDir = path.dirname(dir); // testOutput is <pkg>/tests → jest config lives in <pkg>
|
|
161
|
+
const patch = path.join(pkgDir, '.tf-collect-patch.cjs');
|
|
162
|
+
const cfg = path.join(pkgDir, '.tf-collect.config.cjs');
|
|
163
|
+
const outJson = path.join(os.tmpdir(), `tf-inv-${category}-${process.pid}.json`);
|
|
164
|
+
const files = [];
|
|
165
|
+
walk(dir, ext, files);
|
|
166
|
+
try {
|
|
167
|
+
fs.writeFileSync(patch, COLLECT_PATCH);
|
|
168
|
+
fs.writeFileSync(cfg, COLLECT_CONFIG);
|
|
169
|
+
try {
|
|
170
|
+
execSync('npx jest --config .tf-collect.config.cjs --json --outputFile=' + JSON.stringify(outJson) + ' --silent', {
|
|
171
|
+
cwd: pkgDir, stdio: ['ignore', 'ignore', 'ignore'], timeout: 300000,
|
|
172
|
+
env: { ...process.env, NODE_ENV: process.env.NODE_ENV || 'testing' },
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
catch { /* jest exits non-zero if a file errors at load; the --json is still written */ }
|
|
176
|
+
if (fs.existsSync(outJson)) {
|
|
177
|
+
const j = JSON.parse(fs.readFileSync(outJson, 'utf8'));
|
|
178
|
+
const seen = new Set();
|
|
179
|
+
for (const f of j.testResults || []) {
|
|
180
|
+
const n = suiteName(String(f.name || f.testFilePath || 'suite'));
|
|
181
|
+
seen.add(n);
|
|
182
|
+
out.push({ name: n, category, total: (f.assertionResults || []).length });
|
|
183
|
+
}
|
|
184
|
+
// any file jest couldn't even load → static, and say so loudly
|
|
185
|
+
for (const f of files) {
|
|
186
|
+
const n = suiteName(f);
|
|
187
|
+
if (seen.has(n))
|
|
188
|
+
continue;
|
|
189
|
+
const total = staticCount(f);
|
|
190
|
+
console.warn(` ! couldn't collect ${n} — static estimate (${total})`);
|
|
191
|
+
out.push({ name: n, category, total });
|
|
192
|
+
}
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
console.warn(` ! jest collect produced no output for "${category}" — static estimate for all suites`);
|
|
196
|
+
jestStatic(dir, ext, category, out);
|
|
197
|
+
}
|
|
198
|
+
finally {
|
|
199
|
+
for (const p of [patch, cfg, outJson]) {
|
|
200
|
+
try {
|
|
201
|
+
if (fs.existsSync(p))
|
|
202
|
+
fs.unlinkSync(p);
|
|
203
|
+
}
|
|
204
|
+
catch { /* best effort */ }
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Accurate for Playwright: `playwright test --list` does a collect pass that
|
|
210
|
+
* EXPANDS parameterized tests (loops / .each), so we get the real case count
|
|
211
|
+
* per file. Returns false if it couldn't run (caller falls back to static).
|
|
212
|
+
*/
|
|
213
|
+
function playwrightList(pkgDir, category, out) {
|
|
214
|
+
try {
|
|
215
|
+
const res = execSync('npx playwright test --list', {
|
|
216
|
+
cwd: pkgDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 180000,
|
|
217
|
+
});
|
|
218
|
+
const counts = {};
|
|
219
|
+
for (const line of res.split('\n')) {
|
|
220
|
+
const m = line.match(/([^\s›»]+\.(?:test|spec)\.[jt]sx?):\d+:\d+/);
|
|
221
|
+
if (m) {
|
|
222
|
+
const n = suiteName(m[1]);
|
|
223
|
+
counts[n] = (counts[n] || 0) + 1;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
const names = Object.keys(counts);
|
|
227
|
+
if (!names.length)
|
|
228
|
+
return false;
|
|
229
|
+
for (const n of names)
|
|
230
|
+
out.push({ name: n, category, total: counts[n] });
|
|
231
|
+
return true;
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
return false;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Build the per-suite inventory. Uses the framework's own enumeration where it
|
|
239
|
+
* exists (Playwright `--list` → real counts incl. parameterized), and a static
|
|
240
|
+
* estimate for Jest (whose true count is refined from actual runs on the dashboard).
|
|
241
|
+
*/
|
|
242
|
+
export function scanInventory(cwd, testOutput, ext, framework = {}) {
|
|
243
|
+
const suites = [];
|
|
244
|
+
for (const [category, rel] of Object.entries(testOutput || {})) {
|
|
245
|
+
const dir = path.join(cwd, rel);
|
|
246
|
+
if (!fs.existsSync(dir))
|
|
247
|
+
continue;
|
|
248
|
+
if (framework[category] === 'playwright') {
|
|
249
|
+
// package root (holds playwright.config) is the parent of the tests dir
|
|
250
|
+
if (playwrightList(path.dirname(dir), category, suites))
|
|
251
|
+
continue;
|
|
252
|
+
console.warn(` ! playwright --list failed for "${category}" — using static estimate`);
|
|
253
|
+
jestStatic(dir, ext, category, suites);
|
|
254
|
+
}
|
|
255
|
+
else {
|
|
256
|
+
// jest (and ops/lambda, also jest-based) — accurate collect-only counter
|
|
257
|
+
jestCount(dir, ext, category, suites);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
return suites;
|
|
261
|
+
}
|
|
262
|
+
export async function publishInventory(opts) {
|
|
263
|
+
const res = await fetch(opts.dashboardUrl.replace(/\/$/, '') + '/api/v1/inventory', {
|
|
264
|
+
method: 'POST',
|
|
265
|
+
headers: { Authorization: `Bearer ${opts.apiKey}`, 'Content-Type': 'application/json' },
|
|
266
|
+
body: JSON.stringify({ suites: opts.suites }),
|
|
267
|
+
});
|
|
268
|
+
return { status: res.status, text: await res.text() };
|
|
269
|
+
}
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync, } from 'node:fs';
|
|
20
20
|
import { join, dirname } from 'node:path';
|
|
21
|
+
import { CACHE_PATH, COVERAGE_PATH, PLANS_DIR, STATE_DIR } from './paths.js';
|
|
21
22
|
import { parseJourney } from './parser.js';
|
|
22
23
|
import { generateTestCases } from './generator.js';
|
|
23
24
|
import { writeDeferredStub } from './playwright-writer.js';
|
|
@@ -26,11 +27,10 @@ import { generateSeed } from './seed-generator.js';
|
|
|
26
27
|
import { planSuite } from './suite-planner.js';
|
|
27
28
|
import { isSyntaxValid, checkTestGates, analyzeFile } from './validate.js';
|
|
28
29
|
import { installedModules, transportHint, projectSelectors } from './detect.js';
|
|
29
|
-
import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, } from './harness.js';
|
|
30
|
+
import { harnessInfo, generateHarnessAsync, generateHarness, playwrightConfig, globalSetupContent, fixturesDoc, jestConfig, jestSetupContent, allureCategories, allureDoc, } from './harness.js';
|
|
30
31
|
import { hashContent, loadCache, saveCache } from './cache.js';
|
|
31
32
|
import { findMarkdownFiles } from './find-files.js';
|
|
32
33
|
import { loadConfig, testDirForType, resolveFramework } from './config.js';
|
|
33
|
-
const CACHE_PATH = '.testforge/cache.json';
|
|
34
34
|
/** How many times to re-ask the AI when output won't parse. */
|
|
35
35
|
const MAX_GEN_ATTEMPTS = 2;
|
|
36
36
|
/** How many focused files to generate concurrently within a journey. */
|
|
@@ -176,22 +176,16 @@ export async function runHarness(only) {
|
|
|
176
176
|
mkdirSync(dirname(info.path), { recursive: true });
|
|
177
177
|
writeFileSync(info.path, kit, 'utf-8');
|
|
178
178
|
writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
|
|
179
|
-
|
|
180
|
-
const gsPath = join(info.dir, 'global-setup.js');
|
|
181
|
-
writeFileSync(gsPath, globalSetupContent(), 'utf-8');
|
|
182
|
-
const pkgRoot = dirname(areaDir);
|
|
183
|
-
const cfg = join(pkgRoot, 'playwright.config.js');
|
|
184
|
-
if (!existsSync(cfg)) {
|
|
185
|
-
const rel = areaDir.slice(pkgRoot.length + 1) || '.';
|
|
186
|
-
const gsRel = gsPath.slice(pkgRoot.length + 1);
|
|
187
|
-
writeFileSync(cfg, playwrightConfig(`./${rel}`, `./${gsRel}`), 'utf-8');
|
|
188
|
-
}
|
|
189
|
-
}
|
|
179
|
+
writeHarnessConfigs(framework, areaDir, info.dir, area);
|
|
190
180
|
written.push(info.path);
|
|
191
181
|
}
|
|
192
182
|
return written;
|
|
193
183
|
}
|
|
194
|
-
|
|
184
|
+
/** Plan cache path for a journey, organised per area:
|
|
185
|
+
* `<PLANS_DIR>/<area>/<id>.json` (e.g. test-artifacts/test-plans/api/payment-api.json). */
|
|
186
|
+
function planPath(journey) {
|
|
187
|
+
return join(PLANS_DIR, journey.area, `${journey.id}.json`);
|
|
188
|
+
}
|
|
195
189
|
/** Marker every testforge-written test carries, so stale-file cleanup
|
|
196
190
|
* (F6) only ever removes OUR files, never hand-authored ones. */
|
|
197
191
|
const GEN_MARKER = '@generated by testforge';
|
|
@@ -245,7 +239,7 @@ function hasDeferredStubs(testDir, ext) {
|
|
|
245
239
|
}
|
|
246
240
|
/** Slugs recorded in a journey's existing plan cache (the previous run). */
|
|
247
241
|
function readPlanSlugs(journey) {
|
|
248
|
-
const p =
|
|
242
|
+
const p = planPath(journey);
|
|
249
243
|
try {
|
|
250
244
|
const c = JSON.parse(readFileSync(p, 'utf-8'));
|
|
251
245
|
if (Array.isArray(c.files)) {
|
|
@@ -290,7 +284,7 @@ function cleanStaleFiles(testDir, journeyId, ext, flat, keepSlugs, dropSlugs) {
|
|
|
290
284
|
* the doc changes, the hash changes and we re-plan.
|
|
291
285
|
*/
|
|
292
286
|
function loadOrPlan(journey, docHash) {
|
|
293
|
-
const p =
|
|
287
|
+
const p = planPath(journey);
|
|
294
288
|
if (existsSync(p)) {
|
|
295
289
|
try {
|
|
296
290
|
const cached = JSON.parse(readFileSync(p, 'utf-8'));
|
|
@@ -304,7 +298,7 @@ function loadOrPlan(journey, docHash) {
|
|
|
304
298
|
}
|
|
305
299
|
const files = planSuite(journey);
|
|
306
300
|
try {
|
|
307
|
-
mkdirSync(
|
|
301
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
308
302
|
writeFileSync(p, JSON.stringify({ hash: docHash, files }, null, 2), 'utf-8');
|
|
309
303
|
}
|
|
310
304
|
catch {
|
|
@@ -361,19 +355,9 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
361
355
|
mkdirSync(dirname(info.path), { recursive: true });
|
|
362
356
|
writeFileSync(info.path, kit, 'utf-8');
|
|
363
357
|
writeFileSync(join(info.dir, 'FIXTURES.md'), fixturesDoc(), 'utf-8');
|
|
364
|
-
//
|
|
365
|
-
// AND get their tokens minted once
|
|
366
|
-
|
|
367
|
-
const gsPath = join(info.dir, 'global-setup.js');
|
|
368
|
-
writeFileSync(gsPath, globalSetupContent(), 'utf-8');
|
|
369
|
-
const pkgRoot = dirname(areaDir);
|
|
370
|
-
const cfg = join(pkgRoot, 'playwright.config.js');
|
|
371
|
-
if (!existsSync(cfg)) {
|
|
372
|
-
const rel = areaDir.slice(pkgRoot.length + 1) || '.';
|
|
373
|
-
const gsRel = gsPath.slice(pkgRoot.length + 1);
|
|
374
|
-
writeFileSync(cfg, playwrightConfig(`./${rel}`, `./${gsRel}`), 'utf-8');
|
|
375
|
-
}
|
|
376
|
-
}
|
|
358
|
+
// Config + globalSetup (playwright) / config + setup (jest) so the specs run
|
|
359
|
+
// AND get their tokens minted once, plus Allure reporting scaffolding.
|
|
360
|
+
writeHarnessConfigs(framework, areaDir, info.dir, area);
|
|
377
361
|
harnessByArea.set(area, info.path);
|
|
378
362
|
return info.path;
|
|
379
363
|
}
|
|
@@ -529,8 +513,14 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
529
513
|
}
|
|
530
514
|
// Write IMMEDIATELY (not batched at the end of the journey), so an
|
|
531
515
|
// interruption preserves every file generated so far.
|
|
532
|
-
|
|
533
|
-
|
|
516
|
+
// keepDeferredStubs:false (config) — do NOT write a DEFERRED stub file;
|
|
517
|
+
// it still counts as deferred in the coverage report (analyzeFile's catch
|
|
518
|
+
// treats the missing file as real:false), so nothing is hidden — we just
|
|
519
|
+
// don't litter the tree with placeholder error files.
|
|
520
|
+
if (!(status === 'basic' && config.keepDeferredStubs === false)) {
|
|
521
|
+
mkdirSync(dirname(filePath), { recursive: true });
|
|
522
|
+
writeFileSync(filePath, content, 'utf-8');
|
|
523
|
+
}
|
|
534
524
|
return { slug: pf.slug, path: filePath, status };
|
|
535
525
|
});
|
|
536
526
|
const testFiles = generated.map((g) => ({
|
|
@@ -601,7 +591,7 @@ export async function runGenerate(docsDir, useAI = true, force = false, only, ar
|
|
|
601
591
|
* F8 — write an honest coverage manifest. For every generated file it
|
|
602
592
|
* records real (has assertions) vs deferred (pending only), so a "green"
|
|
603
593
|
* run is never mistaken for full coverage. Written to
|
|
604
|
-
*
|
|
594
|
+
* test-artifacts/coverage.json and summarised to the console.
|
|
605
595
|
*/
|
|
606
596
|
function writeCoverageManifest(results) {
|
|
607
597
|
const journeys = results
|
|
@@ -629,14 +619,51 @@ function writeCoverageManifest(results) {
|
|
|
629
619
|
const totalReal = journeys.reduce((n, j) => n + j.real, 0);
|
|
630
620
|
const totalDeferred = journeys.reduce((n, j) => n + j.deferred, 0);
|
|
631
621
|
try {
|
|
632
|
-
mkdirSync(
|
|
633
|
-
writeFileSync(
|
|
622
|
+
mkdirSync(STATE_DIR, { recursive: true });
|
|
623
|
+
writeFileSync(COVERAGE_PATH, JSON.stringify({ real: totalReal, deferred: totalDeferred, journeys }, null, 2), 'utf-8');
|
|
634
624
|
}
|
|
635
625
|
catch {
|
|
636
626
|
/* best-effort */
|
|
637
627
|
}
|
|
638
|
-
console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see
|
|
628
|
+
console.log(`\nCoverage: ${totalReal} real test file(s), ${totalDeferred} deferred/placeholder — see ${COVERAGE_PATH}`);
|
|
639
629
|
if (totalDeferred) {
|
|
640
630
|
console.log(` (deferred files are pending, NOT real coverage — regenerate or implement them.)`);
|
|
641
631
|
}
|
|
642
632
|
}
|
|
633
|
+
/**
|
|
634
|
+
* Write the per-area runner config + Allure reporting scaffolding.
|
|
635
|
+
* - playwright: global-setup.js + playwright.config.js (allure-playwright reporter)
|
|
636
|
+
* - jest: helpers/jest.setup.js + jest.config.js (allure-jest env)
|
|
637
|
+
* - both: ALLURE.md + allure-categories.json at the package root
|
|
638
|
+
* Config files are written only if absent, so a project's hand-tuned config is
|
|
639
|
+
* never clobbered. `harnessDir` is the area's helpers dir (where setup lives).
|
|
640
|
+
*/
|
|
641
|
+
function writeHarnessConfigs(framework, areaDir, harnessDir, area) {
|
|
642
|
+
const pkgRoot = dirname(areaDir);
|
|
643
|
+
const rel = areaDir.slice(pkgRoot.length + 1) || '.';
|
|
644
|
+
// Allure docs + categories (both frameworks); never overwrite.
|
|
645
|
+
const allureMd = join(pkgRoot, 'ALLURE.md');
|
|
646
|
+
if (!existsSync(allureMd))
|
|
647
|
+
writeFileSync(allureMd, allureDoc(), 'utf-8');
|
|
648
|
+
const cats = join(pkgRoot, 'allure-categories.json');
|
|
649
|
+
if (!existsSync(cats))
|
|
650
|
+
writeFileSync(cats, allureCategories(), 'utf-8');
|
|
651
|
+
if (framework === 'playwright') {
|
|
652
|
+
const gsPath = join(harnessDir, 'global-setup.js');
|
|
653
|
+
writeFileSync(gsPath, globalSetupContent(), 'utf-8');
|
|
654
|
+
const cfg = join(pkgRoot, 'playwright.config.js');
|
|
655
|
+
if (!existsSync(cfg)) {
|
|
656
|
+
const gsRel = gsPath.slice(pkgRoot.length + 1);
|
|
657
|
+
writeFileSync(cfg, playwrightConfig(`./${rel}`, `./${gsRel}`, area), 'utf-8');
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
else if (framework === 'jest') {
|
|
661
|
+
const setupPath = join(harnessDir, 'jest.setup.js');
|
|
662
|
+
writeFileSync(setupPath, jestSetupContent(), 'utf-8');
|
|
663
|
+
const cfg = join(pkgRoot, 'jest.config.js');
|
|
664
|
+
if (!existsSync(cfg)) {
|
|
665
|
+
const setupRel = setupPath.slice(pkgRoot.length + 1);
|
|
666
|
+
writeFileSync(cfg, jestConfig(rel, setupRel, area), 'utf-8');
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
}
|
package/dist/core/run-tests.js
CHANGED
|
@@ -30,7 +30,7 @@ function packageRoot(dir) {
|
|
|
30
30
|
}
|
|
31
31
|
export function runTests(opts = {}) {
|
|
32
32
|
const config = loadConfig();
|
|
33
|
-
const envFile = config.envFile ?? '.env.testing';
|
|
33
|
+
const envFile = opts.envFile ?? config.envFile ?? '.env.testing';
|
|
34
34
|
const envVars = loadEnvFile(envFile);
|
|
35
35
|
if (Object.keys(envVars).length > 0) {
|
|
36
36
|
console.log(`Loaded env from ${envFile} (${Object.keys(envVars).length} vars).`);
|
package/dist/core/setup.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { existsSync } from 'node:fs';
|
|
14
14
|
import { hashProjectSource, loadCache, saveCache } from './cache.js';
|
|
15
|
+
import { CACHE_PATH } from './paths.js';
|
|
15
16
|
import { runInit } from './init.js';
|
|
16
17
|
import { runMap } from './map.js';
|
|
17
18
|
import { runDiscover } from './discover.js';
|
|
@@ -55,9 +56,9 @@ export async function runSetup(srcDir, opts = {}) {
|
|
|
55
56
|
await runDiscover(srcDir, 'docs', opts);
|
|
56
57
|
console.log(' done — docs/\n');
|
|
57
58
|
// Remember the source state so the first `update` can detect "no change".
|
|
58
|
-
const cache = loadCache(
|
|
59
|
+
const cache = loadCache(CACHE_PATH);
|
|
59
60
|
cache['__source__'] = hashProjectSource(srcDir);
|
|
60
|
-
saveCache(
|
|
61
|
+
saveCache(CACHE_PATH, cache);
|
|
61
62
|
// Build an env template so the developer knows which credentials each
|
|
62
63
|
// area/journey needs for running the tests.
|
|
63
64
|
console.log('Building env template...');
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* 2. An external reference — a spec/doc/PDF in the repo, a website, or
|
|
10
10
|
* a Notion page capturing the human/client EXPECTED behavior.
|
|
11
11
|
*
|
|
12
|
-
* The choice is remembered in
|
|
12
|
+
* The choice is remembered in test-artifacts/source.json so later refreshes
|
|
13
13
|
* (`update`) keep using the same source without asking again — otherwise
|
|
14
14
|
* an update would silently regenerate reference-based docs from the code.
|
|
15
15
|
*
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
|
|
23
23
|
import { dirname } from 'node:path';
|
|
24
24
|
import { isInteractive, promptChoice, promptLine } from './prompt.js';
|
|
25
|
-
|
|
25
|
+
import { SOURCE_PATH } from './paths.js';
|
|
26
26
|
/** Read the remembered source choice, or null if none/invalid. */
|
|
27
27
|
export function loadSourceChoice() {
|
|
28
28
|
if (!existsSync(SOURCE_PATH))
|
package/dist/core/update.js
CHANGED
|
@@ -14,7 +14,7 @@ import { runDiscover } from './discover.js';
|
|
|
14
14
|
import { runGenerate } from './run-generate.js';
|
|
15
15
|
import { hashProjectSource, loadCache, saveCache } from './cache.js';
|
|
16
16
|
import { runEnvTemplate } from './env-template.js';
|
|
17
|
-
|
|
17
|
+
import { CACHE_PATH } from './paths.js';
|
|
18
18
|
const SOURCE_KEY = '__source__';
|
|
19
19
|
export async function runUpdate(srcDir) {
|
|
20
20
|
if (!existsSync('testforge.md') || !existsSync('testforge.config.json')) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@activate-core/test-forge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "AI-powered QA automation. Scans a project, drafts journey docs, and generates seeded Jest/Playwright tests. Works in any project, with any AI CLI.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
"main": "./dist/core/index.js",
|
|
10
10
|
"files": [
|
|
11
11
|
"dist",
|
|
12
|
+
"reporters",
|
|
12
13
|
"README.md"
|
|
13
14
|
],
|
|
14
15
|
"scripts": {
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
// Shared helpers for the TestForge auto-publish reporters (plain CJS so jest &
|
|
2
|
+
// playwright can require() them directly). No build step — runtime files.
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { execSync } = require('child_process');
|
|
6
|
+
|
|
7
|
+
// Walk up from cwd to the repo root (where testforge.config.json lives) so the
|
|
8
|
+
// reporter works even when the framework runs from a package subdir (monorepo).
|
|
9
|
+
function findRoot(cwd) {
|
|
10
|
+
let d = cwd;
|
|
11
|
+
for (let i = 0; i < 8; i++) {
|
|
12
|
+
if (fs.existsSync(path.join(d, 'testforge.config.json'))) return d;
|
|
13
|
+
const up = path.dirname(d);
|
|
14
|
+
if (up === d) break;
|
|
15
|
+
d = up;
|
|
16
|
+
}
|
|
17
|
+
return cwd;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Load TESTFORGE_KEY from a gitignored .testforge.env / .env (never overrides a real env var).
|
|
21
|
+
function loadEnv(root) {
|
|
22
|
+
for (const f of ['.testforge.env', '.env']) {
|
|
23
|
+
const p = path.join(root, f);
|
|
24
|
+
if (!fs.existsSync(p)) continue;
|
|
25
|
+
for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
|
|
26
|
+
const s = line.trim();
|
|
27
|
+
if (!s || s.startsWith('#')) continue;
|
|
28
|
+
const i = s.indexOf('=');
|
|
29
|
+
if (i === -1) continue;
|
|
30
|
+
const k = s.slice(0, i).trim();
|
|
31
|
+
let v = s.slice(i + 1).trim();
|
|
32
|
+
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
|
|
33
|
+
if (k && process.env[k] === undefined) process.env[k] = v;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function loadConfig(root) {
|
|
39
|
+
try { return JSON.parse(fs.readFileSync(path.join(root, 'testforge.config.json'), 'utf8')); }
|
|
40
|
+
catch { return {}; }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const strip = (s) => (s || '').replace(/\x1b\[[0-9;]*m/g, '');
|
|
44
|
+
|
|
45
|
+
function sh(cmd, cwd) {
|
|
46
|
+
try { return (execSync(cmd, { cwd, stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim()) || undefined; }
|
|
47
|
+
catch { return undefined; }
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Where did the tests run? branch + sha + ci build — from CI env vars first, else git.
|
|
51
|
+
function gitInfo(root) {
|
|
52
|
+
let branch = process.env.TESTFORGE_BRANCH
|
|
53
|
+
|| process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME
|
|
54
|
+
|| process.env.CIRCLE_BRANCH || process.env.CI_COMMIT_REF_NAME
|
|
55
|
+
|| process.env.GIT_BRANCH || process.env.BRANCH_NAME
|
|
56
|
+
|| sh('git rev-parse --abbrev-ref HEAD', root);
|
|
57
|
+
if (branch) branch = branch.replace(/^refs\/heads\//, '').replace(/^origin\//, '');
|
|
58
|
+
if (branch === 'HEAD') branch = undefined; // detached checkout
|
|
59
|
+
const sha = process.env.GITHUB_SHA || process.env.CIRCLE_SHA1
|
|
60
|
+
|| process.env.CI_COMMIT_SHA || process.env.GIT_COMMIT
|
|
61
|
+
|| sh('git rev-parse --short HEAD', root);
|
|
62
|
+
const ciBuild = process.env.GITHUB_RUN_ID || process.env.CIRCLE_BUILD_NUM || process.env.BUILD_NUMBER;
|
|
63
|
+
return { branch: branch || undefined, sha: sha || undefined, ciBuild: ciBuild || undefined };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Which environment does this run represent? The env is the TARGET you tested (which DB/host),
|
|
67
|
+
// NOT the git branch — a reporter can't sniff the DB, so it's set explicitly via TESTFORGE_ENV
|
|
68
|
+
// in the same env-file that points the tests at that DB (stage DB config → TESTFORGE_ENV=stage).
|
|
69
|
+
// Bare local run with nothing set ⇒ "local". Branch/sha are recorded separately as identity only.
|
|
70
|
+
function resolveEnv() {
|
|
71
|
+
return process.env.TESTFORGE_ENV || 'local';
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function sig(msg) {
|
|
75
|
+
const m = strip(msg || '').replace(/\n/g, ' ');
|
|
76
|
+
if (/403|whoami|invalid_grant|refresh token/i.test(m)) return 'IM auth (403)';
|
|
77
|
+
if (/\$gt|\$exists|injection/i.test(m)) return 'NoSQL injection accepted';
|
|
78
|
+
if (/timeout|timed out|ETIMEDOUT/i.test(m)) return 'Timeout';
|
|
79
|
+
if (/\b500\b|Internal Server/i.test(m)) return 'Server 500';
|
|
80
|
+
if (/expect|toBe|toEqual|Received|Expected/i.test(m)) return 'Assertion mismatch';
|
|
81
|
+
return 'Uncategorised';
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// POST the run to the dashboard. Never throws — a publish problem must not fail the test run.
|
|
85
|
+
async function publish(category, suites, durationSec) {
|
|
86
|
+
const root = findRoot(process.cwd());
|
|
87
|
+
loadEnv(root);
|
|
88
|
+
const cfg = loadConfig(root);
|
|
89
|
+
const project = cfg.project, dashboardUrl = cfg.dashboardUrl, key = process.env.TESTFORGE_KEY;
|
|
90
|
+
if (!project || !dashboardUrl || !key) {
|
|
91
|
+
console.warn('[testforge] skipped publish — need "project" + "dashboardUrl" in testforge.config.json and TESTFORGE_KEY in .testforge.env');
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const { branch, sha, ciBuild } = gitInfo(root);
|
|
95
|
+
const env = resolveEnv();
|
|
96
|
+
try {
|
|
97
|
+
const res = await fetch(dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
|
|
98
|
+
method: 'POST',
|
|
99
|
+
headers: { Authorization: 'Bearer ' + key, 'Content-Type': 'application/json' },
|
|
100
|
+
body: JSON.stringify({ project, env, branch, sha, ciBuild, durationSec, suites }),
|
|
101
|
+
});
|
|
102
|
+
if (res.ok) console.log(`[testforge] ✓ published ${category} run (env: ${env}${branch ? ` · ${branch}` : ''}) → ${dashboardUrl}`);
|
|
103
|
+
else console.warn(`[testforge] publish failed (${res.status}): ${(await res.text()).slice(0, 200)}`);
|
|
104
|
+
} catch (e) {
|
|
105
|
+
console.warn('[testforge] publish error: ' + (e && e.message));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
module.exports = { strip, sig, publish, gitInfo, resolveEnv };
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
|
+
// Add to jest.config.js:
|
|
3
|
+
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
+
const { strip, sig, publish } = require('./_shared.cjs');
|
|
5
|
+
|
|
6
|
+
module.exports = class TestForgeJestReporter {
|
|
7
|
+
constructor(_globalConfig, options) {
|
|
8
|
+
this.category = (options && options.category) || 'api';
|
|
9
|
+
}
|
|
10
|
+
async onRunComplete(_contexts, results) {
|
|
11
|
+
const suites = {};
|
|
12
|
+
for (const f of results.testResults || []) {
|
|
13
|
+
const name = String(f.testFilePath || f.name || 'suite').split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
|
|
14
|
+
const su = suites[name] || (suites[name] = { name, category: this.category, tests: [] });
|
|
15
|
+
for (const a of (f.testResults || f.assertionResults || [])) {
|
|
16
|
+
const status = a.status === 'passed' ? 'passed' : a.status === 'failed' ? 'failed' : 'skipped';
|
|
17
|
+
const raw = (a.failureMessages || [])[0];
|
|
18
|
+
const err = strip(raw || '').split('\n').find((l) => l && l.trim()) || '';
|
|
19
|
+
su.tests.push({
|
|
20
|
+
name: a.fullName || a.title,
|
|
21
|
+
status,
|
|
22
|
+
error: status === 'failed' ? err.slice(0, 200) : undefined,
|
|
23
|
+
sig: status === 'failed' ? sig(raw) : undefined,
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const durationSec = Math.round((results.startTime ? Date.now() - results.startTime : 0) / 1000);
|
|
28
|
+
await publish(this.category, Object.values(suites), durationSec);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
|
|
2
|
+
// Add to playwright.config.js reporter array:
|
|
3
|
+
// ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
|
|
4
|
+
const { strip, sig, publish } = require('./_shared.cjs');
|
|
5
|
+
|
|
6
|
+
module.exports = class TestForgePlaywrightReporter {
|
|
7
|
+
constructor(options) {
|
|
8
|
+
this.category = (options && options.category) || 'web';
|
|
9
|
+
this.suites = {};
|
|
10
|
+
this.start = Date.now();
|
|
11
|
+
}
|
|
12
|
+
onTestEnd(test, result) {
|
|
13
|
+
const file = (test.location && test.location.file) || 'suite';
|
|
14
|
+
const name = String(file).split('/').pop().replace(/\.(test|spec)\.[jt]s$/, '');
|
|
15
|
+
const su = this.suites[name] || (this.suites[name] = { name, category: this.category, tests: [] });
|
|
16
|
+
const status = result.status === 'passed' ? 'passed' : result.status === 'skipped' ? 'skipped' : 'failed';
|
|
17
|
+
const raw = result.error && result.error.message;
|
|
18
|
+
const err = strip(raw || '').split('\n').find((l) => l && l.trim()) || '';
|
|
19
|
+
su.tests.push({
|
|
20
|
+
name: test.title,
|
|
21
|
+
status,
|
|
22
|
+
error: status === 'failed' ? err.slice(0, 200) : undefined,
|
|
23
|
+
sig: status === 'failed' ? sig(raw) : undefined,
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
async onEnd(result) {
|
|
27
|
+
const durationSec = Math.round(((result && result.duration) || (Date.now() - this.start)) / 1000);
|
|
28
|
+
await publish(this.category, Object.values(this.suites), durationSec);
|
|
29
|
+
}
|
|
30
|
+
};
|