@activate-core/test-forge 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/cli/index.js +60 -2
- package/dist/core/docs/discover.js +2 -2
- package/dist/core/docs/grounding.js +2 -2
- package/dist/core/docs/reference.js +8 -2
- package/dist/core/docs/review.js +43 -0
- package/dist/core/docs/tickets.js +56 -0
- package/dist/core/gates/coverage.js +11 -3
- package/dist/core/gates/validate.js +109 -0
- package/dist/core/generate/ai-generator.js +52 -3
- package/dist/core/generate/ai-session.js +8 -0
- package/dist/core/generate/ai-usage.js +39 -0
- package/dist/core/generate/defect-review.js +154 -11
- package/dist/core/generate/harness.js +50 -2
- package/dist/core/generate/required-data.js +75 -0
- package/dist/core/generate/review-journey.js +104 -0
- package/dist/core/generate/run-generate.js +26 -4
- package/dist/core/generate/seed-generator.js +2 -2
- package/dist/core/generate/suite-planner.js +10 -2
- package/dist/core/generate/verify-journey.js +46 -5
- package/dist/core/generate/verify.js +9 -4
- package/dist/core/project/env-diff.js +122 -0
- package/dist/core/project/env-template.js +1 -1
- package/dist/core/project/init.js +2 -2
- package/dist/core/project/map.js +2 -2
- package/dist/core/run/ci.js +94 -0
- package/dist/core/run/clean.js +111 -0
- package/dist/core/run/report.js +2 -0
- package/package.json +1 -1
- package/reporters/_shared.cjs +24 -3
- package/reporters/jest-reporter.cjs +3 -1
- package/reporters/playwright-reporter.cjs +3 -1
- package/reporters/run-data.cjs +66 -0
- package/reporters/ui-helpers.cjs +56 -0
package/reporters/_shared.cjs
CHANGED
|
@@ -332,7 +332,8 @@ async function publish(category, suites, durationSec) {
|
|
|
332
332
|
const root = findRoot(process.cwd());
|
|
333
333
|
loadEnv(root);
|
|
334
334
|
const cfg = loadConfig(root);
|
|
335
|
-
|
|
335
|
+
// In CI the deployed dashboard comes from the environment; locally, from the config.
|
|
336
|
+
const project = cfg.project, dashboardUrl = process.env.TESTFORGE_DASHBOARD_URL || cfg.dashboardUrl, key = process.env.TESTFORGE_KEY;
|
|
336
337
|
if (!project || !dashboardUrl || !key) {
|
|
337
338
|
console.warn('[testforge] skipped publish — need "project" + "dashboardUrl" in testforge.config.json and TESTFORGE_KEY in .testforge.env');
|
|
338
339
|
return;
|
|
@@ -343,7 +344,7 @@ async function publish(category, suites, durationSec) {
|
|
|
343
344
|
const res = await fetch(dashboardUrl.replace(/\/$/, '') + '/api/v1/runs', {
|
|
344
345
|
method: 'POST',
|
|
345
346
|
headers: { Authorization: 'Bearer ' + key, 'Content-Type': 'application/json' },
|
|
346
|
-
body: JSON.stringify({ project, env, branch, sha, ciBuild, durationSec, suites }),
|
|
347
|
+
body: JSON.stringify({ project, env, branch, sha, ciBuild, durationSec, suites, runId: process.env.TESTFORGE_RUN_ID || undefined }),
|
|
347
348
|
});
|
|
348
349
|
if (res.ok) console.log(`[testforge] ✓ published ${category} run (env: ${env}${branch ? ` · ${branch}` : ''}) → ${dashboardUrl}`);
|
|
349
350
|
else console.warn(`[testforge] publish failed (${res.status}): ${(await res.text()).slice(0, 200)}`);
|
|
@@ -373,4 +374,24 @@ function testRoot(root, category) {
|
|
|
373
374
|
return d ? path.resolve(root, d) : undefined;
|
|
374
375
|
}
|
|
375
376
|
|
|
376
|
-
|
|
377
|
+
// The defect review's verdict for each failed test (test-artifacts/defect-review/*.json),
|
|
378
|
+
// so the dashboard can say whether a failure is a product defect, a test fault or the
|
|
379
|
+
// environment. Keyed by absolute file + test title; an absent or broken file means none.
|
|
380
|
+
function reviewVerdicts(root) {
|
|
381
|
+
const map = new Map();
|
|
382
|
+
const dir = path.join(root || process.cwd(), 'test-artifacts', 'defect-review');
|
|
383
|
+
let files = [];
|
|
384
|
+
try { files = fs.readdirSync(dir).filter((f) => f.endsWith('.json')); } catch { /* no reviews yet */ }
|
|
385
|
+
for (const f of files) {
|
|
386
|
+
try {
|
|
387
|
+
for (const r of JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')) || []) {
|
|
388
|
+
for (const t of r.tests || []) {
|
|
389
|
+
map.set(`${path.resolve(String(t.file))}|${t.title}`, { verdict: r.verdict, verdictReason: r.reason || undefined, ...(r.impact ? { impact: r.impact } : {}) });
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
} catch { /* a half-written review is ignored */ }
|
|
393
|
+
}
|
|
394
|
+
return (file, title) => (file ? map.get(`${path.resolve(String(file))}|${title}`) : undefined);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
module.exports = { strip, cleanError, sig, publish, gitInfo, resolveEnv, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts };
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
// TestForge Jest reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to jest.config.js:
|
|
3
3
|
// reporters: ['default', ['<abs>/reporters/jest-reporter.cjs', { category: 'api' }]]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, expectation, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgeJestReporter {
|
|
7
7
|
constructor(_globalConfig, options) {
|
|
8
8
|
this.category = (options && options.category) || 'api';
|
|
9
9
|
this.testRoot = testRoot(findRoot(process.cwd()), this.category);
|
|
10
|
+
this.verdictOf = reviewVerdicts(findRoot(process.cwd()));
|
|
10
11
|
}
|
|
11
12
|
async onRunComplete(_contexts, results) {
|
|
12
13
|
const suites = {};
|
|
@@ -27,6 +28,7 @@ module.exports = class TestForgeJestReporter {
|
|
|
27
28
|
checks: testChecks(f.testFilePath, a.location && a.location.line),
|
|
28
29
|
...(status === 'failed' ? expectation(raw) : {}),
|
|
29
30
|
detail: status === 'failed' ? failureDetail(raw, { file: f.testFilePath, line: a.location && a.location.line }) : undefined,
|
|
31
|
+
...(status === 'failed' ? this.verdictOf(f.testFilePath, a.fullName || a.title) : {}),
|
|
30
32
|
});
|
|
31
33
|
}
|
|
32
34
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// TestForge Playwright reporter — auto-publishes the run to the dashboard on completion.
|
|
2
2
|
// Add to playwright.config.js reporter array:
|
|
3
3
|
// ['<abs>/reporters/playwright-reporter.cjs', { category: 'web' }]
|
|
4
|
-
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot } = require('./_shared.cjs');
|
|
4
|
+
const { cleanError, sig, publish, testChecks, failureDetail, findRoot, suiteName, testRoot, reviewVerdicts } = require('./_shared.cjs');
|
|
5
5
|
|
|
6
6
|
module.exports = class TestForgePlaywrightReporter {
|
|
7
7
|
constructor(options) {
|
|
@@ -10,6 +10,7 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
10
10
|
this.start = Date.now();
|
|
11
11
|
this.root = findRoot(process.cwd());
|
|
12
12
|
this.testRoot = testRoot(this.root, this.category);
|
|
13
|
+
this.verdictOf = reviewVerdicts(this.root);
|
|
13
14
|
}
|
|
14
15
|
onBegin(config) {
|
|
15
16
|
// Playwright's own testDir is the authority when it has one.
|
|
@@ -47,6 +48,7 @@ module.exports = class TestForgePlaywrightReporter {
|
|
|
47
48
|
checks: ((test.annotations || []).find((x) => x.type === 'description') || {}).description
|
|
48
49
|
|| testChecks(test.location && test.location.file, test.location && test.location.line),
|
|
49
50
|
...(status === 'failed' ? this.failed(test, result) : {}),
|
|
51
|
+
...(status === 'failed' ? this.verdictOf(test.location && test.location.file, test.title) : {}),
|
|
50
52
|
});
|
|
51
53
|
}
|
|
52
54
|
failed(test, result) {
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// TestForge run data — one id per test run, shared by every worker, and a record
|
|
2
|
+
// of what the run created so it can be traced on the dashboard and cleaned later.
|
|
3
|
+
//
|
|
4
|
+
// The generated runner config calls runId() at load, so the main process sets
|
|
5
|
+
// TESTFORGE_RUN_ID before any worker starts and every worker inherits it.
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const crypto = require('crypto');
|
|
9
|
+
const { findRoot } = require('./_shared.cjs');
|
|
10
|
+
|
|
11
|
+
const dir = (root) => path.join(root || findRoot(process.cwd()), 'test-artifacts', 'run-data');
|
|
12
|
+
|
|
13
|
+
/** The current run's id: <yyyymmdd>-<6 hex>, created once and kept in the env. */
|
|
14
|
+
function runId() {
|
|
15
|
+
if (!process.env.TESTFORGE_RUN_ID) {
|
|
16
|
+
process.env.TESTFORGE_RUN_ID = `${new Date().toISOString().slice(0, 10).replace(/-/g, '')}-${crypto.randomBytes(3).toString('hex')}`;
|
|
17
|
+
}
|
|
18
|
+
return process.env.TESTFORGE_RUN_ID;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** A value no other run produces: label + the run id's last 6 characters (letters and digits only). */
|
|
22
|
+
function unique(label = '') {
|
|
23
|
+
return `${label}${runId().slice(-6)}`;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Record one thing the run created. entry: { kind, id, role?, cleanup?: { method, url } }
|
|
28
|
+
* — cleanup is the request that removes or cancels it, when the product has one.
|
|
29
|
+
* Appending one JSON line is safe from several workers at once.
|
|
30
|
+
*/
|
|
31
|
+
function record(entry) {
|
|
32
|
+
const d = dir();
|
|
33
|
+
fs.mkdirSync(d, { recursive: true });
|
|
34
|
+
// The testkit that recorded it, so `testforge clean` signs in through the same one.
|
|
35
|
+
const kit = callerFile();
|
|
36
|
+
fs.appendFileSync(path.join(d, `${runId()}.jsonl`), JSON.stringify({ at: new Date().toISOString(), runId: runId(), ...(kit ? { kit } : {}), ...entry }) + '\n');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function callerFile() {
|
|
40
|
+
const lines = String(new Error().stack || '').split('\n').slice(1);
|
|
41
|
+
for (const l of lines) {
|
|
42
|
+
const m = /\(?((?:file:\/\/)?\/[^():]+):\d+:\d+\)?\s*$/.exec(l);
|
|
43
|
+
if (m && !m[1].endsWith('run-data.cjs')) return m[1].replace(/^file:\/\//, '');
|
|
44
|
+
}
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Every record of one run. */
|
|
49
|
+
function readRun(id, root) {
|
|
50
|
+
try {
|
|
51
|
+
return fs.readFileSync(path.join(dir(root), `${id}.jsonl`), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
|
52
|
+
} catch {
|
|
53
|
+
return [];
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Runs that recorded data, oldest first: { runId, at, count }. */
|
|
58
|
+
function listRuns(root) {
|
|
59
|
+
let files = [];
|
|
60
|
+
try { files = fs.readdirSync(dir(root)).filter((f) => f.endsWith('.jsonl')); } catch { return []; }
|
|
61
|
+
return files
|
|
62
|
+
.map((f) => { const id = f.slice(0, -6); const rows = readRun(id, root); return { runId: id, at: rows[0] ? rows[0].at : '', count: rows.length }; })
|
|
63
|
+
.sort((a, b) => (a.at < b.at ? -1 : 1));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
module.exports = { runId, unique, record, readRun, listRuns };
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// TestForge UI helpers for Playwright specs — shipped once, re-exported by every
|
|
2
|
+
// generated testkit, so no spec redefines (and diverges on) the same helper.
|
|
3
|
+
const settle = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Click a control once its layout has stopped moving. A pop-up whose content
|
|
7
|
+
* grows (a payment form opening extra fields, a list loading) scrolls the
|
|
8
|
+
* control while Playwright's own click is in flight, and the click lands on
|
|
9
|
+
* whatever is underneath. Scroll it into view, wait until its box is the same
|
|
10
|
+
* twice in a row, then click.
|
|
11
|
+
*/
|
|
12
|
+
async function settledClick(locator, { settleMs = 300, tries = 10 } = {}) {
|
|
13
|
+
await locator.scrollIntoViewIfNeeded();
|
|
14
|
+
let prev = JSON.stringify(await locator.boundingBox());
|
|
15
|
+
for (let i = 0; i < tries; i++) {
|
|
16
|
+
await settle(settleMs);
|
|
17
|
+
const cur = JSON.stringify(await locator.boundingBox());
|
|
18
|
+
if (cur === prev) break;
|
|
19
|
+
prev = cur;
|
|
20
|
+
}
|
|
21
|
+
await locator.click();
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Fill named inputs inside an iframe found by its name prefix (an embedded
|
|
26
|
+
* payment or address element). values: { inputName: value }. An input the
|
|
27
|
+
* frame does not show is skipped, so one call fits every variant of the form.
|
|
28
|
+
* Returns the names it filled.
|
|
29
|
+
*/
|
|
30
|
+
async function fillFrameInputs(page, framePrefix, values) {
|
|
31
|
+
const frame = page.frameLocator(`iframe[name^="${framePrefix}"]`).first();
|
|
32
|
+
const filled = [];
|
|
33
|
+
for (const [name, value] of Object.entries(values)) {
|
|
34
|
+
const input = frame.locator(`input[name="${name}"]`);
|
|
35
|
+
if (await input.isVisible().catch(() => false)) {
|
|
36
|
+
await input.fill(String(value));
|
|
37
|
+
filled.push(name);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return filled;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Accept every browser dialog (alert / confirm) and keep its message, in order.
|
|
45
|
+
* Call before the action that raises it. Returns the live array of messages.
|
|
46
|
+
*/
|
|
47
|
+
function watchDialogs(page) {
|
|
48
|
+
const messages = [];
|
|
49
|
+
page.on('dialog', (d) => {
|
|
50
|
+
messages.push(d.message());
|
|
51
|
+
d.accept().catch(() => {});
|
|
52
|
+
});
|
|
53
|
+
return messages;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
module.exports = { settledClick, fillFrameInputs, watchDialogs };
|