laxy-verify 1.1.16 → 1.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -77
- package/dist/cli.js +179 -27
- package/dist/config.d.ts +18 -0
- package/dist/config.js +59 -0
- package/dist/crawler.d.ts +35 -0
- package/dist/crawler.js +357 -0
- package/dist/e2e.d.ts +12 -2
- package/dist/e2e.js +127 -6
- package/dist/lighthouse.js +44 -26
- package/dist/multi-viewport.d.ts +13 -1
- package/dist/multi-viewport.js +152 -35
- package/dist/playwright-runner.d.ts +16 -0
- package/dist/playwright-runner.js +208 -0
- package/dist/security-audit.d.ts +9 -0
- package/dist/security-audit.js +64 -0
- package/dist/verification-core/report.js +137 -13
- package/dist/verification-core/types.d.ts +17 -2
- package/package.json +9 -1
package/README.md
CHANGED
|
@@ -5,17 +5,17 @@ CLI verification for frontend apps.
|
|
|
5
5
|
`laxy-verify` runs production build checks, Lighthouse, tiered verify E2E, and plan-gated verification features for Free, Pro, and Pro+ accounts.
|
|
6
6
|
It is designed around three user questions:
|
|
7
7
|
|
|
8
|
-
- Free: "
|
|
9
|
-
- Pro: "
|
|
10
|
-
- Pro+: "Can I
|
|
8
|
+
- Free: "Will it break in production?"
|
|
9
|
+
- Pro: "Would a client accept this?"
|
|
10
|
+
- Pro+: "Can I ship this today?"
|
|
11
11
|
|
|
12
12
|
```bash
|
|
13
13
|
npx laxy-verify --init --run
|
|
14
|
-
npx laxy-verify .
|
|
15
|
-
npx laxy-verify . --plan-override pro
|
|
16
|
-
npx laxy-verify login
|
|
17
|
-
npx laxy-verify whoami
|
|
18
|
-
npx laxy-verify --help
|
|
14
|
+
npx laxy-verify .
|
|
15
|
+
npx laxy-verify . --plan-override pro
|
|
16
|
+
npx laxy-verify login
|
|
17
|
+
npx laxy-verify whoami
|
|
18
|
+
npx laxy-verify --help
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
## Quick Start
|
|
@@ -43,9 +43,9 @@ Commit the generated workflow. Each PR gets a verification run, grade output, an
|
|
|
43
43
|
|
|
44
44
|
| Plan | Question it answers |
|
|
45
45
|
|------|---------------------|
|
|
46
|
-
| Free |
|
|
47
|
-
| Pro |
|
|
48
|
-
| Pro+ | Can I
|
|
46
|
+
| Free | Will it break in production? |
|
|
47
|
+
| Pro | Would a client accept this? |
|
|
48
|
+
| Pro+ | Can I ship this today? |
|
|
49
49
|
|
|
50
50
|
## Grades
|
|
51
51
|
|
|
@@ -66,16 +66,16 @@ npx laxy-verify whoami
|
|
|
66
66
|
npx laxy-verify logout
|
|
67
67
|
```
|
|
68
68
|
|
|
69
|
-
| Feature | Free | Pro | Pro+ |
|
|
70
|
-
|---------|------|-----|------|
|
|
71
|
-
| Build verification | Yes | Yes | Yes |
|
|
72
|
-
| Lighthouse | 1 run | 3 runs | 3 runs |
|
|
73
|
-
| Verify E2E | Smoke | Deeper client-send checks | Deeper client-send checks |
|
|
74
|
-
| Detailed report view | No | Yes | Yes |
|
|
75
|
-
| `laxy-verify-report.md` export | No | Yes | Yes |
|
|
76
|
-
| Multi-viewport verification | No | No | Yes |
|
|
77
|
-
| Visual diff | No | No | Yes |
|
|
78
|
-
| Failure analysis signals | No | No | Yes |
|
|
69
|
+
| Feature | Free | Pro | Pro+ |
|
|
70
|
+
|---------|------|-----|------|
|
|
71
|
+
| Build verification | Yes | Yes | Yes |
|
|
72
|
+
| Lighthouse | 1 run | 3 runs | 3 runs |
|
|
73
|
+
| Verify E2E | Smoke | Deeper client-send checks | Deeper client-send checks |
|
|
74
|
+
| Detailed report view | No | Yes | Yes |
|
|
75
|
+
| `laxy-verify-report.md` export | No | Yes | Yes |
|
|
76
|
+
| Multi-viewport verification | No | No | Yes |
|
|
77
|
+
| Visual diff | No | No | Yes |
|
|
78
|
+
| Failure analysis signals | No | No | Yes |
|
|
79
79
|
|
|
80
80
|
Pro is for delivery verification.
|
|
81
81
|
Pro+ is for release-confidence verification with extra evidence before you say "ship it."
|
|
@@ -119,42 +119,42 @@ Options:
|
|
|
119
119
|
--format console|json
|
|
120
120
|
--ci
|
|
121
121
|
--config <path>
|
|
122
|
-
--fail-on unverified|bronze|silver|gold
|
|
123
|
-
--skip-lighthouse
|
|
124
|
-
--plan-override free|pro|pro_plus
|
|
125
|
-
--badge
|
|
126
|
-
--init
|
|
127
|
-
--multi-viewport
|
|
128
|
-
--help
|
|
129
|
-
|
|
130
|
-
Subcommands:
|
|
131
|
-
login [email]
|
|
132
|
-
logout
|
|
133
|
-
whoami
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
`--plan-override` is for downgrade testing only.
|
|
137
|
-
Example: if your account is Pro+, you can run `--plan-override pro` or `--plan-override free` to verify the lower-tier behavior without changing your subscription.
|
|
138
|
-
It will reject upgrades above your real entitlement.
|
|
139
|
-
|
|
140
|
-
## Result Files
|
|
141
|
-
|
|
142
|
-
Each run writes `.laxy-result.json`.
|
|
143
|
-
|
|
144
|
-
Paid plans also write a readable markdown summary to `laxy-verify-report.md`.
|
|
145
|
-
|
|
146
|
-
- `Pro`: blocker-focused delivery report
|
|
147
|
-
- `Pro+`: release-readiness report with viewport and visual evidence
|
|
148
|
-
|
|
149
|
-
Exit behavior follows the verification verdict, not just the legacy grade.
|
|
150
|
-
|
|
151
|
-
- `build-failed` -> exit 1
|
|
152
|
-
- `hold` -> exit 1
|
|
153
|
-
- `Pro+ investigate` -> exit 1
|
|
154
|
-
- plain lower-tier pass states can still exit 0
|
|
155
|
-
|
|
156
|
-
```json
|
|
157
|
-
{
|
|
122
|
+
--fail-on unverified|bronze|silver|gold
|
|
123
|
+
--skip-lighthouse
|
|
124
|
+
--plan-override free|pro|pro_plus
|
|
125
|
+
--badge
|
|
126
|
+
--init
|
|
127
|
+
--multi-viewport
|
|
128
|
+
--help
|
|
129
|
+
|
|
130
|
+
Subcommands:
|
|
131
|
+
login [email]
|
|
132
|
+
logout
|
|
133
|
+
whoami
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
`--plan-override` is for downgrade testing only.
|
|
137
|
+
Example: if your account is Pro+, you can run `--plan-override pro` or `--plan-override free` to verify the lower-tier behavior without changing your subscription.
|
|
138
|
+
It will reject upgrades above your real entitlement.
|
|
139
|
+
|
|
140
|
+
## Result Files
|
|
141
|
+
|
|
142
|
+
Each run writes `.laxy-result.json`.
|
|
143
|
+
|
|
144
|
+
Paid plans also write a readable markdown summary to `laxy-verify-report.md`.
|
|
145
|
+
|
|
146
|
+
- `Pro`: blocker-focused delivery report
|
|
147
|
+
- `Pro+`: release-readiness report with viewport and visual evidence
|
|
148
|
+
|
|
149
|
+
Exit behavior follows the verification verdict, not just the legacy grade.
|
|
150
|
+
|
|
151
|
+
- `build-failed` -> exit 1
|
|
152
|
+
- `hold` -> exit 1
|
|
153
|
+
- `Pro+ investigate` -> exit 1
|
|
154
|
+
- plain lower-tier pass states can still exit 0
|
|
155
|
+
|
|
156
|
+
```json
|
|
157
|
+
{
|
|
158
158
|
"grade": "Gold",
|
|
159
159
|
"timestamp": "2026-04-09T09:00:00Z",
|
|
160
160
|
"build": { "success": true, "durationMs": 12000, "errors": [] },
|
|
@@ -174,26 +174,26 @@ Exit behavior follows the verification verdict, not just the legacy grade.
|
|
|
174
174
|
},
|
|
175
175
|
"exitCode": 0,
|
|
176
176
|
"_plan": "pro_plus"
|
|
177
|
-
}
|
|
178
|
-
```
|
|
179
|
-
|
|
180
|
-
### `laxy-verify-report.md`
|
|
181
|
-
|
|
182
|
-
For Pro and Pro+ runs, the markdown report is designed to be easy to read and easy to paste into an AI coding tool.
|
|
183
|
-
|
|
184
|
-
It includes:
|
|
185
|
-
|
|
186
|
-
- the main decision in plain English
|
|
187
|
-
- what passed
|
|
188
|
-
- blockers and warnings
|
|
189
|
-
- exact verification evidence
|
|
190
|
-
- failed E2E scenarios
|
|
191
|
-
- a `Copy For AI` section you can paste directly into Codex, Cursor, Claude, or ChatGPT
|
|
192
|
-
|
|
193
|
-
## Regression Fixtures
|
|
194
|
-
|
|
195
|
-
The repo also includes dedicated regression fixtures under `.qa-regression-fixtures/`.
|
|
196
|
-
They intentionally break build, navigation, coverage, performance, viewport behavior, and visual stability so the verifier can be tested against known failure modes.
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### `laxy-verify-report.md`
|
|
181
|
+
|
|
182
|
+
For Pro and Pro+ runs, the markdown report is designed to be easy to read and easy to paste into an AI coding tool.
|
|
183
|
+
|
|
184
|
+
It includes:
|
|
185
|
+
|
|
186
|
+
- the main decision in plain English
|
|
187
|
+
- what passed
|
|
188
|
+
- blockers and warnings
|
|
189
|
+
- exact verification evidence
|
|
190
|
+
- failed E2E scenarios
|
|
191
|
+
- a `Copy For AI` section you can paste directly into Codex, Cursor, Claude, or ChatGPT
|
|
192
|
+
|
|
193
|
+
## Regression Fixtures
|
|
194
|
+
|
|
195
|
+
The repo also includes dedicated regression fixtures under `.qa-regression-fixtures/`.
|
|
196
|
+
They intentionally break build, navigation, coverage, performance, viewport behavior, and visual stability so the verifier can be tested against known failure modes.
|
|
197
197
|
|
|
198
198
|
## Limitations
|
|
199
199
|
|
package/dist/cli.js
CHANGED
|
@@ -53,8 +53,10 @@ const auth_js_1 = require("./auth.js");
|
|
|
53
53
|
const entitlement_js_1 = require("./entitlement.js");
|
|
54
54
|
const multi_viewport_js_1 = require("./multi-viewport.js");
|
|
55
55
|
const e2e_js_1 = require("./e2e.js");
|
|
56
|
+
const playwright_runner_js_1 = require("./playwright-runner.js");
|
|
56
57
|
const report_markdown_js_1 = require("./report-markdown.js");
|
|
57
58
|
const visual_diff_js_1 = require("./visual-diff.js");
|
|
59
|
+
const security_audit_js_1 = require("./security-audit.js");
|
|
58
60
|
const index_js_1 = require("./verification-core/index.js");
|
|
59
61
|
const package_json_1 = __importDefault(require("../package.json"));
|
|
60
62
|
function shouldFailVerificationResult(report, failOn) {
|
|
@@ -126,6 +128,7 @@ function parseArgs() {
|
|
|
126
128
|
initRun: flags.init !== undefined && flags.run !== undefined,
|
|
127
129
|
multiViewport: flags["multi-viewport"] !== undefined,
|
|
128
130
|
failureAnalysis: flags["failure-analysis"] !== undefined,
|
|
131
|
+
crawl: flags.crawl !== undefined,
|
|
129
132
|
planOverride: flags["plan-override"],
|
|
130
133
|
help: flags.help !== undefined || flags.h !== undefined,
|
|
131
134
|
};
|
|
@@ -188,28 +191,73 @@ function consoleOutput(result) {
|
|
|
188
191
|
if (result.e2e) {
|
|
189
192
|
console.log(` E2E: ${result.e2e.passed}/${result.e2e.total} passed`);
|
|
190
193
|
}
|
|
194
|
+
if (result.crossBrowser && result.crossBrowser.length > 0) {
|
|
195
|
+
console.log(" Cross-browser:");
|
|
196
|
+
for (const cbr of result.crossBrowser) {
|
|
197
|
+
const status = cbr.failed === 0 ? "OK" : "FAIL";
|
|
198
|
+
console.log(` ${cbr.browser}: ${cbr.passed}/${cbr.results.length} passed ${status}`);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
191
201
|
if (result.visualDiff) {
|
|
192
202
|
console.log(` Visual diff: ${result.visualDiff.diffPercentage}% (${result.visualDiff.verdict})`);
|
|
193
203
|
}
|
|
204
|
+
if (result.security) {
|
|
205
|
+
console.log(` Security: ${result.security.summary}`);
|
|
206
|
+
}
|
|
207
|
+
if (result.mobileLighthouse) {
|
|
208
|
+
const ml = result.mobileLighthouse;
|
|
209
|
+
console.log(` Mobile LH: P=${ml.performance} A=${ml.accessibility} SEO=${ml.seo} BP=${ml.bestPractices}`);
|
|
210
|
+
}
|
|
194
211
|
if (result.verification) {
|
|
195
212
|
const view = result.verification.view;
|
|
213
|
+
const isPro = view.tier === "pro" || view.tier === "pro_plus";
|
|
214
|
+
const isProPlus = view.tier === "pro_plus";
|
|
215
|
+
// verbose_failure: Pro에서 서버가 활성화하면 blockers 전체 설명 표시
|
|
216
|
+
// failure_analysis: Pro+에서 서버가 활성화하면 warnings 전체 설명 + evidence 전체 표시
|
|
217
|
+
const verboseFailure = isPro && (result._verbose_failure ?? isPro);
|
|
218
|
+
const failureAnalysis = isProPlus && (result._failure_analysis ?? isProPlus);
|
|
196
219
|
console.log(` Verification tier: ${view.tier}`);
|
|
197
220
|
console.log(` Question: ${view.question}`);
|
|
198
221
|
console.log(` Verdict: ${view.verdict} (${view.confidence})`);
|
|
199
222
|
console.log(` Summary: ${view.summary}`);
|
|
223
|
+
// Pro/Pro+: 체크 통과 목록 요약
|
|
224
|
+
if (isPro && view.passes.length > 0) {
|
|
225
|
+
const passedChecks = view.passes.filter((p) => p.passed).map((p) => p.label);
|
|
226
|
+
const failedChecks = view.passes.filter((p) => !p.passed).map((p) => p.label);
|
|
227
|
+
if (passedChecks.length > 0)
|
|
228
|
+
console.log(` Passed: ${passedChecks.join(", ")}`);
|
|
229
|
+
if (failedChecks.length > 0)
|
|
230
|
+
console.log(` Failed: ${failedChecks.join(", ")}`);
|
|
231
|
+
}
|
|
232
|
+
// Blockers: 제목은 모든 티어, Fix 액션은 verbose_failure(Pro+) 이상에서 표시
|
|
200
233
|
if (view.blockers.length > 0) {
|
|
201
234
|
console.log(" Blockers:");
|
|
202
|
-
for (const blocker of view.blockers)
|
|
235
|
+
for (const blocker of view.blockers) {
|
|
203
236
|
console.log(` - ${blocker.title}`);
|
|
237
|
+
if (verboseFailure)
|
|
238
|
+
console.log(` Fix: ${blocker.action}`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
// Warnings: Pro/Pro+에서만 표시, Review 액션은 failure_analysis(Pro+)에서 표시
|
|
242
|
+
if (isPro && view.warnings.length > 0) {
|
|
243
|
+
console.log(" Warnings:");
|
|
244
|
+
for (const warning of view.warnings) {
|
|
245
|
+
console.log(` - ${warning.title}`);
|
|
246
|
+
if (failureAnalysis)
|
|
247
|
+
console.log(` Review: ${warning.action}`);
|
|
248
|
+
}
|
|
204
249
|
}
|
|
205
250
|
if (view.nextActions.length > 0) {
|
|
206
251
|
console.log(" Next actions:");
|
|
207
252
|
for (const action of view.nextActions)
|
|
208
253
|
console.log(` - ${action}`);
|
|
209
254
|
}
|
|
210
|
-
|
|
255
|
+
// Evidence: failure_analysis(Pro+)는 전체, verbose_failure(Pro)는 3개, Free는 2개
|
|
256
|
+
const evidenceLimit = failureAnalysis ? view.failureEvidence.length : verboseFailure ? 3 : 2;
|
|
257
|
+
const evidenceToShow = view.failureEvidence.slice(0, evidenceLimit);
|
|
258
|
+
if (evidenceToShow.length > 0) {
|
|
211
259
|
console.log(" Evidence:");
|
|
212
|
-
for (const item of
|
|
260
|
+
for (const item of evidenceToShow)
|
|
213
261
|
console.log(` - ${item}`);
|
|
214
262
|
}
|
|
215
263
|
}
|
|
@@ -251,10 +299,11 @@ async function run() {
|
|
|
251
299
|
--format console | json (default: console)
|
|
252
300
|
--ci CI mode: -10 Performance threshold, runs=3
|
|
253
301
|
--config <path> Path to .laxy.yml
|
|
254
|
-
--fail-on unverified | bronze | silver | gold
|
|
255
|
-
--skip-lighthouse Skip Lighthouse but still run build and E2E
|
|
256
|
-
--plan-override free | pro | pro_plus (downgrade testing only)
|
|
257
|
-
--multi-viewport Pro+: Lighthouse on desktop/tablet/mobile
|
|
302
|
+
--fail-on unverified | bronze | silver | gold
|
|
303
|
+
--skip-lighthouse Skip Lighthouse but still run build and E2E
|
|
304
|
+
--plan-override free | pro | pro_plus (downgrade testing only)
|
|
305
|
+
--multi-viewport Pro+: Lighthouse on desktop/tablet/mobile
|
|
306
|
+
--crawl Crawl the app to discover routes before E2E
|
|
258
307
|
--badge Print shields.io badge markdown
|
|
259
308
|
--help Show this help
|
|
260
309
|
|
|
@@ -403,8 +452,14 @@ async function run() {
|
|
|
403
452
|
let multiViewportScores = null;
|
|
404
453
|
let allViewportsOk = false;
|
|
405
454
|
let e2eResult;
|
|
455
|
+
let crossBrowserResults;
|
|
406
456
|
let e2eCoverageGaps = [];
|
|
457
|
+
let e2eConsoleErrors = [];
|
|
458
|
+
let e2eStabilityPassed = true;
|
|
407
459
|
let visualDiffResult = null;
|
|
460
|
+
let securityAuditResult = null;
|
|
461
|
+
let mobileLighthouseScores = null;
|
|
462
|
+
const failureEvidence = [];
|
|
408
463
|
if (buildResult.success) {
|
|
409
464
|
let servePid;
|
|
410
465
|
try {
|
|
@@ -439,13 +494,57 @@ async function run() {
|
|
|
439
494
|
multiViewportScores = await (0, multi_viewport_js_1.runMultiViewportLighthouse)(port);
|
|
440
495
|
(0, multi_viewport_js_1.printMultiViewportResults)(multiViewportScores, adjustedThresholds);
|
|
441
496
|
allViewportsOk = (0, multi_viewport_js_1.allViewportsPass)(multiViewportScores, adjustedThresholds);
|
|
497
|
+
// Surface screenshot diff issues in failure evidence
|
|
498
|
+
if (multiViewportScores.screenshotDiffs) {
|
|
499
|
+
for (const diff of multiViewportScores.screenshotDiffs) {
|
|
500
|
+
if (!diff.baselineCreated && diff.diffPercent > 10) {
|
|
501
|
+
failureEvidence.push(`Viewport screenshot: ${diff.viewport} diff ${diff.diffPercent}% exceeds 10% threshold`);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
442
505
|
}
|
|
443
506
|
catch (mvErr) {
|
|
444
507
|
console.error(`Multi-viewport error: ${mvErr instanceof Error ? mvErr.message : String(mvErr)}`);
|
|
445
508
|
}
|
|
446
509
|
}
|
|
510
|
+
// Pro: single mobile Lighthouse check (if not using full multi-viewport)
|
|
511
|
+
if (!args.skipLighthouse &&
|
|
512
|
+
verificationTier === "pro" &&
|
|
513
|
+
!effectiveFeatures.multi_viewport) {
|
|
514
|
+
try {
|
|
515
|
+
mobileLighthouseScores = await (0, multi_viewport_js_1.runMobileLighthouse)(port);
|
|
516
|
+
if (mobileLighthouseScores) {
|
|
517
|
+
const mobilePassed = mobileLighthouseScores.performance >= adjustedThresholds.performance &&
|
|
518
|
+
mobileLighthouseScores.accessibility >= adjustedThresholds.accessibility;
|
|
519
|
+
if (!mobilePassed) {
|
|
520
|
+
failureEvidence.push(`Mobile LH: P=${mobileLighthouseScores.performance} A=${mobileLighthouseScores.accessibility}`);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
catch (mobileLhErr) {
|
|
525
|
+
console.error(`Mobile Lighthouse error: ${mobileLhErr instanceof Error ? mobileLhErr.message : String(mobileLhErr)}`);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
// Pro/Pro+: security audit
|
|
529
|
+
if (verificationTier !== "free") {
|
|
530
|
+
try {
|
|
531
|
+
securityAuditResult = await (0, security_audit_js_1.runSecurityAudit)(args.projectDir);
|
|
532
|
+
if (securityAuditResult.critical > 0 || securityAuditResult.high > 0) {
|
|
533
|
+
failureEvidence.push(`Security: ${securityAuditResult.summary}`);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
catch (secErr) {
|
|
537
|
+
console.error(`Security audit error: ${secErr instanceof Error ? secErr.message : String(secErr)}`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
const crawlEnabled = args.crawl || config.crawl;
|
|
541
|
+
const crawlOpts = crawlEnabled
|
|
542
|
+
? { enabled: true, maxDepth: config.max_crawl_depth, maxPages: config.max_crawl_pages }
|
|
543
|
+
: undefined;
|
|
544
|
+
let lastE2EScenarios;
|
|
447
545
|
try {
|
|
448
|
-
const e2eRuns = await (0, e2e_js_1.runVerifyE2E)(verifyUrl, verificationTier);
|
|
546
|
+
const e2eRuns = await (0, e2e_js_1.runVerifyE2E)(verifyUrl, verificationTier, config.scenarios, crawlOpts);
|
|
547
|
+
lastE2EScenarios = e2eRuns.scenarios;
|
|
449
548
|
e2eResult = {
|
|
450
549
|
passed: e2eRuns.passed,
|
|
451
550
|
failed: e2eRuns.failed,
|
|
@@ -453,13 +552,57 @@ async function run() {
|
|
|
453
552
|
results: e2eRuns.results,
|
|
454
553
|
};
|
|
455
554
|
e2eCoverageGaps = e2eRuns.coverageGaps;
|
|
555
|
+
e2eConsoleErrors = e2eRuns.consoleErrors;
|
|
556
|
+
// Pro+ stability: run E2E a second time if first run passed all
|
|
557
|
+
if (verificationTier === "pro_plus" && e2eRuns.passed === e2eRuns.results.length && e2eRuns.results.length > 0) {
|
|
558
|
+
console.log(" [Pro+] Running stability pass (run 2/2)...");
|
|
559
|
+
const e2eRuns2 = await (0, e2e_js_1.runVerifyE2E)(verifyUrl, verificationTier, config.scenarios, crawlOpts);
|
|
560
|
+
if (e2eRuns2.passed < e2eRuns2.results.length) {
|
|
561
|
+
e2eStabilityPassed = false;
|
|
562
|
+
e2eCoverageGaps.push("Stability check failed on second run");
|
|
563
|
+
const failedNames = e2eRuns2.results
|
|
564
|
+
.filter((r) => !r.passed)
|
|
565
|
+
.map((r) => r.name)
|
|
566
|
+
.join(", ");
|
|
567
|
+
failureEvidence.push(`E2E stability: second run failed (${failedNames})`);
|
|
568
|
+
}
|
|
569
|
+
else {
|
|
570
|
+
console.log(" [Pro+] Stability pass: OK (2/2 runs passed)");
|
|
571
|
+
}
|
|
572
|
+
}
|
|
456
573
|
if (e2eRuns.coverageGaps.length > 0) {
|
|
457
574
|
console.error(`E2E coverage warning: ${e2eRuns.coverageGaps.join(" ")}`);
|
|
458
575
|
}
|
|
576
|
+
if (e2eRuns.consoleErrors.length > 0) {
|
|
577
|
+
console.error(`E2E console errors: ${e2eRuns.consoleErrors.length} detected`);
|
|
578
|
+
}
|
|
459
579
|
}
|
|
460
580
|
catch (e2eErr) {
|
|
461
581
|
console.error(`E2E error: ${e2eErr instanceof Error ? e2eErr.message : String(e2eErr)}`);
|
|
462
582
|
}
|
|
583
|
+
// Cross-browser E2E via Playwright (if non-chromium browsers configured)
|
|
584
|
+
const extraBrowsers = (config.browsers || []).filter((b) => b !== "chromium" && ["firefox", "webkit"].includes(b));
|
|
585
|
+
if (extraBrowsers.length > 0 && lastE2EScenarios && lastE2EScenarios.length > 0) {
|
|
586
|
+
const pwAvailable = await (0, playwright_runner_js_1.isPlaywrightAvailable)();
|
|
587
|
+
if (pwAvailable) {
|
|
588
|
+
try {
|
|
589
|
+
crossBrowserResults = await (0, playwright_runner_js_1.runPlaywrightE2E)(verifyUrl, lastE2EScenarios, extraBrowsers);
|
|
590
|
+
for (const cbr of crossBrowserResults) {
|
|
591
|
+
console.log(` Cross-browser ${cbr.browser}: ${cbr.passed}/${cbr.results.length} passed`);
|
|
592
|
+
if (cbr.failed > 0) {
|
|
593
|
+
const failedNames = cbr.results.filter(r => !r.passed).map(r => r.name).join(", ");
|
|
594
|
+
failureEvidence.push(`Cross-browser ${cbr.browser}: ${failedNames} failed`);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
catch (cbErr) {
|
|
599
|
+
console.error(`Cross-browser error: ${cbErr instanceof Error ? cbErr.message : String(cbErr)}`);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
else {
|
|
603
|
+
console.log(" Note: Cross-browser testing requires playwright. Run: npm install -D playwright && npx playwright install");
|
|
604
|
+
}
|
|
605
|
+
}
|
|
463
606
|
if (verificationTier === "pro_plus") {
|
|
464
607
|
try {
|
|
465
608
|
visualDiffResult = await (0, visual_diff_js_1.runVisualDiff)(args.projectDir, verifyUrl, "verify");
|
|
@@ -480,31 +623,26 @@ async function run() {
|
|
|
480
623
|
}
|
|
481
624
|
const verificationTier = (0, index_js_1.planToVerificationTier)(effectiveFeatures.plan);
|
|
482
625
|
const viewportSummary = summarizeViewportIssues(multiViewportScores, adjustedThresholds);
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
?
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
? [
|
|
496
|
-
visualDiffResult.hasBaseline
|
|
497
|
-
? `Visual diff: ${visualDiffResult.diffPercentage}% (${visualDiffResult.verdict})`
|
|
498
|
-
: "Visual diff: baseline seeded",
|
|
499
|
-
]
|
|
500
|
-
: []),
|
|
501
|
-
];
|
|
626
|
+
failureEvidence.push(...buildResult.errors.slice(0, 3).map((error) => `Build: ${error}`), ...(lighthouseErrorCount > 0 ? [`Lighthouse: ${lighthouseErrorCount} run error(s) were recorded during collection.`] : []), ...(e2eResult
|
|
627
|
+
? e2eResult.results
|
|
628
|
+
.filter((scenario) => !scenario.passed)
|
|
629
|
+
.slice(0, 2)
|
|
630
|
+
.map((scenario) => `E2E: ${scenario.name}${scenario.error ? ` - ${scenario.error}` : ""}`)
|
|
631
|
+
: []), ...e2eConsoleErrors.slice(0, 2).map((e) => `Console: ${e}`), ...e2eCoverageGaps.slice(0, 2).map((gap) => `E2E coverage: ${gap}`), ...(viewportSummary.count > 0 && viewportSummary.summary ? [`Viewport: ${viewportSummary.summary}`] : []), ...(visualDiffResult
|
|
632
|
+
? [
|
|
633
|
+
visualDiffResult.hasBaseline
|
|
634
|
+
? `Visual diff: ${visualDiffResult.diffPercentage}% (${visualDiffResult.verdict})`
|
|
635
|
+
: "Visual diff: baseline seeded",
|
|
636
|
+
]
|
|
637
|
+
: []));
|
|
502
638
|
const verificationReport = (0, index_js_1.buildVerificationReport)({
|
|
503
639
|
buildSuccess: buildResult.success,
|
|
504
640
|
buildErrors: buildResult.errors,
|
|
505
641
|
e2ePassed: e2eResult?.passed,
|
|
506
642
|
e2eTotal: e2eResult?.total,
|
|
507
643
|
e2eCoverageGaps,
|
|
644
|
+
e2eConsoleErrorCount: e2eConsoleErrors.length,
|
|
645
|
+
e2eStabilityPassed,
|
|
508
646
|
lighthouseSkipped: args.skipLighthouse,
|
|
509
647
|
lighthouseErrorCount,
|
|
510
648
|
viewportIssues: multiViewportScores ? viewportSummary.count : undefined,
|
|
@@ -514,6 +652,15 @@ async function run() {
|
|
|
514
652
|
visualDiffPercentage: visualDiffResult?.diffPercentage,
|
|
515
653
|
hasVisualBaseline: visualDiffResult?.hasBaseline,
|
|
516
654
|
lighthouseScores: scores,
|
|
655
|
+
mobileLighthouseScores: mobileLighthouseScores ?? undefined,
|
|
656
|
+
securityAudit: securityAuditResult
|
|
657
|
+
? {
|
|
658
|
+
totalVulnerabilities: securityAuditResult.totalVulnerabilities,
|
|
659
|
+
critical: securityAuditResult.critical,
|
|
660
|
+
high: securityAuditResult.high,
|
|
661
|
+
summary: securityAuditResult.summary,
|
|
662
|
+
}
|
|
663
|
+
: undefined,
|
|
517
664
|
failureEvidence,
|
|
518
665
|
}, {
|
|
519
666
|
tier: verificationTier,
|
|
@@ -531,7 +678,10 @@ async function run() {
|
|
|
531
678
|
errors: buildResult.errors,
|
|
532
679
|
},
|
|
533
680
|
e2e: e2eResult,
|
|
681
|
+
crossBrowser: crossBrowserResults,
|
|
534
682
|
lighthouse: lighthouseResult,
|
|
683
|
+
mobileLighthouse: mobileLighthouseScores,
|
|
684
|
+
security: securityAuditResult,
|
|
535
685
|
visualDiff: visualDiffResult,
|
|
536
686
|
thresholds: adjustedThresholds,
|
|
537
687
|
ciMode: config.ciMode,
|
|
@@ -539,6 +689,8 @@ async function run() {
|
|
|
539
689
|
exitCode,
|
|
540
690
|
config_fail_on: config.fail_on,
|
|
541
691
|
_plan: effectiveFeatures.plan,
|
|
692
|
+
_verbose_failure: effectiveFeatures.verbose_failure,
|
|
693
|
+
_failure_analysis: effectiveFeatures.failure_analysis,
|
|
542
694
|
verification: {
|
|
543
695
|
tier: verificationTier,
|
|
544
696
|
report: verificationReport,
|
package/dist/config.d.ts
CHANGED
|
@@ -5,6 +5,19 @@ export interface Thresholds {
|
|
|
5
5
|
seo: number;
|
|
6
6
|
bestPractices: number;
|
|
7
7
|
}
|
|
8
|
+
export interface UserScenarioStep {
|
|
9
|
+
goto?: string;
|
|
10
|
+
fill?: string;
|
|
11
|
+
with?: string;
|
|
12
|
+
click?: string;
|
|
13
|
+
expect_visible?: string;
|
|
14
|
+
expect_text?: string;
|
|
15
|
+
wait?: number;
|
|
16
|
+
}
|
|
17
|
+
export interface UserScenario {
|
|
18
|
+
name: string;
|
|
19
|
+
steps: UserScenarioStep[];
|
|
20
|
+
}
|
|
8
21
|
export interface LaxyConfig {
|
|
9
22
|
framework: string;
|
|
10
23
|
build_command: string;
|
|
@@ -16,6 +29,11 @@ export interface LaxyConfig {
|
|
|
16
29
|
lighthouse_runs: number;
|
|
17
30
|
thresholds: Thresholds;
|
|
18
31
|
fail_on: FailOn;
|
|
32
|
+
scenarios?: UserScenario[];
|
|
33
|
+
crawl: boolean;
|
|
34
|
+
max_crawl_depth: number;
|
|
35
|
+
max_crawl_pages: number;
|
|
36
|
+
browsers: string[];
|
|
19
37
|
}
|
|
20
38
|
export declare class ConfigParseError extends Error {
|
|
21
39
|
constructor(msg: string);
|
package/dist/config.js
CHANGED
|
@@ -54,6 +54,10 @@ const DEFAULT_CONFIG = {
|
|
|
54
54
|
bestPractices: 80,
|
|
55
55
|
},
|
|
56
56
|
fail_on: "bronze",
|
|
57
|
+
crawl: false,
|
|
58
|
+
max_crawl_depth: 3,
|
|
59
|
+
max_crawl_pages: 10,
|
|
60
|
+
browsers: ["chromium"],
|
|
57
61
|
};
|
|
58
62
|
const VALID_FAIL_ON = ["unverified", "bronze", "silver", "gold"];
|
|
59
63
|
class ConfigParseError extends Error {
|
|
@@ -86,6 +90,19 @@ function parseYaml(filePath) {
|
|
|
86
90
|
result.dev_timeout = raw.dev_timeout;
|
|
87
91
|
if (typeof raw.lighthouse_runs === "number")
|
|
88
92
|
result.lighthouse_runs = raw.lighthouse_runs;
|
|
93
|
+
if (typeof raw.crawl === "boolean")
|
|
94
|
+
result.crawl = raw.crawl;
|
|
95
|
+
if (typeof raw.max_crawl_depth === "number")
|
|
96
|
+
result.max_crawl_depth = raw.max_crawl_depth;
|
|
97
|
+
if (typeof raw.max_crawl_pages === "number")
|
|
98
|
+
result.max_crawl_pages = raw.max_crawl_pages;
|
|
99
|
+
if (Array.isArray(raw.browsers)) {
|
|
100
|
+
const validBrowsers = ["chromium", "firefox", "webkit"];
|
|
101
|
+
const browsers = raw.browsers
|
|
102
|
+
.filter((b) => typeof b === "string" && validBrowsers.includes(b));
|
|
103
|
+
if (browsers.length > 0)
|
|
104
|
+
result.browsers = browsers;
|
|
105
|
+
}
|
|
89
106
|
if (typeof raw.fail_on === "string") {
|
|
90
107
|
const f = raw.fail_on;
|
|
91
108
|
if (!VALID_FAIL_ON.includes(f)) {
|
|
@@ -108,6 +125,43 @@ function parseYaml(filePath) {
|
|
|
108
125
|
thr.bestPractices = t.best_practices;
|
|
109
126
|
result.thresholds = thr;
|
|
110
127
|
}
|
|
128
|
+
if (Array.isArray(raw.scenarios)) {
|
|
129
|
+
const scenarios = [];
|
|
130
|
+
for (const item of raw.scenarios) {
|
|
131
|
+
if (typeof item === "object" &&
|
|
132
|
+
item !== null &&
|
|
133
|
+
typeof item.name === "string" &&
|
|
134
|
+
Array.isArray(item.steps)) {
|
|
135
|
+
const rawScenario = item;
|
|
136
|
+
const steps = [];
|
|
137
|
+
for (const rawStep of rawScenario.steps) {
|
|
138
|
+
if (typeof rawStep === "object" && rawStep !== null) {
|
|
139
|
+
const s = rawStep;
|
|
140
|
+
const step = {};
|
|
141
|
+
if (typeof s.goto === "string")
|
|
142
|
+
step.goto = s.goto;
|
|
143
|
+
if (typeof s.fill === "string")
|
|
144
|
+
step.fill = s.fill;
|
|
145
|
+
if (typeof s.with === "string")
|
|
146
|
+
step.with = s.with;
|
|
147
|
+
if (typeof s.click === "string")
|
|
148
|
+
step.click = s.click;
|
|
149
|
+
if (typeof s.expect_visible === "string")
|
|
150
|
+
step.expect_visible = s.expect_visible;
|
|
151
|
+
if (typeof s.expect_text === "string")
|
|
152
|
+
step.expect_text = s.expect_text;
|
|
153
|
+
if (typeof s.wait === "number")
|
|
154
|
+
step.wait = s.wait;
|
|
155
|
+
steps.push(step);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
scenarios.push({ name: String(rawScenario.name), steps });
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (scenarios.length > 0) {
|
|
162
|
+
result.scenarios = scenarios;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
111
165
|
return result;
|
|
112
166
|
}
|
|
113
167
|
function loadConfig(options) {
|
|
@@ -127,6 +181,11 @@ function loadConfig(options) {
|
|
|
127
181
|
dev_timeout: base.dev_timeout ?? DEFAULT_CONFIG.dev_timeout,
|
|
128
182
|
lighthouse_runs: base.lighthouse_runs ?? DEFAULT_CONFIG.lighthouse_runs,
|
|
129
183
|
fail_on: base.fail_on ?? DEFAULT_CONFIG.fail_on,
|
|
184
|
+
scenarios: base.scenarios,
|
|
185
|
+
crawl: base.crawl ?? DEFAULT_CONFIG.crawl,
|
|
186
|
+
max_crawl_depth: base.max_crawl_depth ?? DEFAULT_CONFIG.max_crawl_depth,
|
|
187
|
+
max_crawl_pages: base.max_crawl_pages ?? DEFAULT_CONFIG.max_crawl_pages,
|
|
188
|
+
browsers: base.browsers ?? DEFAULT_CONFIG.browsers,
|
|
130
189
|
};
|
|
131
190
|
config.thresholds = { ...DEFAULT_CONFIG.thresholds, ...(base.thresholds ?? {}) };
|
|
132
191
|
// CLI flag overrides
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { E2EScenario } from "./e2e.js";
|
|
2
|
+
import type { VerificationTier } from "./verification-core/types.js";
|
|
3
|
+
export interface CrawlPage {
|
|
4
|
+
url: string;
|
|
5
|
+
path: string;
|
|
6
|
+
title: string;
|
|
7
|
+
forms: CrawlForm[];
|
|
8
|
+
buttons: string[];
|
|
9
|
+
internalLinks: string[];
|
|
10
|
+
hasConsoleErrors: boolean;
|
|
11
|
+
}
|
|
12
|
+
export interface CrawlForm {
|
|
13
|
+
selector: string;
|
|
14
|
+
inputs: {
|
|
15
|
+
selector: string;
|
|
16
|
+
type: string;
|
|
17
|
+
placeholder?: string;
|
|
18
|
+
}[];
|
|
19
|
+
submitSelector?: string;
|
|
20
|
+
}
|
|
21
|
+
export interface CrawlResult {
|
|
22
|
+
pages: CrawlPage[];
|
|
23
|
+
totalLinks: number;
|
|
24
|
+
crawledCount: number;
|
|
25
|
+
}
|
|
26
|
+
export interface CrawlOptions {
|
|
27
|
+
maxDepth?: number;
|
|
28
|
+
maxPages?: number;
|
|
29
|
+
timeout?: number;
|
|
30
|
+
}
|
|
31
|
+
export declare function crawlApp(baseUrl: string, options?: CrawlOptions): Promise<CrawlResult>;
|
|
32
|
+
/**
|
|
33
|
+
* Generate E2E scenarios from crawl results.
|
|
34
|
+
*/
|
|
35
|
+
export declare function buildScenariosFromCrawl(crawlResult: CrawlResult, tier: VerificationTier): E2EScenario[];
|