laxy-verify 1.1.15 → 1.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -36
- package/dist/cli.js +214 -32
- package/dist/config.d.ts +18 -0
- package/dist/config.js +59 -0
- package/dist/crawler.d.ts +35 -0
- package/dist/crawler.js +357 -0
- package/dist/e2e.d.ts +14 -2
- package/dist/e2e.js +164 -5
- package/dist/entitlement.d.ts +3 -0
- package/dist/entitlement.js +53 -0
- package/dist/lighthouse.js +44 -26
- package/dist/multi-viewport.d.ts +13 -1
- package/dist/multi-viewport.js +152 -35
- package/dist/playwright-runner.d.ts +16 -0
- package/dist/playwright-runner.js +208 -0
- package/dist/report-markdown.js +142 -17
- package/dist/security-audit.d.ts +9 -0
- package/dist/security-audit.js +64 -0
- package/dist/serve.js +21 -10
- package/dist/verification-core/report.js +168 -16
- package/dist/verification-core/types.d.ts +20 -2
- package/package.json +9 -1
package/README.md
CHANGED
|
@@ -12,6 +12,7 @@ It is designed around three user questions:
|
|
|
12
12
|
```bash
|
|
13
13
|
npx laxy-verify --init --run
|
|
14
14
|
npx laxy-verify .
|
|
15
|
+
npx laxy-verify . --plan-override pro
|
|
15
16
|
npx laxy-verify login
|
|
16
17
|
npx laxy-verify whoami
|
|
17
18
|
npx laxy-verify --help
|
|
@@ -65,16 +66,16 @@ npx laxy-verify whoami
|
|
|
65
66
|
npx laxy-verify logout
|
|
66
67
|
```
|
|
67
68
|
|
|
68
|
-
| Feature | Free | Pro | Pro+ |
|
|
69
|
-
|---------|------|-----|------|
|
|
70
|
-
| Build verification | Yes | Yes | Yes |
|
|
71
|
-
| Lighthouse | 1 run | 3 runs | 3 runs |
|
|
72
|
-
| Verify E2E | Smoke | Deeper client-send checks | Deeper client-send checks |
|
|
73
|
-
| Detailed report view | No | Yes | Yes |
|
|
74
|
-
| `laxy-verify-report.md` export | No | Yes | Yes |
|
|
75
|
-
| Multi-viewport verification | No | No | Yes |
|
|
76
|
-
| Visual diff | No | No | Yes |
|
|
77
|
-
| Failure analysis signals | No | No | Yes |
|
|
69
|
+
| Feature | Free | Pro | Pro+ |
|
|
70
|
+
|---------|------|-----|------|
|
|
71
|
+
| Build verification | Yes | Yes | Yes |
|
|
72
|
+
| Lighthouse | 1 run | 3 runs | 3 runs |
|
|
73
|
+
| Verify E2E | Smoke | Deeper client-send checks | Deeper client-send checks |
|
|
74
|
+
| Detailed report view | No | Yes | Yes |
|
|
75
|
+
| `laxy-verify-report.md` export | No | Yes | Yes |
|
|
76
|
+
| Multi-viewport verification | No | No | Yes |
|
|
77
|
+
| Visual diff | No | No | Yes |
|
|
78
|
+
| Failure analysis signals | No | No | Yes |
|
|
78
79
|
|
|
79
80
|
Pro is for delivery verification.
|
|
80
81
|
Pro+ is for release-confidence verification with extra evidence before you say "ship it."
|
|
@@ -120,6 +121,7 @@ Options:
|
|
|
120
121
|
--config <path>
|
|
121
122
|
--fail-on unverified|bronze|silver|gold
|
|
122
123
|
--skip-lighthouse
|
|
124
|
+
--plan-override free|pro|pro_plus
|
|
123
125
|
--badge
|
|
124
126
|
--init
|
|
125
127
|
--multi-viewport
|
|
@@ -131,17 +133,28 @@ Subcommands:
|
|
|
131
133
|
whoami
|
|
132
134
|
```
|
|
133
135
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
136
|
+
`--plan-override` is for downgrade testing only.
|
|
137
|
+
Example: if your account is Pro+, you can run `--plan-override pro` or `--plan-override free` to verify the lower-tier behavior without changing your subscription.
|
|
138
|
+
It will reject upgrades above your real entitlement.
|
|
139
|
+
|
|
140
|
+
## Result Files
|
|
141
|
+
|
|
142
|
+
Each run writes `.laxy-result.json`.
|
|
143
|
+
|
|
144
|
+
Paid plans also write a readable markdown summary to `laxy-verify-report.md`.
|
|
145
|
+
|
|
146
|
+
- `Pro`: blocker-focused delivery report
|
|
147
|
+
- `Pro+`: release-readiness report with viewport and visual evidence
|
|
148
|
+
|
|
149
|
+
Exit behavior follows the verification verdict, not just the legacy grade.
|
|
150
|
+
|
|
151
|
+
- `build-failed` -> exit 1
|
|
152
|
+
- `hold` -> exit 1
|
|
153
|
+
- `Pro+ investigate` -> exit 1
|
|
154
|
+
- plain lower-tier pass states can still exit 0
|
|
155
|
+
|
|
156
|
+
```json
|
|
157
|
+
{
|
|
145
158
|
"grade": "Gold",
|
|
146
159
|
"timestamp": "2026-04-09T09:00:00Z",
|
|
147
160
|
"build": { "success": true, "durationMs": 12000, "errors": [] },
|
|
@@ -161,21 +174,26 @@ Paid plans also write a readable markdown summary to `laxy-verify-report.md`.
|
|
|
161
174
|
},
|
|
162
175
|
"exitCode": 0,
|
|
163
176
|
"_plan": "pro_plus"
|
|
164
|
-
}
|
|
165
|
-
```
|
|
166
|
-
|
|
167
|
-
### `laxy-verify-report.md`
|
|
168
|
-
|
|
169
|
-
For Pro and Pro+ runs, the markdown report is designed to be easy to read and easy to paste into an AI coding tool.
|
|
170
|
-
|
|
171
|
-
It includes:
|
|
172
|
-
|
|
173
|
-
- the main decision in plain English
|
|
174
|
-
- what passed
|
|
175
|
-
- blockers and warnings
|
|
176
|
-
- exact verification evidence
|
|
177
|
-
- failed E2E scenarios
|
|
178
|
-
- a `Copy For AI` section you can paste directly into Codex, Cursor, Claude, or ChatGPT
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
### `laxy-verify-report.md`
|
|
181
|
+
|
|
182
|
+
For Pro and Pro+ runs, the markdown report is designed to be easy to read and easy to paste into an AI coding tool.
|
|
183
|
+
|
|
184
|
+
It includes:
|
|
185
|
+
|
|
186
|
+
- the main decision in plain English
|
|
187
|
+
- what passed
|
|
188
|
+
- blockers and warnings
|
|
189
|
+
- exact verification evidence
|
|
190
|
+
- failed E2E scenarios
|
|
191
|
+
- a `Copy For AI` section you can paste directly into Codex, Cursor, Claude, or ChatGPT
|
|
192
|
+
|
|
193
|
+
## Regression Fixtures
|
|
194
|
+
|
|
195
|
+
The repo also includes dedicated regression fixtures under `.qa-regression-fixtures/`.
|
|
196
|
+
They intentionally break build, navigation, coverage, performance, viewport behavior, and visual stability so the verifier can be tested against known failure modes.
|
|
179
197
|
|
|
180
198
|
## Limitations
|
|
181
199
|
|
package/dist/cli.js
CHANGED
|
@@ -53,10 +53,21 @@ const auth_js_1 = require("./auth.js");
|
|
|
53
53
|
const entitlement_js_1 = require("./entitlement.js");
|
|
54
54
|
const multi_viewport_js_1 = require("./multi-viewport.js");
|
|
55
55
|
const e2e_js_1 = require("./e2e.js");
|
|
56
|
+
const playwright_runner_js_1 = require("./playwright-runner.js");
|
|
56
57
|
const report_markdown_js_1 = require("./report-markdown.js");
|
|
57
58
|
const visual_diff_js_1 = require("./visual-diff.js");
|
|
59
|
+
const security_audit_js_1 = require("./security-audit.js");
|
|
58
60
|
const index_js_1 = require("./verification-core/index.js");
|
|
59
61
|
const package_json_1 = __importDefault(require("../package.json"));
|
|
62
|
+
function shouldFailVerificationResult(report, failOn) {
|
|
63
|
+
if (failOn === "unverified")
|
|
64
|
+
return false;
|
|
65
|
+
if (report.verdict === "build-failed" || report.verdict === "hold")
|
|
66
|
+
return true;
|
|
67
|
+
if (report.tier === "pro_plus" && report.verdict === "investigate")
|
|
68
|
+
return true;
|
|
69
|
+
return (0, grade_js_1.isWorseOrEqual)(report.grade, failOn);
|
|
70
|
+
}
|
|
60
71
|
function exitGracefully(code) {
|
|
61
72
|
if (process.platform === "win32") {
|
|
62
73
|
setTimeout(() => process.exit(code), 100);
|
|
@@ -117,6 +128,8 @@ function parseArgs() {
|
|
|
117
128
|
initRun: flags.init !== undefined && flags.run !== undefined,
|
|
118
129
|
multiViewport: flags["multi-viewport"] !== undefined,
|
|
119
130
|
failureAnalysis: flags["failure-analysis"] !== undefined,
|
|
131
|
+
crawl: flags.crawl !== undefined,
|
|
132
|
+
planOverride: flags["plan-override"],
|
|
120
133
|
help: flags.help !== undefined || flags.h !== undefined,
|
|
121
134
|
};
|
|
122
135
|
}
|
|
@@ -178,28 +191,73 @@ function consoleOutput(result) {
|
|
|
178
191
|
if (result.e2e) {
|
|
179
192
|
console.log(` E2E: ${result.e2e.passed}/${result.e2e.total} passed`);
|
|
180
193
|
}
|
|
194
|
+
if (result.crossBrowser && result.crossBrowser.length > 0) {
|
|
195
|
+
console.log(" Cross-browser:");
|
|
196
|
+
for (const cbr of result.crossBrowser) {
|
|
197
|
+
const status = cbr.failed === 0 ? "OK" : "FAIL";
|
|
198
|
+
console.log(` ${cbr.browser}: ${cbr.passed}/${cbr.results.length} passed ${status}`);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
181
201
|
if (result.visualDiff) {
|
|
182
202
|
console.log(` Visual diff: ${result.visualDiff.diffPercentage}% (${result.visualDiff.verdict})`);
|
|
183
203
|
}
|
|
204
|
+
if (result.security) {
|
|
205
|
+
console.log(` Security: ${result.security.summary}`);
|
|
206
|
+
}
|
|
207
|
+
if (result.mobileLighthouse) {
|
|
208
|
+
const ml = result.mobileLighthouse;
|
|
209
|
+
console.log(` Mobile LH: P=${ml.performance} A=${ml.accessibility} SEO=${ml.seo} BP=${ml.bestPractices}`);
|
|
210
|
+
}
|
|
184
211
|
if (result.verification) {
|
|
185
212
|
const view = result.verification.view;
|
|
213
|
+
const isPro = view.tier === "pro" || view.tier === "pro_plus";
|
|
214
|
+
const isProPlus = view.tier === "pro_plus";
|
|
215
|
+
// verbose_failure: Pro에서 서버가 활성화하면 blockers 전체 설명 표시
|
|
216
|
+
// failure_analysis: Pro+에서 서버가 활성화하면 warnings 전체 설명 + evidence 전체 표시
|
|
217
|
+
const verboseFailure = isPro && (result._verbose_failure ?? isPro);
|
|
218
|
+
const failureAnalysis = isProPlus && (result._failure_analysis ?? isProPlus);
|
|
186
219
|
console.log(` Verification tier: ${view.tier}`);
|
|
187
220
|
console.log(` Question: ${view.question}`);
|
|
188
221
|
console.log(` Verdict: ${view.verdict} (${view.confidence})`);
|
|
189
222
|
console.log(` Summary: ${view.summary}`);
|
|
223
|
+
// Pro/Pro+: 체크 통과 목록 요약
|
|
224
|
+
if (isPro && view.passes.length > 0) {
|
|
225
|
+
const passedChecks = view.passes.filter((p) => p.passed).map((p) => p.label);
|
|
226
|
+
const failedChecks = view.passes.filter((p) => !p.passed).map((p) => p.label);
|
|
227
|
+
if (passedChecks.length > 0)
|
|
228
|
+
console.log(` Passed: ${passedChecks.join(", ")}`);
|
|
229
|
+
if (failedChecks.length > 0)
|
|
230
|
+
console.log(` Failed: ${failedChecks.join(", ")}`);
|
|
231
|
+
}
|
|
232
|
+
// Blockers: 제목은 모든 티어, Fix 액션은 verbose_failure(Pro+) 이상에서 표시
|
|
190
233
|
if (view.blockers.length > 0) {
|
|
191
234
|
console.log(" Blockers:");
|
|
192
|
-
for (const blocker of view.blockers)
|
|
235
|
+
for (const blocker of view.blockers) {
|
|
193
236
|
console.log(` - ${blocker.title}`);
|
|
237
|
+
if (verboseFailure)
|
|
238
|
+
console.log(` Fix: ${blocker.action}`);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
// Warnings: Pro/Pro+에서만 표시, Review 액션은 failure_analysis(Pro+)에서 표시
|
|
242
|
+
if (isPro && view.warnings.length > 0) {
|
|
243
|
+
console.log(" Warnings:");
|
|
244
|
+
for (const warning of view.warnings) {
|
|
245
|
+
console.log(` - ${warning.title}`);
|
|
246
|
+
if (failureAnalysis)
|
|
247
|
+
console.log(` Review: ${warning.action}`);
|
|
248
|
+
}
|
|
194
249
|
}
|
|
195
250
|
if (view.nextActions.length > 0) {
|
|
196
251
|
console.log(" Next actions:");
|
|
197
252
|
for (const action of view.nextActions)
|
|
198
253
|
console.log(` - ${action}`);
|
|
199
254
|
}
|
|
200
|
-
|
|
255
|
+
// Evidence: failure_analysis(Pro+)는 전체, verbose_failure(Pro)는 3개, Free는 2개
|
|
256
|
+
const evidenceLimit = failureAnalysis ? view.failureEvidence.length : verboseFailure ? 3 : 2;
|
|
257
|
+
const evidenceToShow = view.failureEvidence.slice(0, evidenceLimit);
|
|
258
|
+
if (evidenceToShow.length > 0) {
|
|
201
259
|
console.log(" Evidence:");
|
|
202
|
-
for (const item of
|
|
260
|
+
for (const item of evidenceToShow)
|
|
203
261
|
console.log(` - ${item}`);
|
|
204
262
|
}
|
|
205
263
|
}
|
|
@@ -243,7 +301,9 @@ async function run() {
|
|
|
243
301
|
--config <path> Path to .laxy.yml
|
|
244
302
|
--fail-on unverified | bronze | silver | gold
|
|
245
303
|
--skip-lighthouse Skip Lighthouse but still run build and E2E
|
|
304
|
+
--plan-override free | pro | pro_plus (downgrade testing only)
|
|
246
305
|
--multi-viewport Pro+: Lighthouse on desktop/tablet/mobile
|
|
306
|
+
--crawl Crawl the app to discover routes before E2E
|
|
247
307
|
--badge Print shields.io badge markdown
|
|
248
308
|
--help Show this help
|
|
249
309
|
|
|
@@ -350,6 +410,7 @@ async function run() {
|
|
|
350
410
|
}
|
|
351
411
|
let scores;
|
|
352
412
|
let lighthouseResult = null;
|
|
413
|
+
let lighthouseErrorCount = 0;
|
|
353
414
|
const adjustedThresholds = {
|
|
354
415
|
performance: config.ciMode ? config.thresholds.performance - 10 : config.thresholds.performance,
|
|
355
416
|
accessibility: config.thresholds.accessibility,
|
|
@@ -373,23 +434,43 @@ async function run() {
|
|
|
373
434
|
failure_analysis: false,
|
|
374
435
|
fast_lane: false,
|
|
375
436
|
};
|
|
376
|
-
|
|
437
|
+
let effectiveFeatures = features;
|
|
438
|
+
if (args.planOverride) {
|
|
439
|
+
try {
|
|
440
|
+
effectiveFeatures = (0, entitlement_js_1.applyPlanOverride)(features, args.planOverride);
|
|
441
|
+
console.log(` Plan override: ${(0, entitlement_js_1.normalizePlan)(features.plan)} -> ${effectiveFeatures.plan} (testing lower-tier verification behavior)`);
|
|
442
|
+
}
|
|
443
|
+
catch (overrideErr) {
|
|
444
|
+
console.error(`Plan override error: ${overrideErr instanceof Error ? overrideErr.message : String(overrideErr)}`);
|
|
445
|
+
exitGracefully(2);
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
if (effectiveFeatures.lighthouse_runs_3 && config.lighthouse_runs < 3) {
|
|
377
450
|
config = { ...config, lighthouse_runs: 3 };
|
|
378
451
|
}
|
|
379
452
|
let multiViewportScores = null;
|
|
380
453
|
let allViewportsOk = false;
|
|
381
454
|
let e2eResult;
|
|
455
|
+
let crossBrowserResults;
|
|
456
|
+
let e2eCoverageGaps = [];
|
|
457
|
+
let e2eConsoleErrors = [];
|
|
458
|
+
let e2eStabilityPassed = true;
|
|
382
459
|
let visualDiffResult = null;
|
|
460
|
+
let securityAuditResult = null;
|
|
461
|
+
let mobileLighthouseScores = null;
|
|
462
|
+
const failureEvidence = [];
|
|
383
463
|
if (buildResult.success) {
|
|
384
464
|
let servePid;
|
|
385
465
|
try {
|
|
386
466
|
const serve = await (0, serve_js_1.startDevServer)(devCmd, port, config.dev_timeout, args.projectDir);
|
|
387
467
|
servePid = serve.pid;
|
|
388
468
|
const verifyUrl = `http://127.0.0.1:${port}/`;
|
|
389
|
-
const verificationTier = (0, index_js_1.planToVerificationTier)(
|
|
469
|
+
const verificationTier = (0, index_js_1.planToVerificationTier)(effectiveFeatures.plan);
|
|
390
470
|
if (!args.skipLighthouse) {
|
|
391
471
|
try {
|
|
392
472
|
const lhResult = await (0, lighthouse_js_1.runLighthouse)(port, config.lighthouse_runs);
|
|
473
|
+
lighthouseErrorCount = lhResult.errors.length;
|
|
393
474
|
scores = lhResult.scores ?? undefined;
|
|
394
475
|
if (scores) {
|
|
395
476
|
lighthouseResult = {
|
|
@@ -405,31 +486,123 @@ async function run() {
|
|
|
405
486
|
console.error(`Lighthouse error: ${lhErr instanceof Error ? lhErr.message : String(lhErr)}`);
|
|
406
487
|
}
|
|
407
488
|
}
|
|
408
|
-
if (!args.skipLighthouse && args.multiViewport && !
|
|
489
|
+
if (!args.skipLighthouse && args.multiViewport && !effectiveFeatures.multi_viewport) {
|
|
409
490
|
console.log("\n Note: --multi-viewport requires Pro+. Run laxy-verify login with a paid account to unlock it.");
|
|
410
491
|
}
|
|
411
|
-
else if (!args.skipLighthouse &&
|
|
492
|
+
else if (!args.skipLighthouse && effectiveFeatures.multi_viewport) {
|
|
412
493
|
try {
|
|
413
494
|
multiViewportScores = await (0, multi_viewport_js_1.runMultiViewportLighthouse)(port);
|
|
414
495
|
(0, multi_viewport_js_1.printMultiViewportResults)(multiViewportScores, adjustedThresholds);
|
|
415
496
|
allViewportsOk = (0, multi_viewport_js_1.allViewportsPass)(multiViewportScores, adjustedThresholds);
|
|
497
|
+
// Surface screenshot diff issues in failure evidence
|
|
498
|
+
if (multiViewportScores.screenshotDiffs) {
|
|
499
|
+
for (const diff of multiViewportScores.screenshotDiffs) {
|
|
500
|
+
if (!diff.baselineCreated && diff.diffPercent > 10) {
|
|
501
|
+
failureEvidence.push(`Viewport screenshot: ${diff.viewport} diff ${diff.diffPercent}% exceeds 10% threshold`);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
416
505
|
}
|
|
417
506
|
catch (mvErr) {
|
|
418
507
|
console.error(`Multi-viewport error: ${mvErr instanceof Error ? mvErr.message : String(mvErr)}`);
|
|
419
508
|
}
|
|
420
509
|
}
|
|
510
|
+
// Pro: single mobile Lighthouse check (if not using full multi-viewport)
|
|
511
|
+
if (!args.skipLighthouse &&
|
|
512
|
+
verificationTier === "pro" &&
|
|
513
|
+
!effectiveFeatures.multi_viewport) {
|
|
514
|
+
try {
|
|
515
|
+
mobileLighthouseScores = await (0, multi_viewport_js_1.runMobileLighthouse)(port);
|
|
516
|
+
if (mobileLighthouseScores) {
|
|
517
|
+
const mobilePassed = mobileLighthouseScores.performance >= adjustedThresholds.performance &&
|
|
518
|
+
mobileLighthouseScores.accessibility >= adjustedThresholds.accessibility;
|
|
519
|
+
if (!mobilePassed) {
|
|
520
|
+
failureEvidence.push(`Mobile LH: P=${mobileLighthouseScores.performance} A=${mobileLighthouseScores.accessibility}`);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
catch (mobileLhErr) {
|
|
525
|
+
console.error(`Mobile Lighthouse error: ${mobileLhErr instanceof Error ? mobileLhErr.message : String(mobileLhErr)}`);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
// Pro/Pro+: security audit
|
|
529
|
+
if (verificationTier !== "free") {
|
|
530
|
+
try {
|
|
531
|
+
securityAuditResult = await (0, security_audit_js_1.runSecurityAudit)(args.projectDir);
|
|
532
|
+
if (securityAuditResult.critical > 0 || securityAuditResult.high > 0) {
|
|
533
|
+
failureEvidence.push(`Security: ${securityAuditResult.summary}`);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
catch (secErr) {
|
|
537
|
+
console.error(`Security audit error: ${secErr instanceof Error ? secErr.message : String(secErr)}`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
const crawlEnabled = args.crawl || config.crawl;
|
|
541
|
+
const crawlOpts = crawlEnabled
|
|
542
|
+
? { enabled: true, maxDepth: config.max_crawl_depth, maxPages: config.max_crawl_pages }
|
|
543
|
+
: undefined;
|
|
544
|
+
let lastE2EScenarios;
|
|
421
545
|
try {
|
|
422
|
-
const e2eRuns = await (0, e2e_js_1.runVerifyE2E)(verifyUrl, verificationTier);
|
|
546
|
+
const e2eRuns = await (0, e2e_js_1.runVerifyE2E)(verifyUrl, verificationTier, config.scenarios, crawlOpts);
|
|
547
|
+
lastE2EScenarios = e2eRuns.scenarios;
|
|
423
548
|
e2eResult = {
|
|
424
549
|
passed: e2eRuns.passed,
|
|
425
550
|
failed: e2eRuns.failed,
|
|
426
551
|
total: e2eRuns.results.length,
|
|
427
552
|
results: e2eRuns.results,
|
|
428
553
|
};
|
|
554
|
+
e2eCoverageGaps = e2eRuns.coverageGaps;
|
|
555
|
+
e2eConsoleErrors = e2eRuns.consoleErrors;
|
|
556
|
+
// Pro+ stability: run E2E a second time if first run passed all
|
|
557
|
+
if (verificationTier === "pro_plus" && e2eRuns.passed === e2eRuns.results.length && e2eRuns.results.length > 0) {
|
|
558
|
+
console.log(" [Pro+] Running stability pass (run 2/2)...");
|
|
559
|
+
const e2eRuns2 = await (0, e2e_js_1.runVerifyE2E)(verifyUrl, verificationTier, config.scenarios, crawlOpts);
|
|
560
|
+
if (e2eRuns2.passed < e2eRuns2.results.length) {
|
|
561
|
+
e2eStabilityPassed = false;
|
|
562
|
+
e2eCoverageGaps.push("Stability check failed on second run");
|
|
563
|
+
const failedNames = e2eRuns2.results
|
|
564
|
+
.filter((r) => !r.passed)
|
|
565
|
+
.map((r) => r.name)
|
|
566
|
+
.join(", ");
|
|
567
|
+
failureEvidence.push(`E2E stability: second run failed (${failedNames})`);
|
|
568
|
+
}
|
|
569
|
+
else {
|
|
570
|
+
console.log(" [Pro+] Stability pass: OK (2/2 runs passed)");
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
if (e2eRuns.coverageGaps.length > 0) {
|
|
574
|
+
console.error(`E2E coverage warning: ${e2eRuns.coverageGaps.join(" ")}`);
|
|
575
|
+
}
|
|
576
|
+
if (e2eRuns.consoleErrors.length > 0) {
|
|
577
|
+
console.error(`E2E console errors: ${e2eRuns.consoleErrors.length} detected`);
|
|
578
|
+
}
|
|
429
579
|
}
|
|
430
580
|
catch (e2eErr) {
|
|
431
581
|
console.error(`E2E error: ${e2eErr instanceof Error ? e2eErr.message : String(e2eErr)}`);
|
|
432
582
|
}
|
|
583
|
+
// Cross-browser E2E via Playwright (if non-chromium browsers configured)
|
|
584
|
+
const extraBrowsers = (config.browsers || []).filter((b) => b !== "chromium" && ["firefox", "webkit"].includes(b));
|
|
585
|
+
if (extraBrowsers.length > 0 && lastE2EScenarios && lastE2EScenarios.length > 0) {
|
|
586
|
+
const pwAvailable = await (0, playwright_runner_js_1.isPlaywrightAvailable)();
|
|
587
|
+
if (pwAvailable) {
|
|
588
|
+
try {
|
|
589
|
+
crossBrowserResults = await (0, playwright_runner_js_1.runPlaywrightE2E)(verifyUrl, lastE2EScenarios, extraBrowsers);
|
|
590
|
+
for (const cbr of crossBrowserResults) {
|
|
591
|
+
console.log(` Cross-browser ${cbr.browser}: ${cbr.passed}/${cbr.results.length} passed`);
|
|
592
|
+
if (cbr.failed > 0) {
|
|
593
|
+
const failedNames = cbr.results.filter(r => !r.passed).map(r => r.name).join(", ");
|
|
594
|
+
failureEvidence.push(`Cross-browser ${cbr.browser}: ${failedNames} failed`);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
catch (cbErr) {
|
|
599
|
+
console.error(`Cross-browser error: ${cbErr instanceof Error ? cbErr.message : String(cbErr)}`);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
else {
|
|
603
|
+
console.log(" Note: Cross-browser testing requires playwright. Run: npm install -D playwright && npx playwright install");
|
|
604
|
+
}
|
|
605
|
+
}
|
|
433
606
|
if (verificationTier === "pro_plus") {
|
|
434
607
|
try {
|
|
435
608
|
visualDiffResult = await (0, visual_diff_js_1.runVisualDiff)(args.projectDir, verifyUrl, "verify");
|
|
@@ -448,31 +621,30 @@ async function run() {
|
|
|
448
621
|
}
|
|
449
622
|
}
|
|
450
623
|
}
|
|
451
|
-
const verificationTier = (0, index_js_1.planToVerificationTier)(
|
|
624
|
+
const verificationTier = (0, index_js_1.planToVerificationTier)(effectiveFeatures.plan);
|
|
452
625
|
const viewportSummary = summarizeViewportIssues(multiViewportScores, adjustedThresholds);
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
? `Visual diff: ${visualDiffResult.diffPercentage}% (${visualDiffResult.verdict})`
|
|
466
|
-
: "Visual diff: baseline seeded",
|
|
467
|
-
]
|
|
468
|
-
: []),
|
|
469
|
-
];
|
|
626
|
+
failureEvidence.push(...buildResult.errors.slice(0, 3).map((error) => `Build: ${error}`), ...(lighthouseErrorCount > 0 ? [`Lighthouse: ${lighthouseErrorCount} run error(s) were recorded during collection.`] : []), ...(e2eResult
|
|
627
|
+
? e2eResult.results
|
|
628
|
+
.filter((scenario) => !scenario.passed)
|
|
629
|
+
.slice(0, 2)
|
|
630
|
+
.map((scenario) => `E2E: ${scenario.name}${scenario.error ? ` - ${scenario.error}` : ""}`)
|
|
631
|
+
: []), ...e2eConsoleErrors.slice(0, 2).map((e) => `Console: ${e}`), ...e2eCoverageGaps.slice(0, 2).map((gap) => `E2E coverage: ${gap}`), ...(viewportSummary.count > 0 && viewportSummary.summary ? [`Viewport: ${viewportSummary.summary}`] : []), ...(visualDiffResult
|
|
632
|
+
? [
|
|
633
|
+
visualDiffResult.hasBaseline
|
|
634
|
+
? `Visual diff: ${visualDiffResult.diffPercentage}% (${visualDiffResult.verdict})`
|
|
635
|
+
: "Visual diff: baseline seeded",
|
|
636
|
+
]
|
|
637
|
+
: []));
|
|
470
638
|
const verificationReport = (0, index_js_1.buildVerificationReport)({
|
|
471
639
|
buildSuccess: buildResult.success,
|
|
472
640
|
buildErrors: buildResult.errors,
|
|
473
641
|
e2ePassed: e2eResult?.passed,
|
|
474
642
|
e2eTotal: e2eResult?.total,
|
|
643
|
+
e2eCoverageGaps,
|
|
644
|
+
e2eConsoleErrorCount: e2eConsoleErrors.length,
|
|
645
|
+
e2eStabilityPassed,
|
|
475
646
|
lighthouseSkipped: args.skipLighthouse,
|
|
647
|
+
lighthouseErrorCount,
|
|
476
648
|
viewportIssues: multiViewportScores ? viewportSummary.count : undefined,
|
|
477
649
|
multiViewportPassed: multiViewportScores ? allViewportsOk : undefined,
|
|
478
650
|
multiViewportSummary: multiViewportScores ? viewportSummary.summary : undefined,
|
|
@@ -480,6 +652,15 @@ async function run() {
|
|
|
480
652
|
visualDiffPercentage: visualDiffResult?.diffPercentage,
|
|
481
653
|
hasVisualBaseline: visualDiffResult?.hasBaseline,
|
|
482
654
|
lighthouseScores: scores,
|
|
655
|
+
mobileLighthouseScores: mobileLighthouseScores ?? undefined,
|
|
656
|
+
securityAudit: securityAuditResult
|
|
657
|
+
? {
|
|
658
|
+
totalVulnerabilities: securityAuditResult.totalVulnerabilities,
|
|
659
|
+
critical: securityAuditResult.critical,
|
|
660
|
+
high: securityAuditResult.high,
|
|
661
|
+
summary: securityAuditResult.summary,
|
|
662
|
+
}
|
|
663
|
+
: undefined,
|
|
483
664
|
failureEvidence,
|
|
484
665
|
}, {
|
|
485
666
|
tier: verificationTier,
|
|
@@ -487,11 +668,7 @@ async function run() {
|
|
|
487
668
|
});
|
|
488
669
|
const verificationView = (0, index_js_1.getTierVerificationView)(verificationReport);
|
|
489
670
|
const unifiedGrade = verificationReport.grade;
|
|
490
|
-
const exitCode = config.fail_on
|
|
491
|
-
? 0
|
|
492
|
-
: (0, grade_js_1.isWorseOrEqual)(unifiedGrade, config.fail_on)
|
|
493
|
-
? 1
|
|
494
|
-
: 0;
|
|
671
|
+
const exitCode = shouldFailVerificationResult(verificationReport, config.fail_on) ? 1 : 0;
|
|
495
672
|
const resultObj = {
|
|
496
673
|
grade: unifiedGrade.charAt(0).toUpperCase() + unifiedGrade.slice(1),
|
|
497
674
|
timestamp: new Date().toISOString(),
|
|
@@ -501,14 +678,19 @@ async function run() {
|
|
|
501
678
|
errors: buildResult.errors,
|
|
502
679
|
},
|
|
503
680
|
e2e: e2eResult,
|
|
681
|
+
crossBrowser: crossBrowserResults,
|
|
504
682
|
lighthouse: lighthouseResult,
|
|
683
|
+
mobileLighthouse: mobileLighthouseScores,
|
|
684
|
+
security: securityAuditResult,
|
|
505
685
|
visualDiff: visualDiffResult,
|
|
506
686
|
thresholds: adjustedThresholds,
|
|
507
687
|
ciMode: config.ciMode,
|
|
508
688
|
framework: detected.framework,
|
|
509
689
|
exitCode,
|
|
510
690
|
config_fail_on: config.fail_on,
|
|
511
|
-
_plan:
|
|
691
|
+
_plan: effectiveFeatures.plan,
|
|
692
|
+
_verbose_failure: effectiveFeatures.verbose_failure,
|
|
693
|
+
_failure_analysis: effectiveFeatures.failure_analysis,
|
|
512
694
|
verification: {
|
|
513
695
|
tier: verificationTier,
|
|
514
696
|
report: verificationReport,
|
package/dist/config.d.ts
CHANGED
|
@@ -5,6 +5,19 @@ export interface Thresholds {
|
|
|
5
5
|
seo: number;
|
|
6
6
|
bestPractices: number;
|
|
7
7
|
}
|
|
8
|
+
export interface UserScenarioStep {
|
|
9
|
+
goto?: string;
|
|
10
|
+
fill?: string;
|
|
11
|
+
with?: string;
|
|
12
|
+
click?: string;
|
|
13
|
+
expect_visible?: string;
|
|
14
|
+
expect_text?: string;
|
|
15
|
+
wait?: number;
|
|
16
|
+
}
|
|
17
|
+
export interface UserScenario {
|
|
18
|
+
name: string;
|
|
19
|
+
steps: UserScenarioStep[];
|
|
20
|
+
}
|
|
8
21
|
export interface LaxyConfig {
|
|
9
22
|
framework: string;
|
|
10
23
|
build_command: string;
|
|
@@ -16,6 +29,11 @@ export interface LaxyConfig {
|
|
|
16
29
|
lighthouse_runs: number;
|
|
17
30
|
thresholds: Thresholds;
|
|
18
31
|
fail_on: FailOn;
|
|
32
|
+
scenarios?: UserScenario[];
|
|
33
|
+
crawl: boolean;
|
|
34
|
+
max_crawl_depth: number;
|
|
35
|
+
max_crawl_pages: number;
|
|
36
|
+
browsers: string[];
|
|
19
37
|
}
|
|
20
38
|
export declare class ConfigParseError extends Error {
|
|
21
39
|
constructor(msg: string);
|
package/dist/config.js
CHANGED
|
@@ -54,6 +54,10 @@ const DEFAULT_CONFIG = {
|
|
|
54
54
|
bestPractices: 80,
|
|
55
55
|
},
|
|
56
56
|
fail_on: "bronze",
|
|
57
|
+
crawl: false,
|
|
58
|
+
max_crawl_depth: 3,
|
|
59
|
+
max_crawl_pages: 10,
|
|
60
|
+
browsers: ["chromium"],
|
|
57
61
|
};
|
|
58
62
|
const VALID_FAIL_ON = ["unverified", "bronze", "silver", "gold"];
|
|
59
63
|
class ConfigParseError extends Error {
|
|
@@ -86,6 +90,19 @@ function parseYaml(filePath) {
|
|
|
86
90
|
result.dev_timeout = raw.dev_timeout;
|
|
87
91
|
if (typeof raw.lighthouse_runs === "number")
|
|
88
92
|
result.lighthouse_runs = raw.lighthouse_runs;
|
|
93
|
+
if (typeof raw.crawl === "boolean")
|
|
94
|
+
result.crawl = raw.crawl;
|
|
95
|
+
if (typeof raw.max_crawl_depth === "number")
|
|
96
|
+
result.max_crawl_depth = raw.max_crawl_depth;
|
|
97
|
+
if (typeof raw.max_crawl_pages === "number")
|
|
98
|
+
result.max_crawl_pages = raw.max_crawl_pages;
|
|
99
|
+
if (Array.isArray(raw.browsers)) {
|
|
100
|
+
const validBrowsers = ["chromium", "firefox", "webkit"];
|
|
101
|
+
const browsers = raw.browsers
|
|
102
|
+
.filter((b) => typeof b === "string" && validBrowsers.includes(b));
|
|
103
|
+
if (browsers.length > 0)
|
|
104
|
+
result.browsers = browsers;
|
|
105
|
+
}
|
|
89
106
|
if (typeof raw.fail_on === "string") {
|
|
90
107
|
const f = raw.fail_on;
|
|
91
108
|
if (!VALID_FAIL_ON.includes(f)) {
|
|
@@ -108,6 +125,43 @@ function parseYaml(filePath) {
|
|
|
108
125
|
thr.bestPractices = t.best_practices;
|
|
109
126
|
result.thresholds = thr;
|
|
110
127
|
}
|
|
128
|
+
if (Array.isArray(raw.scenarios)) {
|
|
129
|
+
const scenarios = [];
|
|
130
|
+
for (const item of raw.scenarios) {
|
|
131
|
+
if (typeof item === "object" &&
|
|
132
|
+
item !== null &&
|
|
133
|
+
typeof item.name === "string" &&
|
|
134
|
+
Array.isArray(item.steps)) {
|
|
135
|
+
const rawScenario = item;
|
|
136
|
+
const steps = [];
|
|
137
|
+
for (const rawStep of rawScenario.steps) {
|
|
138
|
+
if (typeof rawStep === "object" && rawStep !== null) {
|
|
139
|
+
const s = rawStep;
|
|
140
|
+
const step = {};
|
|
141
|
+
if (typeof s.goto === "string")
|
|
142
|
+
step.goto = s.goto;
|
|
143
|
+
if (typeof s.fill === "string")
|
|
144
|
+
step.fill = s.fill;
|
|
145
|
+
if (typeof s.with === "string")
|
|
146
|
+
step.with = s.with;
|
|
147
|
+
if (typeof s.click === "string")
|
|
148
|
+
step.click = s.click;
|
|
149
|
+
if (typeof s.expect_visible === "string")
|
|
150
|
+
step.expect_visible = s.expect_visible;
|
|
151
|
+
if (typeof s.expect_text === "string")
|
|
152
|
+
step.expect_text = s.expect_text;
|
|
153
|
+
if (typeof s.wait === "number")
|
|
154
|
+
step.wait = s.wait;
|
|
155
|
+
steps.push(step);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
scenarios.push({ name: String(rawScenario.name), steps });
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (scenarios.length > 0) {
|
|
162
|
+
result.scenarios = scenarios;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
111
165
|
return result;
|
|
112
166
|
}
|
|
113
167
|
function loadConfig(options) {
|
|
@@ -127,6 +181,11 @@ function loadConfig(options) {
|
|
|
127
181
|
dev_timeout: base.dev_timeout ?? DEFAULT_CONFIG.dev_timeout,
|
|
128
182
|
lighthouse_runs: base.lighthouse_runs ?? DEFAULT_CONFIG.lighthouse_runs,
|
|
129
183
|
fail_on: base.fail_on ?? DEFAULT_CONFIG.fail_on,
|
|
184
|
+
scenarios: base.scenarios,
|
|
185
|
+
crawl: base.crawl ?? DEFAULT_CONFIG.crawl,
|
|
186
|
+
max_crawl_depth: base.max_crawl_depth ?? DEFAULT_CONFIG.max_crawl_depth,
|
|
187
|
+
max_crawl_pages: base.max_crawl_pages ?? DEFAULT_CONFIG.max_crawl_pages,
|
|
188
|
+
browsers: base.browsers ?? DEFAULT_CONFIG.browsers,
|
|
130
189
|
};
|
|
131
190
|
config.thresholds = { ...DEFAULT_CONFIG.thresholds, ...(base.thresholds ?? {}) };
|
|
132
191
|
// CLI flag overrides
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { E2EScenario } from "./e2e.js";
|
|
2
|
+
import type { VerificationTier } from "./verification-core/types.js";
|
|
3
|
+
export interface CrawlPage {
|
|
4
|
+
url: string;
|
|
5
|
+
path: string;
|
|
6
|
+
title: string;
|
|
7
|
+
forms: CrawlForm[];
|
|
8
|
+
buttons: string[];
|
|
9
|
+
internalLinks: string[];
|
|
10
|
+
hasConsoleErrors: boolean;
|
|
11
|
+
}
|
|
12
|
+
export interface CrawlForm {
|
|
13
|
+
selector: string;
|
|
14
|
+
inputs: {
|
|
15
|
+
selector: string;
|
|
16
|
+
type: string;
|
|
17
|
+
placeholder?: string;
|
|
18
|
+
}[];
|
|
19
|
+
submitSelector?: string;
|
|
20
|
+
}
|
|
21
|
+
export interface CrawlResult {
|
|
22
|
+
pages: CrawlPage[];
|
|
23
|
+
totalLinks: number;
|
|
24
|
+
crawledCount: number;
|
|
25
|
+
}
|
|
26
|
+
export interface CrawlOptions {
|
|
27
|
+
maxDepth?: number;
|
|
28
|
+
maxPages?: number;
|
|
29
|
+
timeout?: number;
|
|
30
|
+
}
|
|
31
|
+
export declare function crawlApp(baseUrl: string, options?: CrawlOptions): Promise<CrawlResult>;
|
|
32
|
+
/**
|
|
33
|
+
* Generate E2E scenarios from crawl results.
|
|
34
|
+
*/
|
|
35
|
+
export declare function buildScenariosFromCrawl(crawlResult: CrawlResult, tier: VerificationTier): E2EScenario[];
|