laxy-verify 1.1.14 → 1.1.16

package/README.md

@@ -1,171 +1,207 @@
-# laxy-verify
-
-CLI verification for frontend apps.
-
-`laxy-verify` runs production build checks, Lighthouse, tiered verify E2E, and plan-gated verification features for Free, Pro, and Pro+ accounts.
-It is designed around three user questions:
-
-- Free: "Is this likely to break right now?"
-- Pro: "Is this strong enough to send to a client?"
-- Pro+: "Can I call this release-ready with confidence?"
-
-```bash
-npx laxy-verify --init --run
+# laxy-verify
+
+CLI verification for frontend apps.
+
+`laxy-verify` runs production build checks, Lighthouse, tiered verify E2E, and plan-gated verification features for Free, Pro, and Pro+ accounts.
+It is designed around three user questions:
+
+- Free: "Is this likely to break right now?"
+- Pro: "Is this strong enough to send to a client?"
+- Pro+: "Can I call this release-ready with confidence?"
+
+```bash
+npx laxy-verify --init --run
 npx laxy-verify .
+npx laxy-verify . --plan-override pro
 npx laxy-verify login
 npx laxy-verify whoami
 npx laxy-verify --help
-```
-
-## Quick Start
-
-### 1. Initialize
-
-```bash
-cd your-project
-npx laxy-verify --init
-```
-
-This generates `.laxy.yml` and a GitHub Actions workflow.
-
-### 2. Run locally
-
-```bash
-npx laxy-verify .
-```
-
-### 3. Add to CI
-
-Commit the generated workflow. Each PR gets a verification run, grade output, and optional GitHub reporting.
-
-## Verification Tiers
-
-| Plan | Question it answers |
-|------|---------------------|
-| Free | Is this likely to break right now? |
-| Pro | Is this strong enough to send to a client? |
-| Pro+ | Can I call this release-ready with confidence? |
-
-## Grades
-
-| Grade | Meaning |
-|-------|---------|
-| Gold | Build passed + E2E passed + Lighthouse passed + Pro+ viewport evidence passed |
-| Silver | Build passed + E2E passed |
-| Bronze | Build passed |
-| Unverified | Build failed |
-
-## Paid Features
-
-Log in with your Laxy account to unlock paid plan features.
-
-```bash
-npx laxy-verify login
-npx laxy-verify whoami
-npx laxy-verify logout
-```
-
+```
+
+## Quick Start
+
+### 1. Initialize
+
+```bash
+cd your-project
+npx laxy-verify --init
+```
+
+This generates `.laxy.yml` and a GitHub Actions workflow.
+
+### 2. Run locally
+
+```bash
+npx laxy-verify .
+```
+
+### 3. Add to CI
+
+Commit the generated workflow. Each PR gets a verification run, grade output, and optional GitHub reporting.
+
+## Verification Tiers
+
+| Plan | Question it answers |
+|------|---------------------|
+| Free | Is this likely to break right now? |
+| Pro | Is this strong enough to send to a client? |
+| Pro+ | Can I call this release-ready with confidence? |
+
+## Grades
+
+| Grade | Meaning |
+|-------|---------|
+| Gold | Build passed + E2E passed + Lighthouse passed + Pro+ viewport evidence passed |
+| Silver | Build passed + E2E passed |
+| Bronze | Build passed |
+| Unverified | Build failed |
+
+## Paid Features
+
+Log in with your Laxy account to unlock paid plan features.
+
+```bash
+npx laxy-verify login
+npx laxy-verify whoami
+npx laxy-verify logout
+```
+
 | Feature | Free | Pro | Pro+ |
 |---------|------|-----|------|
 | Build verification | Yes | Yes | Yes |
 | Lighthouse | 1 run | 3 runs | 3 runs |
 | Verify E2E | Smoke | Deeper client-send checks | Deeper client-send checks |
 | Detailed report view | No | Yes | Yes |
-| Report export | No | Yes | Yes |
+| `laxy-verify-report.md` export | No | Yes | Yes |
 | Multi-viewport verification | No | No | Yes |
 | Visual diff | No | No | Yes |
 | Failure analysis signals | No | No | Yes |
-
-Pro is for delivery verification.
-Pro+ is for release-confidence verification with extra evidence before you say "ship it."
-
-For CI, set `LAXY_TOKEN` instead of using interactive login.
-
-```yaml
-env:
-  LAXY_TOKEN: ${{ secrets.LAXY_TOKEN }}
-```
-
-## Configuration
-
-All fields are optional in `.laxy.yml`.
-
-```yaml
-framework: "auto"
-build_command: ""
-dev_command: ""
-package_manager: "auto"
-port: 3000
-build_timeout: 300
-dev_timeout: 60
-lighthouse_runs: 1
-
-thresholds:
-  performance: 70
-  accessibility: 85
-  seo: 80
-  best_practices: 80
-
-fail_on: "bronze"
-```
-
-## CLI Options
-
-```text
-npx laxy-verify [project-dir]
-
-Options:
-  --format console|json
-  --ci
-  --config <path>
+
+Pro is for delivery verification.
+Pro+ is for release-confidence verification with extra evidence before you say "ship it."
+
+For CI, set `LAXY_TOKEN` instead of using interactive login.
+
+```yaml
+env:
+  LAXY_TOKEN: ${{ secrets.LAXY_TOKEN }}
+```
+
+## Configuration
+
+All fields are optional in `.laxy.yml`.
+
+```yaml
+framework: "auto"
+build_command: ""
+dev_command: ""
+package_manager: "auto"
+port: 3000
+build_timeout: 300
+dev_timeout: 60
+lighthouse_runs: 1
+
+thresholds:
+  performance: 70
+  accessibility: 85
+  seo: 80
+  best_practices: 80
+
+fail_on: "bronze"
+```
+
+## CLI Options
+
+```text
+npx laxy-verify [project-dir]
+
+Options:
+  --format console|json
+  --ci
+  --config <path>
   --fail-on unverified|bronze|silver|gold
   --skip-lighthouse
+  --plan-override free|pro|pro_plus
   --badge
   --init
   --multi-viewport
   --help
-
+
 Subcommands:
   login [email]
   logout
   whoami
 ```
 
-## Result File
+`--plan-override` is for downgrade testing only.
+Example: if your account is Pro+, you can run `--plan-override pro` or `--plan-override free` to verify the lower-tier behavior without changing your subscription.
+It will reject upgrades above your real entitlement.
+
+## Result Files
 
 Each run writes `.laxy-result.json`.
 
+Paid plans also write a readable markdown summary to `laxy-verify-report.md`.
+
+- `Pro`: blocker-focused delivery report
+- `Pro+`: release-readiness report with viewport and visual evidence
+
+Exit behavior follows the verification verdict, not just the legacy grade.
+
+- `build-failed` -> exit 1
+- `hold` -> exit 1
+- `Pro+ investigate` -> exit 1
+- plain lower-tier pass states can still exit 0
+
 ```json
 {
-  "grade": "Gold",
-  "timestamp": "2026-04-09T09:00:00Z",
-  "build": { "success": true, "durationMs": 12000, "errors": [] },
-  "e2e": { "passed": 5, "failed": 0, "total": 5, "results": [] },
-  "lighthouse": { "performance": 82, "accessibility": 94, "seo": 90, "bestPractices": 92, "runs": 3 },
-  "multiViewport": {
-    "allPassed": true,
-    "summary": "Desktop, tablet, and mobile checks passed."
-  },
-  "visualDiff": {
-    "verdict": "pass",
-    "differencePercentage": 0
-  },
-  "verification": {
-    "tier": "pro_plus",
-    "report": { "verdict": "release-ready" }
-  },
-  "exitCode": 0,
-  "_plan": "pro_plus"
+  "grade": "Gold",
+  "timestamp": "2026-04-09T09:00:00Z",
+  "build": { "success": true, "durationMs": 12000, "errors": [] },
+  "e2e": { "passed": 5, "failed": 0, "total": 5, "results": [] },
+  "lighthouse": { "performance": 82, "accessibility": 94, "seo": 90, "bestPractices": 92, "runs": 3 },
+  "multiViewport": {
+    "allPassed": true,
+    "summary": "Desktop, tablet, and mobile checks passed."
+  },
+  "visualDiff": {
+    "verdict": "pass",
+    "differencePercentage": 0
+  },
+  "verification": {
+    "tier": "pro_plus",
+    "report": { "verdict": "release-ready" }
+  },
+  "exitCode": 0,
+  "_plan": "pro_plus"
 }
 ```
 
-## Limitations
-
-- Monorepos require targeting the app subdirectory explicitly.
-- Dev-server-based Lighthouse can differ from production hosting.
-- Pro+ visual diff and viewport checks increase runtime.
-- Local verification is most stable on current LTS Node releases.
-
-## License
-
-MIT
+### `laxy-verify-report.md`
+
+For Pro and Pro+ runs, the markdown report is designed to be easy to read and easy to paste into an AI coding tool.
+
+It includes:
+
+- the main decision in plain English
+- what passed
+- blockers and warnings
+- exact verification evidence
+- failed E2E scenarios
+- a `Copy For AI` section you can paste directly into Codex, Cursor, Claude, or ChatGPT
+
+## Regression Fixtures
+
+The repo also includes dedicated regression fixtures under `.qa-regression-fixtures/`.
+They intentionally break build, navigation, coverage, performance, viewport behavior, and visual stability so the verifier can be tested against known failure modes.
+
+## Limitations
+
+- Monorepos require targeting the app subdirectory explicitly.
+- Dev-server-based Lighthouse can differ from production hosting.
+- Pro+ visual diff and viewport checks increase runtime.
+- Local verification is most stable on current LTS Node releases.
+
+## License
+
+MIT

package/dist/cli.js

@@ -53,9 +53,19 @@ const auth_js_1 = require("./auth.js");
 const entitlement_js_1 = require("./entitlement.js");
 const multi_viewport_js_1 = require("./multi-viewport.js");
 const e2e_js_1 = require("./e2e.js");
+const report_markdown_js_1 = require("./report-markdown.js");
 const visual_diff_js_1 = require("./visual-diff.js");
 const index_js_1 = require("./verification-core/index.js");
 const package_json_1 = __importDefault(require("../package.json"));
+function shouldFailVerificationResult(report, failOn) {
+    if (failOn === "unverified")
+        return false;
+    if (report.verdict === "build-failed" || report.verdict === "hold")
+        return true;
+    if (report.tier === "pro_plus" && report.verdict === "investigate")
+        return true;
+    return (0, grade_js_1.isWorseOrEqual)(report.grade, failOn);
+}
 function exitGracefully(code) {
     if (process.platform === "win32") {
         setTimeout(() => process.exit(code), 100);
@@ -116,6 +126,7 @@ function parseArgs() {
         initRun: flags.init !== undefined && flags.run !== undefined,
         multiViewport: flags["multi-viewport"] !== undefined,
         failureAnalysis: flags["failure-analysis"] !== undefined,
+        planOverride: flags["plan-override"],
         help: flags.help !== undefined || flags.h !== undefined,
     };
 }
@@ -209,6 +220,9 @@ function consoleOutput(result) {
             console.log(`  Status check: ${result.github.grade}`);
     }
     console.log("  Result: .laxy-result.json");
+    if (result.markdownReportPath) {
+        console.log(`  Report: ${path.basename(result.markdownReportPath)}`);
+    }
     console.log(`  Exit code: ${result.exitCode}`);
     if ((result.grade === "Silver" || result.grade === "Bronze") && (!result._plan || result._plan === "free")) {
         console.log("\n  Unlock deeper verification and Gold-grade confidence with Pro or Pro+:");
@@ -218,43 +232,44 @@ function consoleOutput(result) {
 async function run() {
     const args = parseArgs();
     if (args.help) {
-        console.log(`
-  laxy-verify v${package_json_1.default.version}
-  Frontend quality gate: build + Lighthouse verification
-
-  Usage:
-    npx laxy-verify [project-dir] [options]
-    npx laxy-verify <subcommand>
-
-  Subcommands:
-    login [email]     Log in to unlock Pro/Pro+ features
-    logout            Remove saved credentials
-    whoami            Show current login status
-
-  Options:
-    --init            Generate .laxy.yml + GitHub workflow file
-    --init --run      Generate config and immediately run verification
-    --format          console | json  (default: console)
-    --ci              CI mode: -10 Performance threshold, runs=3
-    --config <path>   Path to .laxy.yml
+        console.log(`
+  laxy-verify v${package_json_1.default.version}
+  Frontend quality gate: build + Lighthouse verification
+
+  Usage:
+    npx laxy-verify [project-dir] [options]
+    npx laxy-verify <subcommand>
+
+  Subcommands:
+    login [email]     Log in to unlock Pro/Pro+ features
+    logout            Remove saved credentials
+    whoami            Show current login status
+
+  Options:
+    --init            Generate .laxy.yml + GitHub workflow file
+    --init --run      Generate config and immediately run verification
+    --format          console | json  (default: console)
+    --ci              CI mode: -10 Performance threshold, runs=3
+    --config <path>   Path to .laxy.yml
     --fail-on         unverified | bronze | silver | gold
     --skip-lighthouse Skip Lighthouse but still run build and E2E
+    --plan-override   free | pro | pro_plus (downgrade testing only)
     --multi-viewport  Pro+: Lighthouse on desktop/tablet/mobile
-    --badge           Print shields.io badge markdown
-    --help            Show this help
-
-  Exit codes:
-    0   Grade meets or exceeds fail_on threshold
-    1   Grade worse than fail_on, or build failed
-    2   Configuration error
-
-  Examples:
-    npx laxy-verify --init --run        # Setup + first verification
-    npx laxy-verify .                   # Run in current directory
-    npx laxy-verify . --ci              # CI mode
-    npx laxy-verify . --fail-on silver  # Require Silver or better
-
-  Docs: https://github.com/psungmin24/laxy-verify
+    --badge           Print shields.io badge markdown
+    --help            Show this help
+
+  Exit codes:
+    0   Grade meets or exceeds fail_on threshold
+    1   Grade worse than fail_on, or build failed
+    2   Configuration error
+
+  Examples:
+    npx laxy-verify --init --run        # Setup + first verification
+    npx laxy-verify .                   # Run in current directory
+    npx laxy-verify . --ci              # CI mode
+    npx laxy-verify . --fail-on silver  # Require Silver or better
+
+  Docs: https://github.com/psungmin24/laxy-verify
 `);
         exitGracefully(0);
         return;
@@ -346,6 +361,7 @@ async function run() {
     }
     let scores;
     let lighthouseResult = null;
+    let lighthouseErrorCount = 0;
     const adjustedThresholds = {
         performance: config.ciMode ? config.thresholds.performance - 10 : config.thresholds.performance,
         accessibility: config.thresholds.accessibility,
@@ -369,12 +385,25 @@ async function run() {
         failure_analysis: false,
         fast_lane: false,
     };
-    if (features.lighthouse_runs_3 && config.lighthouse_runs < 3) {
+    let effectiveFeatures = features;
+    if (args.planOverride) {
+        try {
+            effectiveFeatures = (0, entitlement_js_1.applyPlanOverride)(features, args.planOverride);
+            console.log(`  Plan override: ${(0, entitlement_js_1.normalizePlan)(features.plan)} -> ${effectiveFeatures.plan} (testing lower-tier verification behavior)`);
+        }
+        catch (overrideErr) {
+            console.error(`Plan override error: ${overrideErr instanceof Error ? overrideErr.message : String(overrideErr)}`);
+            exitGracefully(2);
+            return;
+        }
+    }
+    if (effectiveFeatures.lighthouse_runs_3 && config.lighthouse_runs < 3) {
         config = { ...config, lighthouse_runs: 3 };
     }
     let multiViewportScores = null;
     let allViewportsOk = false;
     let e2eResult;
+    let e2eCoverageGaps = [];
     let visualDiffResult = null;
     if (buildResult.success) {
         let servePid;
@@ -382,10 +411,11 @@ async function run() {
             const serve = await (0, serve_js_1.startDevServer)(devCmd, port, config.dev_timeout, args.projectDir);
             servePid = serve.pid;
             const verifyUrl = `http://127.0.0.1:${port}/`;
-            const verificationTier = (0, index_js_1.planToVerificationTier)(features.plan);
+            const verificationTier = (0, index_js_1.planToVerificationTier)(effectiveFeatures.plan);
             if (!args.skipLighthouse) {
                 try {
                     const lhResult = await (0, lighthouse_js_1.runLighthouse)(port, config.lighthouse_runs);
+                    lighthouseErrorCount = lhResult.errors.length;
                     scores = lhResult.scores ?? undefined;
                     if (scores) {
                         lighthouseResult = {
@@ -401,10 +431,10 @@ async function run() {
                     console.error(`Lighthouse error: ${lhErr instanceof Error ? lhErr.message : String(lhErr)}`);
                 }
             }
-            if (!args.skipLighthouse && args.multiViewport && !features.multi_viewport) {
+            if (!args.skipLighthouse && args.multiViewport && !effectiveFeatures.multi_viewport) {
                 console.log("\n  Note: --multi-viewport requires Pro+. Run laxy-verify login with a paid account to unlock it.");
             }
-            else if (!args.skipLighthouse && features.multi_viewport) {
+            else if (!args.skipLighthouse && effectiveFeatures.multi_viewport) {
                 try {
                     multiViewportScores = await (0, multi_viewport_js_1.runMultiViewportLighthouse)(port);
                     (0, multi_viewport_js_1.printMultiViewportResults)(multiViewportScores, adjustedThresholds);
@@ -422,6 +452,10 @@ async function run() {
                     total: e2eRuns.results.length,
                     results: e2eRuns.results,
                 };
+                e2eCoverageGaps = e2eRuns.coverageGaps;
+                if (e2eRuns.coverageGaps.length > 0) {
+                    console.error(`E2E coverage warning: ${e2eRuns.coverageGaps.join(" ")}`);
+                }
             }
             catch (e2eErr) {
                 console.error(`E2E error: ${e2eErr instanceof Error ? e2eErr.message : String(e2eErr)}`);
@@ -444,16 +478,18 @@ async function run() {
             }
         }
     }
-    const verificationTier = (0, index_js_1.planToVerificationTier)(features.plan);
+    const verificationTier = (0, index_js_1.planToVerificationTier)(effectiveFeatures.plan);
     const viewportSummary = summarizeViewportIssues(multiViewportScores, adjustedThresholds);
     const failureEvidence = [
         ...buildResult.errors.slice(0, 3).map((error) => `Build: ${error}`),
+        ...(lighthouseErrorCount > 0 ? [`Lighthouse: ${lighthouseErrorCount} run error(s) were recorded during collection.`] : []),
         ...(e2eResult
             ? e2eResult.results
                 .filter((scenario) => !scenario.passed)
                 .slice(0, 2)
                 .map((scenario) => `E2E: ${scenario.name}${scenario.error ? ` - ${scenario.error}` : ""}`)
             : []),
+        ...e2eCoverageGaps.slice(0, 2).map((gap) => `E2E coverage: ${gap}`),
         ...(viewportSummary.count > 0 && viewportSummary.summary ? [`Viewport: ${viewportSummary.summary}`] : []),
         ...(visualDiffResult
             ? [
@@ -468,7 +504,9 @@ async function run() {
         buildErrors: buildResult.errors,
         e2ePassed: e2eResult?.passed,
         e2eTotal: e2eResult?.total,
+        e2eCoverageGaps,
         lighthouseSkipped: args.skipLighthouse,
+        lighthouseErrorCount,
         viewportIssues: multiViewportScores ? viewportSummary.count : undefined,
         multiViewportPassed: multiViewportScores ? allViewportsOk : undefined,
         multiViewportSummary: multiViewportScores ? viewportSummary.summary : undefined,
@@ -483,11 +521,7 @@ async function run() {
     });
     const verificationView = (0, index_js_1.getTierVerificationView)(verificationReport);
     const unifiedGrade = verificationReport.grade;
-    const exitCode = config.fail_on === "unverified"
-        ? 0
-        : (0, grade_js_1.isWorseOrEqual)(unifiedGrade, config.fail_on)
-            ? 1
-            : 0;
+    const exitCode = shouldFailVerificationResult(verificationReport, config.fail_on) ? 1 : 0;
     const resultObj = {
         grade: unifiedGrade.charAt(0).toUpperCase() + unifiedGrade.slice(1),
         timestamp: new Date().toISOString(),
@@ -504,13 +538,22 @@ async function run() {
         framework: detected.framework,
         exitCode,
         config_fail_on: config.fail_on,
-        _plan: features.plan,
+        _plan: effectiveFeatures.plan,
         verification: {
             tier: verificationTier,
             report: verificationReport,
             view: verificationView,
         },
     };
+    const markdownReportPath = (0, report_markdown_js_1.getMarkdownReportPath)(args.projectDir);
+    if ((0, report_markdown_js_1.shouldWriteMarkdownReport)(resultObj)) {
+        const markdownReport = (0, report_markdown_js_1.buildMarkdownReport)(args.projectDir, resultObj);
+        fs.writeFileSync(markdownReportPath, markdownReport, "utf-8");
+        resultObj.markdownReportPath = markdownReportPath;
+    }
+    else if (fs.existsSync(markdownReportPath)) {
+        fs.rmSync(markdownReportPath, { force: true });
+    }
     const inGitHubActions = !!process.env.GITHUB_ACTIONS;
     if (inGitHubActions) {
         try {

package/dist/e2e.d.ts

@@ -1,6 +1,6 @@
 import type { VerificationTier } from "./verification-core/types.js";
 export interface E2EStep {
-    type: "click" | "fill" | "check_visible" | "wait" | "scroll" | "clear_fill" | "check_validation";
+    type: "click" | "fill" | "check_visible" | "wait" | "scroll" | "clear_fill" | "check_validation" | "check_healthy_page";
     selector?: string;
     value?: string;
     duration?: number;
@@ -26,11 +26,13 @@ interface DomSnapshot {
     structures: string[];
 }
 export declare function isNavigableInternalHref(href: string): boolean;
+export declare function getVerificationCoverageGaps(scenarios: E2EScenario[], tier: VerificationTier): string[];
 export declare function buildVerifyScenarios(snapshot: DomSnapshot, tier: VerificationTier): E2EScenario[];
 export declare function runVerifyE2E(url: string, tier: VerificationTier): Promise<{
     scenarios: E2EScenario[];
     results: E2EScenarioResult[];
     passed: number;
     failed: number;
+    coverageGaps: string[];
 }>;
 export {};