lighthouse-reporting 1.3.0 → 1.4.0

package/README.md

@@ -4,10 +4,295 @@
 
 The reports include trend history support, allowing you to track performance improvements over time.
 
+<p align="center">
+  <img alt="HTML" src="./docs/html_report.png" width="45%">
+&nbsp; &nbsp; &nbsp; &nbsp;
+  <img alt="Plot" src="./docs/plot_report.png" width="45%">
+</p>
+
 ## Usage
 
-### Jenkins
+Examples of usage with Playwright + Lighthouse (and Storybook).
+
+### In your frontend or testing framework
+The example of usage [playwright](https://github.com/microsoft/playwright) and [playwright-lighthouse](https://github.com/abhinaba-ghosh/playwright-lighthouse) together.
+
+<details>
+  <summary>lighthouse_pages.spec</summary>
+
+```ts
+import path from 'path'
+import { playAudit } from 'playwright-lighthouse'
+import { playwrightLighthouseTest, getScores, writeCsvResult, writeHtmlListEntryWithRetry, LighthouseResult } from 'lighthouse-reporting'
+import { MyPage1 } from '../../pages/my-page-1.page.js'
+import { MyPage2 } from '../../pages/my-page-1.page.js'
+
+playwrightLighthouseTest.setTimeout(60000)
+const reportDir = path.join(process.cwd(), 'lighthouse')
+const htmlFilePath = path.join(reportDir, 'index.html')
+
+const swimlanes = ['performance']
+const lighthousePages = [
+    { name: 'MyPage1', po: MyPage1, thresholds: { performance: 80, accessibility: 88, seo: 92 }, swimlanes },
+    { name: 'MyPage2', po: MyPage2, thresholds: { performance: 90, accessibility: 100, seo: 90 }, swimlanes },
+]
+
+lighthousePages.forEach(({ name, po, thresholds, swimlanes }) => {
+    playwrightLighthouseTest(name, async ({ port, baseURL }) => {
+        const onlyCategories = ['accessibility', 'seo', 'performance']
+
+        const result: LighthouseResult = await playAudit({
+            url: baseURL + po.getPath('123'),
+            port,
+            thresholds,
+            reports: {
+                formats: {
+                    html: true,
+                },
+                name,
+                directory: reportDir,
+            },
+            opts: {
+                onlyCategories,
+                screenEmulation: { disabled: true },
+            },
+            disableLogs: true,
+            ignoreError: true,
+        })
+
+        const scores = getScores(result)
+        await writeCsvResult(reportDir, name, scores, thresholds, swimlanes)
+        await writeHtmlListEntryWithRetry(htmlFilePath, name, scores, thresholds, result.comparisonError)
+
+        if (result.comparisonError) {
+            throw new Error(result.comparisonError)
+        }
+    })
+})
+```
+
+</details>
+
+### Or in Storybook
+
+An example for storybook with lighthouse and screenshot testing
+
+<details>
+  <summary>storybook.spec.ts</summary>
+
+```ts
+import path from 'path'
+import { BrowserContext } from '@playwright/test'
+import { playAudit } from 'playwright-lighthouse'
+import {
+    playwrightLighthouseTest,
+    getScores,
+    writeCsvResult,
+    writeHtmlListEntryWithRetry,
+    LighthouseResult,
+    StorybookIndexStory,
+    storybookPlaywright,
+    writeScoresToJson
+} from 'lighthouse-reporting'
+
+playwrightLighthouseTest.setTimeout(60000)
+const lhScoresDir = path.join(process.cwd(), process.env.LH_SCORES_DIR || 'lh-scores')
+const reportDir = path.join(process.cwd(), process.env.LH_REPORT_DIR || 'lighthouse')
+const htmlFilePath = path.join(reportDir, 'index.html')
+
+const stories = storybookPlaywright.getStories('./storybook-static/index.json', (story) => {
+    // skip docs, etc
+    if (story.type !== 'story') {
+        return false
+    }
+    // only include stories with test tag
+    if (!story.tags.includes('test')) {
+        return false
+    }
+    return true
+})
+
+stories.forEach((story) => {
+    playwrightLighthouseTest(`${story.title} - ${story.name}`, async ({ context, port, baseURL }) => {
+        await runLighthouse(story, context, port, baseURL)
+        await storybookPlaywright.captureScreenshot(story, context)
+    })
+})
+
+const runLighthouse = async (story: StorybookIndexStory, context: BrowserContext, port: number, baseURL?: string) => {
+    const onlyCategories = ['accessibility']
+    const thresholds = { accessibility: 100 }
+    const name = story.id
+
+    const page = context.pages()[0]
+    await page.goto(`/iframe.html?id=${story.id}`)
+
+    const result: LighthouseResult = await playAudit({
+        url: baseURL + `/iframe.html?id=${story.id}`,
+        port,
+        thresholds,
+        reports: {
+            formats: {
+                html: true,
+            },
+            name,
+            directory: reportDir,
+        },
+        opts: {
+            onlyCategories,
+            screenEmulation: { disabled: true },
+        },
+        disableLogs: true,
+        ignoreError: true,
+    })
+
+    const scores = getScores(result)
+    await writeCsvResult(reportDir, name, scores, thresholds)
+    await writeHtmlListEntryWithRetry(htmlFilePath, name, scores, thresholds, result.comparisonError)
+    // write score results in JSON, allows generating the Average csv report
+    await writeScoresToJson(lhScoresDir, name, scores, result)
+}
+```
+
+</details>
+
+### Configs examples
+
+<details>
+  <summary>playwright.storybook.config.ts</summary>
+
+```ts
+import { PlaywrightTestConfig } from '@playwright/test'
+
+const baseURL = 'http://127.0.0.1:6009'
+// process.env.LH_REPORT_DIR = 'lighthouse-storybook' // adjust lighthouse output folder if required
+// process.env.LH_SCORES_DIR = 'lh-scores' // to write and store scores in json format or write average report
+
+
+const config: PlaywrightTestConfig = {
+    use: {
+        viewport: { width: 1280, height: 820 },
+        ignoreHTTPSErrors: true,
+        acceptDownloads: false,
+        trace: 'off',
+        baseURL,
+        screenshot: { mode: 'off' },
+    },
+    projects: [
+        {
+            name: 'chromium',
+            use: {
+                browserName: 'chromium',
+                launchOptions: { args: ['--disable-gpu'] },
+            },
+            retries: 0,
+        },
+    ],
+    expect: { toMatchSnapshot: { threshold: 0.2 } },
+    reporter: 'line',
+    testDir: 'test/storybook',
+    testMatch: '*.spec.ts',
+    fullyParallel: true,
+    globalSetup: './src/global-setup.ts',
+    globalTeardown: './src/global-teardown.ts',
+    forbidOnly: true,
+    webServer: [
+        {
+            command: 'npx http-server ./storybook-static --port 6009 --silent',
+            url: `${baseURL}/index.json`,
+            timeout: 15 * 1000,
+            reuseExistingServer: false,
+            ignoreHTTPSErrors: true,
+        },
+    ],
+}
+export default config
+```
+
+</details>
+
+<details>
+  <summary>global-setup.ts</summary>
+
+```ts
+import { lighthouseSetup } from 'lighthouse-reporting'
+
+async function globalSetup() {
+    await lighthouseSetup()
+}
+
+export default globalSetup
+```
+
+</details>
+
+<details>
+  <summary>global-teardown.ts</summary>
+
+```ts
+import { lighthousePlaywrightTeardown, buildAverageCsv } from 'lighthouse-reporting'
+
+const lhScoresDir = path.join(process.cwd(), process.env.LH_SCORES_DIR || 'lh-scores')
+
+async function globalTeardown() {
+    await lighthousePlaywrightTeardown()
+    await buildAverageCsv(lhScoresDir)
+}
+
+export default globalTeardown
+```
+
+</details>
+
+## Jenkins
+
+Plugins used [HTML Publisher](https://plugins.jenkins.io/htmlpublisher/) and [Plot](https://plugins.jenkins.io/plot/).
+
+<details>
+  <summary>Jenkinsfile</summary>
+
+```
+stage('Lighthouse') {
+    steps {
+        # run playwright-lighthouse tests
+    }
+
+    post {
+        always {
+            # html report per build
+            publishHTML(target: [
+                reportName           : 'Lighthouse',
+                reportDir            : "$WORKSPACE/lighthouse",
+                reportFiles          : 'index.html',
+                keepAll              : true,
+                alwaysLinkToLastBuild: true,
+                allowMissing         : false
+            ])
+
+            # csv trend history report
+            script {
+                csvFiles = findFiles(glob: 'lighthouse/*.csv')
+                for (csvFile in csvFiles) {
+                    filePath = "${csvFile}"
+                    plot(csvFileName: filePath.substring(filePath.lastIndexOf("/") + 1),
+                        csvSeries: [[displayTableFlag: false, exclusionValues: '', file: filePath, inclusionFlag: 'OFF', url: '']],
+                        exclZero: true,
+                        group: 'my app',
+                        numBuilds: '100',
+                        style: 'line',
+                        title: filePath.substring(filePath.lastIndexOf("/") + 1, filePath.indexOf(".")),
+                        yaxis: 'score',
+                        yaxisMaximum: '101',
+                        yaxisMinimum: '15') # adjust scale
+                }
+            }
+        }
+    }
+}
+```
+
+</details>
 
-### GitHub Actions
+## GitHub Actions
 
 TODO

package/dist/index.cjs

@@ -13,8 +13,10 @@ require("get-port");
 require("@playwright/test");
 exports.lighthousePlaywrightTeardown = hooks.lighthousePlaywrightTeardown;
 exports.lighthouseSetup = hooks.lighthouseSetup;
+exports.buildAverageCsv = lighthouseReports.buildAverageCsv;
 exports.getScores = lighthouseReports.getScores;
 exports.writeCsvResult = lighthouseReports.writeCsvResult;
 exports.writeHtmlListEntryWithRetry = lighthouseReports.writeHtmlListEntryWithRetry;
+exports.writeScoresToJson = lighthouseReports.writeScoresToJson;
 exports.playwrightLighthouseTest = playwrightLighthouseTest.playwrightLighthouseTest;
 exports.storybookPlaywright = storybookPlaywright.storybookPlaywright;

package/dist/index.js

@@ -1,5 +1,5 @@
 import { lighthousePlaywrightTeardown, lighthouseSetup } from "./hooks.js";
-import { getScores, writeCsvResult, writeHtmlListEntryWithRetry } from "./lighthouseReports.js";
+import { buildAverageCsv, getScores, writeCsvResult, writeHtmlListEntryWithRetry, writeScoresToJson } from "./lighthouseReports.js";
 import { playwrightLighthouseTest } from "./playwrightLighthouseTest.js";
 import { storybookPlaywright } from "./storybookPlaywright.js";
 import "os";
@@ -10,11 +10,13 @@ import "./constants-226e9774.js";
 import "get-port";
 import "@playwright/test";
 export {
+  buildAverageCsv,
   getScores,
   lighthousePlaywrightTeardown,
   lighthouseSetup,
   playwrightLighthouseTest,
   storybookPlaywright,
   writeCsvResult,
-  writeHtmlListEntryWithRetry
+  writeHtmlListEntryWithRetry,
+  writeScoresToJson
 };

package/dist/lighthouse-reporting.umd.cjs

@@ -21,14 +21,14 @@
   async function lighthousePlaywrightTeardown() {
     await fse.remove(path.join(os.tmpdir(), PW_TMP_DIR));
   }
-  const writeCsvResult = async (reportDir2, name, scores, thresholds, swimlanes = []) => {
+  const writeCsvResult = async (reportDir2, name, scores, thresholds = {}, swimlanes = []) => {
     let csvData = Object.keys(scores).join(",");
     if (swimlanes.length > 0) {
       csvData += "," + swimlanes.map((swimlane) => `${swimlane}_threshold`);
     }
     csvData += "\n";
     csvData += Object.values(scores).join(",");
-    if (swimlanes.length > 0) {
+    if (swimlanes.length > 0 && Object.keys(thresholds).length > 0) {
       csvData += "," + swimlanes.map((swimlane) => thresholds[swimlane]);
     }
     await fse.writeFile(path.join(reportDir2, `${name}.csv`), csvData);
@@ -61,6 +61,42 @@
     prev[key] = Math.floor(c.score * 100);
     return prev;
   }, {});
+  const writeScoresToJson = async (lhScoresDir, name, scores, result) => {
+    const json = Object.entries(scores).reduce((prev, [k, score]) => {
+      prev[k] = { score };
+      return prev;
+    }, {});
+    const accessibilityViolations = result.artifacts.Accessibility.violations.map((v) => {
+      return {
+        title: result.lhr.audits[v.id].title,
+        nodes: v.nodes.length
+      };
+    });
+    if (accessibilityViolations.length > 0) {
+      json.accessibility.issues = accessibilityViolations;
+    }
+    await fse.writeFile(path.join(lhScoresDir, `${name}.json`), JSON.stringify(json, null, 2));
+  };
+  const buildAverageCsv = async (lhScoresDir) => {
+    const files = await fse.readdir(lhScoresDir);
+    const jsonFiles = files.filter((f) => f.endsWith(".json"));
+    const scores = {};
+    for (const fileName of jsonFiles) {
+      const score = await fse.readJson(
+        path.join(lhScoresDir, fileName)
+      );
+      Object.entries(score).forEach(([k, v]) => {
+        if (!scores[k]) {
+          scores[k] = 0;
+        }
+        scores[k] += v.score;
+      });
+    }
+    Object.entries(scores).forEach(([k, v]) => {
+      scores[k] = v / jsonFiles.length;
+    });
+    await writeCsvResult(lhScoresDir, "_AVERAGE_", scores);
+  };
   class Locked extends Error {
     constructor(port) {
       super(`${port} is locked`);
@@ -71,7 +107,7 @@
     young: /* @__PURE__ */ new Set()
   };
   const releaseOldLockedPortsIntervalMs = 1e3 * 15;
-  let interval;
+  let timeout;
   const getLocalHosts = () => {
     const interfaces = os.networkInterfaces();
     const results = /* @__PURE__ */ new Set([void 0, "0.0.0.0"]);
@@ -137,13 +173,14 @@
         exclude = new Set(excludeIterable);
       }
     }
-    if (interval === void 0) {
-      interval = setInterval(() => {
+    if (timeout === void 0) {
+      timeout = setTimeout(() => {
+        timeout = void 0;
         lockedPorts.old = lockedPorts.young;
         lockedPorts.young = /* @__PURE__ */ new Set();
       }, releaseOldLockedPortsIntervalMs);
-      if (interval.unref) {
-        interval.unref();
+      if (timeout.unref) {
+        timeout.unref();
       }
     }
     const hosts = getLocalHosts();
@@ -214,6 +251,7 @@
       await test.expect(page).toHaveScreenshot(`${story.id}.png`, screenshotOptions);
     }
   };
+  exports2.buildAverageCsv = buildAverageCsv;
   exports2.getScores = getScores;
   exports2.lighthousePlaywrightTeardown = lighthousePlaywrightTeardown;
   exports2.lighthouseSetup = lighthouseSetup;
@@ -221,5 +259,6 @@
   exports2.storybookPlaywright = storybookPlaywright;
   exports2.writeCsvResult = writeCsvResult;
   exports2.writeHtmlListEntryWithRetry = writeHtmlListEntryWithRetry;
+  exports2.writeScoresToJson = writeScoresToJson;
   Object.defineProperty(exports2, Symbol.toStringTag, { value: "Module" });
 });

package/dist/lighthouseReports.cjs

@@ -2,14 +2,14 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const path = require("path");
 const fse = require("fs-extra");
-const writeCsvResult = async (reportDir, name, scores, thresholds, swimlanes = []) => {
+const writeCsvResult = async (reportDir, name, scores, thresholds = {}, swimlanes = []) => {
   let csvData = Object.keys(scores).join(",");
   if (swimlanes.length > 0) {
     csvData += "," + swimlanes.map((swimlane) => `${swimlane}_threshold`);
   }
   csvData += "\n";
   csvData += Object.values(scores).join(",");
-  if (swimlanes.length > 0) {
+  if (swimlanes.length > 0 && Object.keys(thresholds).length > 0) {
     csvData += "," + swimlanes.map((swimlane) => thresholds[swimlane]);
   }
   await fse.writeFile(path.join(reportDir, `${name}.csv`), csvData);
@@ -42,6 +42,44 @@ const getScores = (result) => Object.entries(result.lhr.categories).reduce((prev
   prev[key] = Math.floor(c.score * 100);
   return prev;
 }, {});
+const writeScoresToJson = async (lhScoresDir, name, scores, result) => {
+  const json = Object.entries(scores).reduce((prev, [k, score]) => {
+    prev[k] = { score };
+    return prev;
+  }, {});
+  const accessibilityViolations = result.artifacts.Accessibility.violations.map((v) => {
+    return {
+      title: result.lhr.audits[v.id].title,
+      nodes: v.nodes.length
+    };
+  });
+  if (accessibilityViolations.length > 0) {
+    json.accessibility.issues = accessibilityViolations;
+  }
+  await fse.writeFile(path.join(lhScoresDir, `${name}.json`), JSON.stringify(json, null, 2));
+};
+const buildAverageCsv = async (lhScoresDir) => {
+  const files = await fse.readdir(lhScoresDir);
+  const jsonFiles = files.filter((f) => f.endsWith(".json"));
+  const scores = {};
+  for (const fileName of jsonFiles) {
+    const score = await fse.readJson(
+      path.join(lhScoresDir, fileName)
+    );
+    Object.entries(score).forEach(([k, v]) => {
+      if (!scores[k]) {
+        scores[k] = 0;
+      }
+      scores[k] += v.score;
+    });
+  }
+  Object.entries(scores).forEach(([k, v]) => {
+    scores[k] = v / jsonFiles.length;
+  });
+  await writeCsvResult(lhScoresDir, "_AVERAGE_", scores);
+};
+exports.buildAverageCsv = buildAverageCsv;
 exports.getScores = getScores;
 exports.writeCsvResult = writeCsvResult;
 exports.writeHtmlListEntryWithRetry = writeHtmlListEntryWithRetry;
+exports.writeScoresToJson = writeScoresToJson;

package/dist/lighthouseReports.d.ts

@@ -1,3 +1,29 @@
+interface NodeDetails {
+    lhId: string;
+    devtoolsNodePath: string;
+    selector: string;
+    boudingRect: {
+        [k: string]: number;
+    };
+    snippet: string;
+    nodeLabel: string;
+}
+interface RuleExecutionError {
+    name: string;
+    message: string;
+}
+interface AxeRuleResult {
+    id: string;
+    impact?: string;
+    tags: Array<string>;
+    nodes: Array<{
+        target: Array<string>;
+        failureSummary?: string;
+        node: NodeDetails;
+        relatedNodes: NodeDetails[];
+    }>;
+    error?: RuleExecutionError;
+}
 export interface LighthouseResult {
     lhr: {
         categories: {
@@ -5,12 +31,28 @@ export interface LighthouseResult {
                 score: number;
             };
         };
+        audits: {
+            [k: string]: {
+                title: string;
+            };
+        };
+    };
+    artifacts: {
+        Accessibility: {
+            violations: Array<AxeRuleResult>;
+        };
     };
     comparisonError?: string;
 }
-export declare const writeCsvResult: (reportDir: string, name: string, scores: Record<string, number>, thresholds: Record<string, number>, swimlanes?: Array<string>) => Promise<void>;
+export declare const writeCsvResult: (reportDir: string, name: string, scores: Record<string, number>, thresholds?: Record<string, number>, swimlanes?: Array<string>) => Promise<void>;
 /**
  * workaround conflicts when multiple threads write the same file
  */
 export declare const writeHtmlListEntryWithRetry: (htmlFilePath: string, name: string, scores: Record<string, number>, thresholds: Record<string, number>, comparisonError?: string, addReportLink?: boolean) => Promise<void>;
 export declare const getScores: (result: LighthouseResult) => Record<string, number>;
+export declare const writeScoresToJson: (lhScoresDir: string, name: string, scores: Record<string, number>, result: LighthouseResult) => Promise<void>;
+/**
+ * Generate average csv file. Make sure to use writeScoresToJson in your test!
+ */
+export declare const buildAverageCsv: (lhScoresDir: string) => Promise<void>;
+export {};

package/dist/lighthouseReports.js

@@ -1,13 +1,13 @@
 import path from "path";
 import fse from "fs-extra";
-const writeCsvResult = async (reportDir, name, scores, thresholds, swimlanes = []) => {
+const writeCsvResult = async (reportDir, name, scores, thresholds = {}, swimlanes = []) => {
   let csvData = Object.keys(scores).join(",");
   if (swimlanes.length > 0) {
     csvData += "," + swimlanes.map((swimlane) => `${swimlane}_threshold`);
   }
   csvData += "\n";
   csvData += Object.values(scores).join(",");
-  if (swimlanes.length > 0) {
+  if (swimlanes.length > 0 && Object.keys(thresholds).length > 0) {
     csvData += "," + swimlanes.map((swimlane) => thresholds[swimlane]);
   }
   await fse.writeFile(path.join(reportDir, `${name}.csv`), csvData);
@@ -40,8 +40,46 @@ const getScores = (result) => Object.entries(result.lhr.categories).reduce((prev
   prev[key] = Math.floor(c.score * 100);
   return prev;
 }, {});
+const writeScoresToJson = async (lhScoresDir, name, scores, result) => {
+  const json = Object.entries(scores).reduce((prev, [k, score]) => {
+    prev[k] = { score };
+    return prev;
+  }, {});
+  const accessibilityViolations = result.artifacts.Accessibility.violations.map((v) => {
+    return {
+      title: result.lhr.audits[v.id].title,
+      nodes: v.nodes.length
+    };
+  });
+  if (accessibilityViolations.length > 0) {
+    json.accessibility.issues = accessibilityViolations;
+  }
+  await fse.writeFile(path.join(lhScoresDir, `${name}.json`), JSON.stringify(json, null, 2));
+};
+const buildAverageCsv = async (lhScoresDir) => {
+  const files = await fse.readdir(lhScoresDir);
+  const jsonFiles = files.filter((f) => f.endsWith(".json"));
+  const scores = {};
+  for (const fileName of jsonFiles) {
+    const score = await fse.readJson(
+      path.join(lhScoresDir, fileName)
+    );
+    Object.entries(score).forEach(([k, v]) => {
+      if (!scores[k]) {
+        scores[k] = 0;
+      }
+      scores[k] += v.score;
+    });
+  }
+  Object.entries(scores).forEach(([k, v]) => {
+    scores[k] = v / jsonFiles.length;
+  });
+  await writeCsvResult(lhScoresDir, "_AVERAGE_", scores);
+};
 export {
+  buildAverageCsv,
   getScores,
   writeCsvResult,
-  writeHtmlListEntryWithRetry
+  writeHtmlListEntryWithRetry,
+  writeScoresToJson
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lighthouse-reporting",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "type": "module",
   "scripts": {
     "build": "vite build && vite build -c vite.umd.config.ts && tsc -p ./tsconfig.build.json",
@@ -24,25 +24,25 @@
   },
   "optionalDependencies": {
     "@playwright/test": "^1",
-    "get-port": "^6"
+    "get-port": ">=7"
   },
   "devDependencies": {
     "@playwright/test": "^1.34.3",
     "@types/fs-extra": "^11.0.1",
-    "@types/node": "^20.2.4",
-    "@typescript-eslint/eslint-plugin": "^5.59.7",
-    "@typescript-eslint/parser": "^5.59.7",
-    "eslint": "^8.41.0",
+    "@types/node": "^20.3.1",
+    "@typescript-eslint/eslint-plugin": "^5.60.0",
+    "@typescript-eslint/parser": "^5.60.0",
+    "eslint": "^8.43.0",
     "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-prettier": "^4.2.1",
-    "get-port": "^6.1.2",
+    "get-port": "^7.0.0",
     "husky": "^8.0.3",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.8.8",
-    "semantic-release": "^21.0.2",
-    "typescript": "^5.0.4",
-    "vite": "^4.3.8",
-    "vite-plugin-static-copy": "^0.15.0"
+    "semantic-release": "^21.0.5",
+    "typescript": "^5.1.3",
+    "vite": "^4.3.9",
+    "vite-plugin-static-copy": "^0.16.0"
   },
   "dependencies": {
     "fs-extra": "^11.1.1"