lighthouse-reporting 1.3.1 → 1.5.0

package/README.md

@@ -92,12 +92,16 @@ import {
     LighthouseResult,
     StorybookIndexStory,
     storybookPlaywright,
+    writeScoresToJson
 } from 'lighthouse-reporting'
 
 playwrightLighthouseTest.setTimeout(60000)
-const reportDir = path.join(process.cwd(), process.env.LH_REPORT_DIR as string)
+const lhScoresDir = path.join(process.cwd(), process.env.LH_SCORES_DIR || 'lh-scores')
+const reportDir = path.join(process.cwd(), process.env.LH_REPORT_DIR || 'lighthouse')
 const htmlFilePath = path.join(reportDir, 'index.html')
 
+// use stories.json instead of index.json for storybook v6
+// make sure to set buildStoriesJson to true in storybook main.js feature section
 const stories = storybookPlaywright.getStories('./storybook-static/index.json', (story) => {
     // skip docs, etc
     if (story.type !== 'story') {
@@ -147,6 +151,8 @@ const runLighthouse = async (story: StorybookIndexStory, context: BrowserContext
     const scores = getScores(result)
     await writeCsvResult(reportDir, name, scores, thresholds)
     await writeHtmlListEntryWithRetry(htmlFilePath, name, scores, thresholds, result.comparisonError)
+    // write score results in JSON, allows generating the Average csv report
+    await writeScoresToJson(lhScoresDir, name, scores, result)
 }
 ```
 
@@ -162,6 +168,8 @@ import { PlaywrightTestConfig } from '@playwright/test'
 
 const baseURL = 'http://127.0.0.1:6009'
 // process.env.LH_REPORT_DIR = 'lighthouse-storybook' // adjust lighthouse output folder if required
+// process.env.LH_SCORES_DIR = 'lh-scores' // to write and store scores in json format or write average report
+
 
 const config: PlaywrightTestConfig = {
     use: {
@@ -209,49 +217,13 @@ export default config
   <summary>global-setup.ts</summary>
 
 ```ts
-import { PlaywrightTestConfig } from '@playwright/test'
-
-const baseURL = 'http://127.0.0.1:6009'
-// process.env.LH_REPORT_DIR = 'lighthouse-storybook' // adjust lighthouse output folder if required
+import { lighthouseSetup } from 'lighthouse-reporting'
 
-const config: PlaywrightTestConfig = {
-    use: {
-        viewport: { width: 1280, height: 820 },
-        ignoreHTTPSErrors: true,
-        acceptDownloads: false,
-        trace: 'off',
-        baseURL,
-        screenshot: { mode: 'off' },
-    },
-    projects: [
-        {
-            name: 'chromium',
-            use: {
-                browserName: 'chromium',
-                launchOptions: { args: ['--disable-gpu'] },
-            },
-            retries: 0,
-        },
-    ],
-    expect: { toMatchSnapshot: { threshold: 0.2 } },
-    reporter: 'line',
-    testDir: 'test/storybook',
-    testMatch: '*.spec.ts',
-    fullyParallel: true,
-    globalSetup: './src/global-setup.ts',
-    globalTeardown: './src/global-teardown.ts',
-    forbidOnly: true,
-    webServer: [
-        {
-            command: 'npx http-server ./storybook-static --port 6009 --silent',
-            url: `${baseURL}/index.json`,
-            timeout: 15 * 1000,
-            reuseExistingServer: false,
-            ignoreHTTPSErrors: true,
-        },
-    ],
+async function globalSetup() {
+    await lighthouseSetup()
 }
-export default config
+
+export default globalSetup
 ```
 
 </details>
@@ -260,10 +232,14 @@ export default config
   <summary>global-teardown.ts</summary>
 
 ```ts
-import { lighthousePlaywrightTeardown } from 'lighthouse-reporting'
+import { lighthousePlaywrightTeardown, buildAverageCsv } from 'lighthouse-reporting'
+
+const lhScoresDir = path.join(process.cwd(), process.env.LH_SCORES_DIR || 'lh-scores')
+const reportDir = path.join(process.cwd(), 'lighthouse')
 
 async function globalTeardown() {
     await lighthousePlaywrightTeardown()
+    await buildAverageCsv(lhScoresDir, reportDir)
 }
 
 export default globalTeardown

package/dist/index.cjs

@@ -13,8 +13,10 @@ require("get-port");
 require("@playwright/test");
 exports.lighthousePlaywrightTeardown = hooks.lighthousePlaywrightTeardown;
 exports.lighthouseSetup = hooks.lighthouseSetup;
+exports.buildAverageCsv = lighthouseReports.buildAverageCsv;
 exports.getScores = lighthouseReports.getScores;
 exports.writeCsvResult = lighthouseReports.writeCsvResult;
 exports.writeHtmlListEntryWithRetry = lighthouseReports.writeHtmlListEntryWithRetry;
+exports.writeScoresToJson = lighthouseReports.writeScoresToJson;
 exports.playwrightLighthouseTest = playwrightLighthouseTest.playwrightLighthouseTest;
 exports.storybookPlaywright = storybookPlaywright.storybookPlaywright;

package/dist/index.js

@@ -1,5 +1,5 @@
 import { lighthousePlaywrightTeardown, lighthouseSetup } from "./hooks.js";
-import { getScores, writeCsvResult, writeHtmlListEntryWithRetry } from "./lighthouseReports.js";
+import { buildAverageCsv, getScores, writeCsvResult, writeHtmlListEntryWithRetry, writeScoresToJson } from "./lighthouseReports.js";
 import { playwrightLighthouseTest } from "./playwrightLighthouseTest.js";
 import { storybookPlaywright } from "./storybookPlaywright.js";
 import "os";
@@ -10,11 +10,13 @@ import "./constants-226e9774.js";
 import "get-port";
 import "@playwright/test";
 export {
+  buildAverageCsv,
   getScores,
   lighthousePlaywrightTeardown,
   lighthouseSetup,
   playwrightLighthouseTest,
   storybookPlaywright,
   writeCsvResult,
-  writeHtmlListEntryWithRetry
+  writeHtmlListEntryWithRetry,
+  writeScoresToJson
 };

package/dist/lighthouse-reporting.umd.cjs

@@ -21,14 +21,14 @@
   async function lighthousePlaywrightTeardown() {
     await fse.remove(path.join(os.tmpdir(), PW_TMP_DIR));
   }
-  const writeCsvResult = async (reportDir2, name, scores, thresholds, swimlanes = []) => {
+  const writeCsvResult = async (reportDir2, name, scores, thresholds = {}, swimlanes = []) => {
     let csvData = Object.keys(scores).join(",");
     if (swimlanes.length > 0) {
       csvData += "," + swimlanes.map((swimlane) => `${swimlane}_threshold`);
     }
     csvData += "\n";
     csvData += Object.values(scores).join(",");
-    if (swimlanes.length > 0) {
+    if (swimlanes.length > 0 && Object.keys(thresholds).length > 0) {
       csvData += "," + swimlanes.map((swimlane) => thresholds[swimlane]);
     }
     await fse.writeFile(path.join(reportDir2, `${name}.csv`), csvData);
@@ -61,6 +61,45 @@
     prev[key] = Math.floor(c.score * 100);
     return prev;
   }, {});
+  const writeScoresToJson = async (lhScoresDir, name, scores, result) => {
+    const json = Object.entries(scores).reduce((prev, [k, score]) => {
+      prev[k] = { score };
+      return prev;
+    }, {});
+    const accessibilityViolations = result.artifacts.Accessibility.violations.map((v) => {
+      return {
+        title: result.lhr.audits[v.id].title,
+        nodes: v.nodes.length
+      };
+    });
+    if (accessibilityViolations.length > 0) {
+      json.accessibility.issues = accessibilityViolations;
+    }
+    await fse.writeFile(path.join(lhScoresDir, `${name}.json`), JSON.stringify(json, null, 2));
+  };
+  const buildAverageCsv = async (lhScoresDir, reportDir2) => {
+    const files = await fse.readdir(lhScoresDir);
+    const jsonFiles = files.filter((f) => f.endsWith(".json"));
+    if (jsonFiles.length === 0) {
+      return;
+    }
+    const scores = {};
+    for (const fileName of jsonFiles) {
+      const score = await fse.readJson(
+        path.join(lhScoresDir, fileName)
+      );
+      Object.entries(score).forEach(([k, v]) => {
+        if (!scores[k]) {
+          scores[k] = 0;
+        }
+        scores[k] += v.score;
+      });
+    }
+    Object.entries(scores).forEach(([k, v]) => {
+      scores[k] = v / jsonFiles.length;
+    });
+    await writeCsvResult(reportDir2, "_AVERAGE_", scores);
+  };
   class Locked extends Error {
     constructor(port) {
       super(`${port} is locked`);
@@ -71,7 +110,7 @@
     young: /* @__PURE__ */ new Set()
   };
   const releaseOldLockedPortsIntervalMs = 1e3 * 15;
-  let interval;
+  let timeout;
   const getLocalHosts = () => {
     const interfaces = os.networkInterfaces();
     const results = /* @__PURE__ */ new Set([void 0, "0.0.0.0"]);
@@ -137,13 +176,14 @@
         exclude = new Set(excludeIterable);
       }
     }
-    if (interval === void 0) {
-      interval = setInterval(() => {
+    if (timeout === void 0) {
+      timeout = setTimeout(() => {
+        timeout = void 0;
         lockedPorts.old = lockedPorts.young;
         lockedPorts.young = /* @__PURE__ */ new Set();
       }, releaseOldLockedPortsIntervalMs);
-      if (interval.unref) {
-        interval.unref();
+      if (timeout.unref) {
+        timeout.unref();
       }
     }
     const hosts = getLocalHosts();
@@ -199,7 +239,8 @@
         throw new Error("Please build storybook before running tests!");
       }
       const storybookIndexJson = fse.readJsonSync(pathToStorybookIndexJson);
-      const stories = Object.values(storybookIndexJson.entries).filter(storyFilterFn);
+      const storyObject = storybookIndexJson.entries || storybookIndexJson.stories;
+      const stories = Object.values(storyObject).filter(storyFilterFn);
       return stories;
     },
     captureScreenshot: async (story, context, screenshotOptions, actionBeforeScreenshot) => {
@@ -214,6 +255,7 @@
       await test.expect(page).toHaveScreenshot(`${story.id}.png`, screenshotOptions);
     }
   };
+  exports2.buildAverageCsv = buildAverageCsv;
   exports2.getScores = getScores;
   exports2.lighthousePlaywrightTeardown = lighthousePlaywrightTeardown;
   exports2.lighthouseSetup = lighthouseSetup;
@@ -221,5 +263,6 @@
   exports2.storybookPlaywright = storybookPlaywright;
   exports2.writeCsvResult = writeCsvResult;
   exports2.writeHtmlListEntryWithRetry = writeHtmlListEntryWithRetry;
+  exports2.writeScoresToJson = writeScoresToJson;
   Object.defineProperty(exports2, Symbol.toStringTag, { value: "Module" });
 });

package/dist/lighthouseReports.cjs

@@ -2,14 +2,14 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const path = require("path");
 const fse = require("fs-extra");
-const writeCsvResult = async (reportDir, name, scores, thresholds, swimlanes = []) => {
+const writeCsvResult = async (reportDir, name, scores, thresholds = {}, swimlanes = []) => {
   let csvData = Object.keys(scores).join(",");
   if (swimlanes.length > 0) {
     csvData += "," + swimlanes.map((swimlane) => `${swimlane}_threshold`);
   }
   csvData += "\n";
   csvData += Object.values(scores).join(",");
-  if (swimlanes.length > 0) {
+  if (swimlanes.length > 0 && Object.keys(thresholds).length > 0) {
     csvData += "," + swimlanes.map((swimlane) => thresholds[swimlane]);
   }
   await fse.writeFile(path.join(reportDir, `${name}.csv`), csvData);
@@ -42,6 +42,47 @@ const getScores = (result) => Object.entries(result.lhr.categories).reduce((prev
   prev[key] = Math.floor(c.score * 100);
   return prev;
 }, {});
+const writeScoresToJson = async (lhScoresDir, name, scores, result) => {
+  const json = Object.entries(scores).reduce((prev, [k, score]) => {
+    prev[k] = { score };
+    return prev;
+  }, {});
+  const accessibilityViolations = result.artifacts.Accessibility.violations.map((v) => {
+    return {
+      title: result.lhr.audits[v.id].title,
+      nodes: v.nodes.length
+    };
+  });
+  if (accessibilityViolations.length > 0) {
+    json.accessibility.issues = accessibilityViolations;
+  }
+  await fse.writeFile(path.join(lhScoresDir, `${name}.json`), JSON.stringify(json, null, 2));
+};
+const buildAverageCsv = async (lhScoresDir, reportDir) => {
+  const files = await fse.readdir(lhScoresDir);
+  const jsonFiles = files.filter((f) => f.endsWith(".json"));
+  if (jsonFiles.length === 0) {
+    return;
+  }
+  const scores = {};
+  for (const fileName of jsonFiles) {
+    const score = await fse.readJson(
+      path.join(lhScoresDir, fileName)
+    );
+    Object.entries(score).forEach(([k, v]) => {
+      if (!scores[k]) {
+        scores[k] = 0;
+      }
+      scores[k] += v.score;
+    });
+  }
+  Object.entries(scores).forEach(([k, v]) => {
+    scores[k] = v / jsonFiles.length;
+  });
+  await writeCsvResult(reportDir, "_AVERAGE_", scores);
+};
+exports.buildAverageCsv = buildAverageCsv;
 exports.getScores = getScores;
 exports.writeCsvResult = writeCsvResult;
 exports.writeHtmlListEntryWithRetry = writeHtmlListEntryWithRetry;
+exports.writeScoresToJson = writeScoresToJson;

package/dist/lighthouseReports.d.ts

@@ -1,3 +1,29 @@
+interface NodeDetails {
+    lhId: string;
+    devtoolsNodePath: string;
+    selector: string;
+    boudingRect: {
+        [k: string]: number;
+    };
+    snippet: string;
+    nodeLabel: string;
+}
+interface RuleExecutionError {
+    name: string;
+    message: string;
+}
+interface AxeRuleResult {
+    id: string;
+    impact?: string;
+    tags: Array<string>;
+    nodes: Array<{
+        target: Array<string>;
+        failureSummary?: string;
+        node: NodeDetails;
+        relatedNodes: NodeDetails[];
+    }>;
+    error?: RuleExecutionError;
+}
 export interface LighthouseResult {
     lhr: {
         categories: {
@@ -5,12 +31,30 @@ export interface LighthouseResult {
                 score: number;
             };
         };
+        audits: {
+            [k: string]: {
+                title: string;
+            };
+        };
+    };
+    artifacts: {
+        Accessibility: {
+            violations: Array<AxeRuleResult>;
+        };
     };
     comparisonError?: string;
 }
-export declare const writeCsvResult: (reportDir: string, name: string, scores: Record<string, number>, thresholds: Record<string, number>, swimlanes?: Array<string>) => Promise<void>;
+export declare const writeCsvResult: (reportDir: string, name: string, scores: Record<string, number>, thresholds?: Record<string, number>, swimlanes?: Array<string>) => Promise<void>;
 /**
  * workaround conflicts when multiple threads write the same file
  */
 export declare const writeHtmlListEntryWithRetry: (htmlFilePath: string, name: string, scores: Record<string, number>, thresholds: Record<string, number>, comparisonError?: string, addReportLink?: boolean) => Promise<void>;
 export declare const getScores: (result: LighthouseResult) => Record<string, number>;
+export declare const writeScoresToJson: (lhScoresDir: string, name: string, scores: Record<string, number>, result: LighthouseResult) => Promise<void>;
+/**
+ * Generate average csv file. Make sure to use `writeScoresToJson` in your test!
+ * @param lhScoresDir path to folder with lighthouse json files. See `writeScoresToJson`.
+ * @param reportDir folder where `_AVERAGE_.json` will be generated
+ */
+export declare const buildAverageCsv: (lhScoresDir: string, reportDir: string) => Promise<void>;
+export {};

package/dist/lighthouseReports.js

@@ -1,13 +1,13 @@
 import path from "path";
 import fse from "fs-extra";
-const writeCsvResult = async (reportDir, name, scores, thresholds, swimlanes = []) => {
+const writeCsvResult = async (reportDir, name, scores, thresholds = {}, swimlanes = []) => {
   let csvData = Object.keys(scores).join(",");
   if (swimlanes.length > 0) {
     csvData += "," + swimlanes.map((swimlane) => `${swimlane}_threshold`);
   }
   csvData += "\n";
   csvData += Object.values(scores).join(",");
-  if (swimlanes.length > 0) {
+  if (swimlanes.length > 0 && Object.keys(thresholds).length > 0) {
     csvData += "," + swimlanes.map((swimlane) => thresholds[swimlane]);
   }
   await fse.writeFile(path.join(reportDir, `${name}.csv`), csvData);
@@ -40,8 +40,49 @@ const getScores = (result) => Object.entries(result.lhr.categories).reduce((prev
   prev[key] = Math.floor(c.score * 100);
   return prev;
 }, {});
+const writeScoresToJson = async (lhScoresDir, name, scores, result) => {
+  const json = Object.entries(scores).reduce((prev, [k, score]) => {
+    prev[k] = { score };
+    return prev;
+  }, {});
+  const accessibilityViolations = result.artifacts.Accessibility.violations.map((v) => {
+    return {
+      title: result.lhr.audits[v.id].title,
+      nodes: v.nodes.length
+    };
+  });
+  if (accessibilityViolations.length > 0) {
+    json.accessibility.issues = accessibilityViolations;
+  }
+  await fse.writeFile(path.join(lhScoresDir, `${name}.json`), JSON.stringify(json, null, 2));
+};
+const buildAverageCsv = async (lhScoresDir, reportDir) => {
+  const files = await fse.readdir(lhScoresDir);
+  const jsonFiles = files.filter((f) => f.endsWith(".json"));
+  if (jsonFiles.length === 0) {
+    return;
+  }
+  const scores = {};
+  for (const fileName of jsonFiles) {
+    const score = await fse.readJson(
+      path.join(lhScoresDir, fileName)
+    );
+    Object.entries(score).forEach(([k, v]) => {
+      if (!scores[k]) {
+        scores[k] = 0;
+      }
+      scores[k] += v.score;
+    });
+  }
+  Object.entries(scores).forEach(([k, v]) => {
+    scores[k] = v / jsonFiles.length;
+  });
+  await writeCsvResult(reportDir, "_AVERAGE_", scores);
+};
 export {
+  buildAverageCsv,
   getScores,
   writeCsvResult,
-  writeHtmlListEntryWithRetry
+  writeHtmlListEntryWithRetry,
+  writeScoresToJson
 };

package/dist/storybookPlaywright.cjs

@@ -9,7 +9,8 @@ const storybookPlaywright = {
       throw new Error("Please build storybook before running tests!");
     }
     const storybookIndexJson = fse.readJsonSync(pathToStorybookIndexJson);
-    const stories = Object.values(storybookIndexJson.entries).filter(storyFilterFn);
+    const storyObject = storybookIndexJson.entries || storybookIndexJson.stories;
+    const stories = Object.values(storyObject).filter(storyFilterFn);
     return stories;
   },
   captureScreenshot: async (story, context, screenshotOptions, actionBeforeScreenshot) => {

package/dist/storybookPlaywright.d.ts

@@ -1,4 +1,7 @@
 import { BrowserContext, Locator, Page } from '@playwright/test';
+/**
+ * Storybook v7
+ */
 export interface StorybookIndexStory {
     id: string;
     title: string;
@@ -7,7 +10,23 @@ export interface StorybookIndexStory {
     tags: Array<string>;
     type: 'story' | 'docs';
 }
-type StoriesFilterFn = (story: StorybookIndexStory) => boolean;
+/**
+ * Storybook v6
+ */
+export interface StorybookStoriesStory {
+    id: string;
+    title: string;
+    name: string;
+    importPath: string;
+    kind: string;
+    story: string;
+    parameters: {
+        __id: string;
+        docsOnly: boolean;
+        fileName: string;
+    };
+}
+type StoriesFilterFn = <V7 = true>(story: V7 extends true ? StorybookIndexStory : StorybookStoriesStory) => boolean;
 export declare const storybookPlaywright: {
     getStories: (pathToStorybookIndexJson: string, storyFilterFn: StoriesFilterFn) => StorybookIndexStory[];
     captureScreenshot: (story: StorybookIndexStory, context: BrowserContext, screenshotOptions?: {

package/dist/storybookPlaywright.js

@@ -7,7 +7,8 @@ const storybookPlaywright = {
       throw new Error("Please build storybook before running tests!");
     }
     const storybookIndexJson = fse.readJsonSync(pathToStorybookIndexJson);
-    const stories = Object.values(storybookIndexJson.entries).filter(storyFilterFn);
+    const storyObject = storybookIndexJson.entries || storybookIndexJson.stories;
+    const stories = Object.values(storyObject).filter(storyFilterFn);
     return stories;
   },
   captureScreenshot: async (story, context, screenshotOptions, actionBeforeScreenshot) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lighthouse-reporting",
-  "version": "1.3.1",
+  "version": "1.5.0",
   "type": "module",
   "scripts": {
     "build": "vite build && vite build -c vite.umd.config.ts && tsc -p ./tsconfig.build.json",
@@ -24,25 +24,25 @@
   },
   "optionalDependencies": {
     "@playwright/test": "^1",
-    "get-port": "^6"
+    "get-port": ">=7"
   },
   "devDependencies": {
     "@playwright/test": "^1.34.3",
     "@types/fs-extra": "^11.0.1",
-    "@types/node": "^20.2.4",
-    "@typescript-eslint/eslint-plugin": "^5.59.7",
-    "@typescript-eslint/parser": "^5.59.7",
-    "eslint": "^8.41.0",
+    "@types/node": "^20.3.1",
+    "@typescript-eslint/eslint-plugin": "^5.60.0",
+    "@typescript-eslint/parser": "^5.60.0",
+    "eslint": "^8.43.0",
     "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-prettier": "^4.2.1",
-    "get-port": "^6.1.2",
+    "get-port": "^7.0.0",
     "husky": "^8.0.3",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.8.8",
-    "semantic-release": "^21.0.2",
-    "typescript": "^5.0.4",
+    "semantic-release": "^21.0.5",
+    "typescript": "^5.1.3",
     "vite": "^4.3.9",
-    "vite-plugin-static-copy": "^0.15.0"
+    "vite-plugin-static-copy": "^0.16.0"
   },
   "dependencies": {
     "fs-extra": "^11.1.1"