pagean 8.0.4 → 10.0.0

package/lib/sitemap.js

@@ -0,0 +1,120 @@
+'use strict';
+
+const fs = require('node:fs');
+const axios = require('axios');
+const { parseStringPromise } = require('xml2js');
+
+/**
+ * Gets a sitemap, via file path or URL, and returns the string contents.
+ *
+ * @param   {string} url The URL or file path to the sitemap.
+ * @returns {string}     The string contents of the sitemap.
+ * @async
+ * @throws  {Error}      Throws if the sitemap cannot be retrieved.
+ * @static
+ * @private
+ */
+const getSitemap = async (url) => {
+    try {
+        if (url.startsWith('http')) {
+            const response = await axios.get(url);
+            return response.data;
+        }
+        // Allow users to specify a local sitemap filename.
+        // nosemgrep: eslint.detect-non-literal-fs-filename
+        return fs.readFileSync(url, 'utf8');
+    } catch {
+        throw new Error(`Error retrieving sitemap "${url}"`);
+    }
+};
+
+/**
+ * Parses a sitemap string and returns an array of URLs.
+ *
+ * @param   {string}   sitemapXml The string contents of the sitemap.
+ * @returns {string[]}            The URLs from the sitemap.
+ * @async
+ * @static
+ * @private
+ */
+const parseSitemap = async (sitemapXml) => {
+    const sitemapData = await parseStringPromise(sitemapXml);
+    if (!sitemapData.urlset.url) {
+        return [];
+    }
+    return sitemapData.urlset.url.map((url) => url.loc[0]);
+};
+
+/**
+ * Removes excluded URLs from the list of URLs.
+ *
+ * @param   {string[]} urls    The URLs from the sitemap.
+ * @param   {string[]} exclude The URLs to exclude (regular expressions).
+ * @returns {string[]}         The non-excluded URLs from the sitemap.
+ * @static
+ * @private
+ */
+const removeExcludedUrls = (urls, exclude = []) => {
+    // Allow URLs to be excluded by regular expression.
+    // nosemgrep: eslint.detect-non-literal-regexp
+    const excludeRegex = exclude.map((url) => new RegExp(url));
+    return urls.filter((url) => {
+        for (const excludeUrlRegex of excludeRegex) {
+            if (excludeUrlRegex.test(url)) {
+                return false;
+            }
+        }
+        return true;
+    });
+};
+
+/**
+ * Replaces the "find" string with the "replace" string in each of the URLs.
+ *
+ * @param   {string[]} urls      The URLs to process.
+ * @param   {string}   [find]    The string to find in the URLs.
+ * @param   {string}   [replace] The string to replace in the URLs.
+ * @returns {string[]}           The processed URLs.
+ * @async
+ * @static
+ * @private
+ */
+const replaceUrlStrings = (urls, find, replace) => {
+    if (!find || !replace) {
+        return urls;
+    }
+    return urls.map((url) => url.replace(find, replace));
+};
+
+/**
+ * Retrieves a list of URLs from a sitemap, via a local file path or URL.
+ * Options are available to exclude URLs from the results, and to find and
+ * replace strings in the URLs (primarily intended for isolated testing,
+ * for example change "https://somewhere.test/" to "http://localhost:3000/").
+ *
+ * @param   {object}   options           The sitemap processing options.
+ * @param   {string}   options.url       The URL or file path to the sitemap.
+ * @param   {string}   [options.find]    The string to find in the URLs.
+ * @param   {string}   [options.replace] The string to replace in the URLs.
+ * @param   {string[]} [options.exclude] The URLs to exclude (regular expressions).
+ * @returns {string[]}                   The processed URLs from the sitemap.
+ * @async
+ * @throws  {Error}                      Throws if the sitemap cannot be processed.
+ * @static
+ * @public
+ */
+const getUrlsFromSitemap = async ({ url, find, replace, exclude } = {}) => {
+    const sitemapXml = await getSitemap(url);
+    try {
+        const rawUrls = await parseSitemap(sitemapXml);
+        return replaceUrlStrings(
+            removeExcludedUrls(rawUrls, exclude),
+            find,
+            replace
+        );
+    } catch {
+        throw new Error(`Error processing sitemap "${url}"`);
+    }
+};
+
+module.exports.getUrlsFromSitemap = getUrlsFromSitemap;

package/lib/test-utils.js

@@ -7,9 +7,9 @@
  */
 
 const testResultStates = Object.freeze({
+    failed: 'failed',
     passed: 'passed',
-    warning: 'warning',
-    failed: 'failed'
+    warning: 'warning'
 });
 
 const getTestSettings = (testSettingProperty, urlSettings) => {
@@ -26,7 +26,7 @@ const getTestSettings = (testSettingProperty, urlSettings) => {
  * test-specific settings from configuration, passing the test context,
  * and logging results.
  *
- * @param {string}   name                The name of the test.
+ * @param {string}   name                The name of the test being run.
  * @param {Function} testFunction        The test function to be executed.
  * @param {object}   testContext         The test execution context.
  * @param {string}   testSettingProperty The name of the config property
@@ -50,7 +50,7 @@ const pageanTest = async (
         try {
             results = await testFunction(testContext);
         } catch (error) {
-            results = { result: testResultStates.failed, data: { error } };
+            results = { data: { error }, result: testResultStates.failed };
         }
         if (
             results.result === testResultStates.failed &&

package/lib/tests.js

@@ -27,7 +27,10 @@ const horizontalScrollbarTest = async (context) => {
             // istanbul ignore next: injects script causing puppeteer error, see #48
             const scrollbar = await context.page.evaluate(() => {
                 document.scrollingElement.scrollLeft = 1;
-                return document.scrollingElement.scrollLeft === 1;
+                // The scrollLeft value may not end up exactly 1. But, any
+                // value greater than 0 indicates the presence of a horizontal
+                // scrollbar.
+                return document.scrollingElement.scrollLeft > 0;
             });
             return {
                 result:
@@ -136,7 +139,7 @@ const renderedHtmlTest = async (context) => {
  * @param {object} context Test execution context.
  * @static
  */
-// eslint-disable-next-line max-lines-per-function
+// eslint-disable-next-line max-lines-per-function -- exceeded due to formatting
 const pageLoadTimeTest = async (context) => {
     const testSettingName = 'pageLoadTimeTest';
     await pageanTest(
@@ -179,18 +182,16 @@ const pageLoadTimeTest = async (context) => {
  * @param {object} context Test execution context.
  * @static
  */
-// eslint-disable-next-line max-lines-per-function
+// eslint-disable-next-line max-lines-per-function -- exceeded due to formatting
 const externalScriptTest = async (context) => {
     await pageanTest(
         'should not have external scripts',
         // eslint-disable-next-line no-shadow -- less intuitive
         async (context) => {
             // istanbul ignore next: injects script causing puppeteer error, see #48
-            const scripts = await context.page.evaluate(() => {
-                return [...document.querySelectorAll('script[src]')].map(
-                    (s) => s.src
-                );
-            });
+            const scripts = await context.page.evaluate(() =>
+                [...document.querySelectorAll('script[src]')].map((s) => s.src)
+            );
 
             const pageUrl = context.page.url();
             const externalScripts = scripts.filter((script) =>
@@ -223,23 +224,21 @@ const externalScriptTest = async (context) => {
  * @param {object} context Test execution context.
  * @static
  */
-// eslint-disable-next-line max-lines-per-function
+// eslint-disable-next-line max-lines-per-function -- exceeded due to formatting
 const brokenLinkTest = async (context) => {
     await pageanTest(
         'should not have broken links',
         // eslint-disable-next-line no-shadow -- less intuitive
         async (context) => {
             // istanbul ignore next: injects script causing puppeteer error, see #48
-            const links = await context.page.evaluate(() => {
-                return [...document.querySelectorAll('a[href]')].map(
-                    (a) => a.href
-                );
-            });
+            const links = await context.page.evaluate(() =>
+                [...document.querySelectorAll('a[href]')].map((a) => a.href)
+            );
 
             // All links are returned from puppeteer as absolute links, so this filters out
             // javascript and other values and leaves only pages to request.
             const httpLinks = links.filter((link) =>
-                link.match(/(http(s?)|file):\/\//)
+                link.match(/(?<protocol>https?|file):\/\//)
             );
             // Reduce to unique page links so only checked once
             const uniqueHttpLinks = [...new Set(httpLinks)];

package/package.json

@@ -1,24 +1,24 @@
 {
   "name": "pagean",
-  "version": "8.0.4",
+  "version": "10.0.0",
   "description": "Pagean is a web page analysis tool designed to automate tests requiring web pages to be loaded in a browser window (e.g. horizontal scrollbar, console errors)",
   "bin": {
     "pagean": "./bin/pagean.js",
     "pageanrc-lint": "./bin/pageanrc-lint.js"
   },
   "scripts": {
-    "hooks-pre-commit": "npm run lint && npm run prettier-check",
-    "hooks-pre-push": "npm audit --audit-level=high && npm test",
-    "lint": "npm run lint-css && npm run lint-html && npm run lint-js && npm run lint-md",
-    "lint-css": "stylelint ./lib/report-template.handlebars",
-    "lint-html": "htmlhint ./lib/report-template.handlebars",
-    "lint-js": "eslint \"**/*.js\"",
-    "lint-md": "markdownlint **/*.md --ignore node_modules",
-    "prettier-check": "prettier --check .",
-    "prettier-fix": "prettier --write .",
+    "hooks:pre-commit": "npm run lint && npm run prettier:check",
+    "hooks:pre-push": "npm audit --audit-level=high && npm test",
+    "lint": "npm run lint:css && npm run lint:html && npm run lint:js && npm run lint:md",
+    "lint:css": "stylelint ./lib/report-template.handlebars",
+    "lint:html": "htmlhint ./lib/report-template.handlebars",
+    "lint:js": "eslint .",
+    "lint:md": "markdownlint-cli2 \"**/*.md\" \"#node_modules\" \"#Archive\"",
+    "prettier:check": "prettier --check .",
+    "prettier:fix": "prettier --write .",
     "start": "node ./bin/pagean.js",
-    "start-lint": "node ./bin/pageanrc-lint.js",
-    "start-lint-all": "npm run start-lint && npm run start-lint static-server.pageanrc.json && npm run start-lint ./lib/default-config.json",
+    "start:lint": "node ./bin/pageanrc-lint.js",
+    "start:lint-all": "npm run start:lint && npm run start:lint static-server.pageanrc.json && npm run start:lint ./lib/default-config.json",
     "test": "jest --ci --config jest.config.json"
   },
   "repository": {
@@ -37,7 +37,7 @@
   "author": "Aaron Goldenthal <npm@aarongoldenthal.com>",
   "license": "MIT",
   "engines": {
-    "node": "^14.15.0 || ^16.13.0 || >=18.0.0"
+    "node": "^18.12.0 || >=20.0.0"
   },
   "files": [
     "index.js",
@@ -51,29 +51,30 @@
   },
   "homepage": "https://gitlab.com/gitlab-ci-utils/pagean",
   "devDependencies": {
-    "@aarongoldenthal/eslint-config-standard": "^20.0.0",
-    "@aarongoldenthal/stylelint-config-standard": "^13.0.1",
-    "bin-tester": "^3.0.1",
-    "eslint": "^8.34.0",
-    "jest": "^29.4.3",
-    "jest-junit": "^15.0.0",
-    "markdownlint-cli": "^0.33.0",
-    "prettier": "^2.8.4",
+    "@aarongoldenthal/eslint-config-standard": "^25.0.0",
+    "@aarongoldenthal/stylelint-config-standard": "^17.0.0",
+    "bin-tester": "^5.0.0",
+    "eslint": "^8.56.0",
+    "jest": "^29.7.0",
+    "jest-junit": "^16.0.0",
+    "markdownlint-cli2": "^0.12.1",
+    "prettier": "^3.2.4",
     "strip-ansi": "^6.0.1",
-    "stylelint": "^15.2.0"
+    "stylelint": "^16.2.0"
   },
   "dependencies": {
     "ajv": "^8.12.0",
     "ajv-errors": "^3.0.0",
-    "axios": "^1.3.3",
-    "ci-logger": "^5.1.1",
-    "commander": "^10.0.0",
+    "axios": "^1.6.7",
+    "ci-logger": "^6.0.0",
+    "commander": "^11.1.0",
     "cssesc": "^3.0.0",
-    "handlebars": "^4.7.7",
+    "handlebars": "^4.7.8",
     "htmlhint": "^1.1.4",
     "kleur": "^4.1.5",
     "normalize-url": "^6.1.0",
     "protocolify": "^3.0.0",
-    "puppeteer": "^19.7.1"
+    "puppeteer": "^21.9.0",
+    "xml2js": "^0.6.2"
   }
 }

package/schemas/pageanrc.schema.json

@@ -30,6 +30,48 @@
       "description": "The global test settings to apply to all tests, unless overridden for a specific URL.",
       "$ref": "#/definitions/allTestSettings"
     },
+    "sitemap": {
+      "description": "The sitemap to test",
+      "type": "object",
+      "properties": {
+        "url": {
+          "description": "The URL of the sitemap to test",
+          "type": "string"
+        },
+        "find": {
+          "description": "The string or regex pattern to find in sitemap URLs",
+          "type": "string"
+        },
+        "replace": {
+          "description": "The replacement string in sitemap URLs",
+          "type": "string"
+        },
+        "exclude": {
+          "description": "An array of strings or regex patterns to exclude from the sitemap",
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "uniqueItems": true,
+          "minItems": 1,
+          "maxItems": 100
+        }
+      },
+      "allOf": [
+        { "required": ["url"], "errorMessage": "must provide 'url'" },
+        {
+          "dependentRequired": {
+            "find": ["replace"],
+            "replace": ["find"]
+          },
+          "errorMessage": "must use 'find' and 'replace' together"
+        }
+      ],
+      "additionalProperties": {
+        "not": true,
+        "errorMessage": "must NOT contain additional properties: ${0#}"
+      }
+    },
     "urls": {
       "description": "The collection of URLs to test, with any URL-specific test settings.",
       "type": "array",
@@ -71,7 +113,16 @@
     "not": true,
     "errorMessage": "must NOT contain additional properties: ${0#}"
   },
-  "required": ["urls"],
+  "anyOf": [
+    {
+      "required": ["sitemap"],
+      "errorMessage": "must have either property 'sitemap' or 'urls' (or both)"
+    },
+    {
+      "required": ["urls"],
+      "errorMessage": "must have either property 'sitemap' or 'urls' (or both)"
+    }
+  ],
   "definitions": {
     "detailedSetting": {
       "description": "The complete set of test-specific settings for most tests.",