@uxf/scripts 1.4.2 → 1.5.2

package/.eslintrc.json

@@ -1,3 +1,3 @@
 {
-  "parser": "babel-eslint"
+  "parser": "@babel/eslint-parser"
 }

package/babel.config.js

@@ -0,0 +1,22 @@
+"use strict";
+/**
+babel.config.js with useful plugins.
+*/
+module.exports = function (api) {
+    api.cache(true);
+    api.assertVersion("^7.4.5");
+
+    return {
+        presets: [
+            [
+                "@babel/preset-env",
+                {
+                    targets: {
+                        esmodules: true,
+                        node: true,
+                    },
+                },
+            ],
+        ],
+    };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uxf/scripts",
-  "version": "1.4.2",
+  "version": "1.5.2",
   "description": "",
   "main": "index.js",
   "bin": {
@@ -23,8 +23,9 @@
     "yargs": "^16.0.3"
   },
   "devDependencies": {
-    "babel-eslint": "^10.1.0",
-    "eslint": "^7.20.0",
+    "@babel/eslint-parser": "^7.17.0",
+    "@babel/preset-env": "^7.16.11",
+    "eslint": "^8.8.0",
     "prettier": "^2.1.2"
   },
   "scripts": {

package/src/Sitemap.js

@@ -4,31 +4,32 @@ const cheerio = require("cheerio");
 const { HTTP_USERNAME, HTTP_PASSWORD } = process.env;
 
 async function getSitemap(xml) {
-  const { data } = await axios.get(xml);
-  const $ = cheerio.load(data, { xmlMode: true });
+    const { data } = await axios.get(xml);
+    const $ = cheerio.load(data, { xmlMode: true });
 
-  const urls = [];
+    const urls = [];
 
-  $("loc").each(function () {
-    urls.push($(this).text());
-  });
+    $("loc").each(function () {
+        urls.push($(this).text());
+    });
 
-  return urls;
+    return urls;
 }
 
 const axios = create({
-  auth:
-      HTTP_PASSWORD && HTTP_USERNAME
-          ? {
-            username: HTTP_USERNAME,
-            password: HTTP_PASSWORD,
-          }
-          : undefined,
-  withCredentials: true,
-  maxRedirects: 0,
+    auth:
+        HTTP_PASSWORD && HTTP_USERNAME
+            ? {
+                  username: HTTP_USERNAME,
+                  password: HTTP_PASSWORD,
+              }
+            : undefined,
+    withCredentials: true,
+    maxRedirects: 0,
+    timeout: 20000,
 });
 
 module.exports = {
-  getSitemap,
-  axios
+    getSitemap,
+    axios,
 };

package/src/Slack.js

@@ -15,12 +15,14 @@ const axios = create({
 async function chatPostMessage(channel, data, dryRun = false) {
     if (env.SLACK_TOKEN && !dryRun) {
         const res = await axios.post("/chat.postMessage", { ...data, channel });
-        if (res.data.ok !== true) {
+        if (res.data.ok === false) {
             process.stdout.write("SLACK: chat.postMessage error - " + JSON.stringify(res.data));
+            return;
         }
-    } else {
-        process.stdout.write("SLACK: chat.postMessage - skipped");
+        process.stdout.write("SLACK: chat.postMessage - done\n");
+        return;
     }
+    process.stdout.write("SLACK: chat.postMessage - skipped\n");
 }
 
 module.exports = {

package/src/uxf-sitemap-check/cli.js

@@ -39,12 +39,23 @@ Environment variables:
             type: "string",
             group: "Options",
         })
+        .option("skip", {
+            describe: "Number of skipped urls.",
+            type: "string",
+            group: "Options",
+        })
+        .option("test-nested", {
+            describe: "If nested urls should be tested.",
+            type: "boolean",
+            group: "Options",
+        })
         .option("h", { alias: "help", group: "Options" })
         .strict(false)
         .exitProcess(false);
 
     try {
         const { help, url, ...options } = cli.parse(argv.slice(2));
+        const skip = options.skip ? Number.parseInt(options.skip) : null;
         const webUrl = options["web-url"] || null;
 
         if (Boolean(help)) {
@@ -56,7 +67,7 @@ Environment variables:
             env.HTTP_PASSWORD = options["http-password"];
         }
 
-        await require("./index")(url, webUrl, options["slack-channel"]);
+        await require("./index")(url, webUrl, options["slack-channel"], skip, options["test-nested"]);
     } catch (e) {
         console.error(e);
         return 1;

package/src/uxf-sitemap-check/index.js

@@ -2,11 +2,114 @@ const Slack = require("../Slack");
 const Sitemap = require("../Sitemap");
 const { performance } = require("perf_hooks");
 const { env, stdout } = require("process");
+const { axios } = require("../Sitemap");
+const cheerio = require("cheerio");
+
+/**
+ *
+ * @typedef {{parentUrl: (string | undefined), isImg: boolean, time: number, ttl: number, url: string, status: number}} UrlCheckResponse
+ */
 
 const MAX_TTL = 3;
+const TESTED_URLS = [];
+const IMAGES_LABEL = "🏞 Images:";
+const URLS_LABEL = "🔗 Links:";
+
+/**
+ *
+ * @param length {number}
+ * @return {string}
+ */
+function createTabSpace(length = 1) {
+    return "".padStart(4 * length);
+}
+
+/**
+ *
+ * @param url {string}
+ * @return {boolean}
+ */
+function isImageUrl(url) {
+    return new RegExp("[a-z].*(\\.jpg|\\.png|\\.webp|\\.avif|\\.gif)$", "gim").test(url);
+}
+
+/**
+ *
+ * @param errors {UrlCheckResponse[]}
+ * @return {string}
+ */
+function createErrorList(errors) {
+    return errors.map(err => `${createTabSpace(3)}${err.url}${createTabSpace()}${err.status}`).join("\n");
+}
+
+/**
+ *
+ * @param errors {UrlCheckResponse[]}
+ * @return {string}
+ */
+function createErrorResult(errors) {
+    let parentPages = "";
+    let nestedPages = "";
+
+    const parentPagesErrors = errors.filter(url => url.parentUrl === undefined);
+    if (parentPagesErrors.length > 0) {
+        parentPages = `${createTabSpace()}Pages from sitemap:\n${createErrorList(parentPagesErrors)}\n`;
+    }
+
+    const nestedPagesErrors = errors
+        .filter(url => url.parentUrl !== undefined)
+        .sort((prev, curr) => prev.parentUrl.localeCompare(curr.parentUrl));
+    for (let i = 0; i < nestedPagesErrors.length; i++) {
+        if (i === 0) {
+            nestedPages = `${createTabSpace()}Nested pages:\n`;
+            nestedPages += `${createTabSpace(1)}Page: ${nestedPagesErrors[i].parentUrl}\n`;
+        } else {
+            if (nestedPagesErrors[i].parentUrl === nestedPagesErrors[i - 1].parentUrl) {
+                continue;
+            } else {
+                nestedPages += `${createTabSpace(1)}Page: ${nestedPagesErrors[i].parentUrl}\n`;
+            }
+        }
+        const images = nestedPagesErrors.filter(err => err.parentUrl === nestedPagesErrors[i].parentUrl && err.isImg);
+        const links = nestedPagesErrors.filter(err => err.parentUrl === nestedPagesErrors[i].parentUrl && !err.isImg);
+        if (images.length > 0) {
+            nestedPages += `${createTabSpace(2)}${IMAGES_LABEL}\n${createErrorList(images)}\n`;
+        }
+        if (links.length > 0) {
+            nestedPages += `${createTabSpace(2)}${URLS_LABEL}\n${createErrorList(links)}\n`;
+        }
+    }
 
-async function tryUrl(url, ttl = 1) {
+    return parentPages + nestedPages;
+}
+
+/**
+ *
+ * @param incorrectLinks {string[]}
+ * @param webUrl {string}
+ * @return {string[]}
+ */
+function createCorrectLinks(incorrectLinks, webUrl) {
+    return [
+        ...new Set(
+            incorrectLinks
+                .filter((i, url) => url && new RegExp("^(\\/|http)").test(url))
+                .map((i, url) => (url.startsWith("http") ? url : webUrl + url)),
+        ),
+    ];
+}
+
+/**
+ *
+ * @param url {string}
+ * @param parentUrl {string | undefined}
+ * @param ttl {number}
+ * @return {Promise<UrlCheckResponse>}
+ */
+async function fetchUrl(url, parentUrl = undefined, ttl = 1) {
     try {
+        Sitemap.axios.defaults.maxRedirects = parentUrl ? 1 : 0;
+
         const t0 = performance.now();
         const { status } = await Sitemap.axios.get(url);
         const t1 = performance.now();
@@ -17,17 +120,21 @@ async function tryUrl(url, ttl = 1) {
 
         return {
             url,
-            status,
+            parentUrl,
+            isImg: isImageUrl(url),
             ttl,
+            status,
             time: Math.ceil(t1 - t0),
         };
     } catch (e) {
         if (ttl < MAX_TTL) {
-            return tryUrl(url, ttl + 1);
+            return fetchUrl(url, parentUrl, ttl + 1);
         }
 
         return {
             url,
+            parentUrl,
+            isImg: isImageUrl(url),
             ttl,
             status: Number.parseInt((e && e.response && e.response.status) || "0"),
             time: 0,
@@ -35,7 +142,183 @@ async function tryUrl(url, ttl = 1) {
     }
 }
 
-module.exports = async function run(sitemapUrl, webUrl, slackChannel) {
+/**
+ *
+ * @param url {string}
+ * @param parentUrl {string | undefined}
+ * @return {UrlCheckResponse}
+ */
+async function testUrl(url, parentUrl = undefined) {
+    const indexInChecked = TESTED_URLS.findIndex(result => result.url === url);
+    if (indexInChecked === -1) {
+        const result = await fetchUrl(url, parentUrl);
+        TESTED_URLS.push(result);
+        return result;
+    }
+    return TESTED_URLS[indexInChecked];
+}
+
+/**
+ *
+ * @param urls {string[]}
+ * @param webUrl {string}
+ * @param sitemapUrl {string}
+ * @param skip {number}
+ * @param testNested {boolean}
+ * @return {Promise<void>}
+ */
+async function testSitemapUrls(urls, webUrl, sitemapUrl, skip, testNested) {
+    for (let i = skip || 0; i < urls.length; i++) {
+        const url = urls[i];
+        const changedUrl = webUrl ? `${webUrl}${new URL(url).pathname}` : null;
+
+        printUrlInfo(changedUrl ?? url, i, urls.length);
+        printUrlResult(await testUrl(changedUrl ?? url));
+
+        if (testNested) {
+            await testAllNestedUrls(changedUrl ?? url, i, webUrl ?? sitemapUrl.split("/").slice(0, 3).join("/"));
+        }
+    }
+}
+
+/**
+ *
+ * @param parentUrl {string}
+ * @param parentIndex {number}
+ * @param webUrl {string}
+ * @return {Promise<void>}
+ */
+async function testAllNestedUrls(parentUrl, parentIndex, webUrl) {
+    const { data } = await axios.get(parentUrl);
+    const $ = cheerio.load(data);
+    const urls = createCorrectLinks(
+        $("a").map((i, node) => $(node).attr("href")),
+        webUrl,
+    );
+    const images = createCorrectLinks(
+        $("img").map((i, node) => $(node).attr("src")),
+        webUrl,
+    );
+
+    await testNested(images, parentIndex, parentUrl, createTabSpace() + IMAGES_LABEL);
+    await testNested(urls, parentIndex, parentUrl, createTabSpace() + URLS_LABEL);
+}
+
+/**
+ *
+ * @param urls {string[]}
+ * @param parentIndex {number}
+ * @param parentUrl {string}
+ * @param label {string}
+ * @return {Promise<void>}
+ */
+async function testNested(urls, parentIndex, parentUrl, label) {
+    if (urls.length === 0) {
+        return;
+    }
+
+    stdout.write(label + "\n");
+    for (let i = 0; i < urls.length; i++) {
+        printUrlInfo(urls[i], i, urls.length, `${createTabSpace(2)}(${parentIndex + 1}) `);
+        const result = await testUrl(urls[i], parentUrl);
+        printUrlResult(result);
+    }
+}
+
+/**
+ *
+ * @param url {string}
+ * @param urlIndex {number}
+ * @param allUrlsCount {number}
+ * @param prefix {string}
+ */
+function printUrlInfo(url, urlIndex, allUrlsCount, prefix = "") {
+    stdout.write(`${prefix}${urlIndex + 1} / ${allUrlsCount}${createTabSpace()}${url}`);
+}
+
+/**
+ *
+ * @param result {UrlCheckResponse}
+ */
+function printUrlResult(result) {
+    const { ttl, status, time } = result;
+    stdout.write(`${createTabSpace()}${status} (${time}ms) ttl=${ttl} ${status === 200 ? "✅ " : "❌ "}\n`);
+}
+
+/**
+ *
+ * @param errorText {string}
+ */
+function logErrors(errorText) {
+    stdout.write("\nErrors:\n");
+    stdout.write(`${errorText}\n\n`);
+}
+
+/**
+ *
+ * @param millis {number}
+ * @return {string}
+ */
+function convertTime(millis) {
+    let minutes = Math.floor(millis / 60000);
+    let seconds = ((millis % 60000) / 1000).toFixed(0);
+    return `${minutes} min ${seconds < 10 ? "0" : ""}${seconds} sec`;
+}
+
+/**
+ *
+ * @param okResults {UrlCheckResponse[]}
+ * @param time {number}
+ */
+function logStatistics(okResults, time) {
+    const avgTime = Math.round(okResults.reduce((prev, curr) => prev + curr.time, 0) / TESTED_URLS.length);
+    const maxTime = okResults.reduce((prev, curr) => (curr.time > prev.time ? curr : prev));
+    const minTime = okResults.reduce((prev, curr) => (curr.time < prev.time ? curr : prev));
+
+    stdout.write("\nSummary:\n");
+    stdout.write(createTabSpace() + `Time ${convertTime(time)}\n`);
+    stdout.write(
+        createTabSpace() + "Images tested:" + createTabSpace() + TESTED_URLS.filter(url => url.isImg).length + "\n",
+    );
+    stdout.write(
+        createTabSpace() + "Links tested:" + createTabSpace() + TESTED_URLS.filter(url => !url.isImg).length + "\n",
+    );
+    stdout.write(createTabSpace() + "Avg time:" + createTabSpace() + avgTime + "ms\n");
+    stdout.write(
+        createTabSpace() + "Min time:" + createTabSpace() + minTime.time + "ms" + createTabSpace() + minTime.url + "\n",
+    );
+    stdout.write(
+        createTabSpace() + "Max time" + createTabSpace() + maxTime.time + "ms" + createTabSpace() + maxTime.url + "\n",
+    );
+}
+
+/**
+ *
+ * @param errorText {string}
+ * @param slackChannel {string}
+ * @return {Promise<void>}
+ */
+async function sendSlackMessage(errorText, slackChannel) {
+    await Slack.chatPostMessage(slackChannel, {
+        text: ":warning: Odkazy uvedené v sitemap.xml nejsou dostupné",
+        attachments: [
+            {
+                text: errorText,
+            },
+        ],
+    });
+}
+
+/**
+ *
+ * @param sitemapUrl {string}
+ * @param webUrl {string}
+ * @param slackChannel {string}
+ * @param skip {number}
+ * @param testNested {boolean}
+ * @return {Promise<*>}
+ */
+module.exports = async function run(sitemapUrl, webUrl, slackChannel, skip, testNested) {
     if (!sitemapUrl) {
         stdout.write("⛔ Required parameter --url is empty.\n");
         return process.exit(1);
@@ -47,48 +330,23 @@ module.exports = async function run(sitemapUrl, webUrl, slackChannel) {
     }
 
     if (webUrl) {
-        stdout.write(`    Sitemap url: ${sitemapUrl}\n`);
-        stdout.write(`❗   Web url is defined: ${webUrl}\n\n`);
+        stdout.write(`${createTabSpace()}Sitemap url: ${sitemapUrl}\n`);
+        stdout.write(`❗${createTabSpace()}Web url is defined: ${webUrl}\n\n`);
     }
 
-    const urls = await Sitemap.getSitemap(sitemapUrl);
-
-    const results = [];
-
-    let i = 0;
-    for (const url of urls) {
-        const changedUrl = webUrl ? `${webUrl}${new URL(url).pathname}` : null;
+    const startTime = performance.now();
+    await testSitemapUrls(await Sitemap.getSitemap(sitemapUrl), webUrl, sitemapUrl, skip, testNested);
+    const finishTime = performance.now();
 
-        stdout.write(`${++i} / ${urls.length} ${url}${changedUrl ? ` => ${changedUrl}` : ""}`);
-
-        const result = await tryUrl(changedUrl || url);
-        results.push(result);
-        const { ttl, status, time } = result;
-
-        stdout.write(` ${status} (${time}ms) ttl=${ttl} ${status === 200 ? "✅ " : "❌ "}\n`);
-    }
+    const errors = TESTED_URLS.filter(r => r.status !== 200);
+    const ok = TESTED_URLS.filter(r => r.status === 200);
 
-    const avgTime = Math.round(results.reduce((prev, curr) => prev + curr.time, 0) / results.length);
-    const maxTime = Math.round(results.reduce((prev, curr) => (curr.time > prev ? curr.time : prev), 0));
-    const minTime = Math.round(results.reduce((prev, curr) => (curr.time < prev ? curr.time : prev), Number.MAX_VALUE));
-    stdout.write("\n");
-    stdout.write("Summary:\n");
-    stdout.write(`    Average time  ${avgTime}ms\n`);
-    stdout.write(`    Min time      ${minTime}ms\n`);
-    stdout.write(`    Max time      ${maxTime}ms\n\n`);
-
-    const resultErrors = results.filter(r => r.status !== 200);
-
-    if (resultErrors.length > 0) {
-        await Slack.chatPostMessage(slackChannel, {
-            text: ":warning: Odkazy uvedené v sitemap.xml nejsou dostupné",
-            attachments: [
-                {
-                    text: resultErrors.map(r => `${r.url} ${r.status}`).join("\n"),
-                },
-            ],
-        });
+    if (errors.length > 0) {
+        const errorText = createErrorResult(errors);
+        logErrors(errorText);
+        await sendSlackMessage(errorText, slackChannel);
     }
+    logStatistics(ok, Math.ceil(finishTime - startTime));
 
-    process.exit(resultErrors.length > 0 ? 1 : 0);
+    process.exit(errors.length > 0 ? 1 : 0);
 };