@uxf/scripts 11.49.3 → 11.58.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@uxf/scripts",
-    "version": "11.49.3",
+    "version": "11.58.0",
     "description": "",
     "main": "index.js",
     "bin": {
@@ -33,6 +33,7 @@
         "dayjs": "1.11.13",
         "fast-glob": "3.3.2",
         "madge": "8.0.0",
+        "robots-txt-parser": "2.0.3",
         "yargs": "17.7.2"
     }
 }

package/src/Sitemap.js

@@ -1,4 +1,4 @@
-const https = require('https');
+const https = require("https");
 const { create } = require("axios");
 const cheerio = require("cheerio");
 
@@ -18,7 +18,7 @@ async function getSitemap(xml) {
 }
 
 const agent = new https.Agent({
-    rejectUnauthorized: false
+    rejectUnauthorized: false,
 });
 
 const axios = create({

package/src/uxf-sitemap-check/cli.js

@@ -2,31 +2,20 @@ const { argv, env } = require("process");
 
 module.exports = async () => {
     const cli = require("yargs")
-        .command("$0", "UXF sitemap checker", yargs => {
+        .command("$0", "UXF sitemap checker", (yargs) => {
             yargs.demandCommand(0, 0).usage(`UXF sitemap checker
 Usage:
   uxf-sitemap-check [options]
   
 Environment variables:
   HTTP_USERNAME - optional
-  HTTP_PASSWORD - optional
-  SLACK_TOKEN   - optional`);
+  HTTP_PASSWORD - optional`);
         })
         .option("url", {
             describe: "Sitemap url",
             type: "string",
             group: "Options",
         })
-        .option("web-url", {
-            describe: "Web url for check",
-            type: "string",
-            group: "Options",
-        })
-        .option("slack-channel", {
-            describe: "Slack channel id.",
-            type: "string",
-            group: "Options",
-        })
         .option("u", {
             alias: "http-username",
             describe: "Http authorization username (or use environment variable HTTP_USERNAME)",
@@ -44,11 +33,16 @@ Environment variables:
             type: "string",
             group: "Options",
         })
-        .option("test-nested", {
+        .option("with-nested", {
             describe: "If nested urls should be tested.",
             type: "boolean",
             group: "Options",
         })
+        .option("with-images", {
+            describe: "If images should be tested.",
+            type: "boolean",
+            group: "Options",
+        })
         .option("h", { alias: "help", group: "Options" })
         .strict(false)
         .exitProcess(false);
@@ -56,7 +50,6 @@ Environment variables:
     try {
         const { help, url, ...options } = cli.parse(argv.slice(2));
         const skip = options.skip ? Number.parseInt(options.skip) : null;
-        const webUrl = options["web-url"] || null;
 
         if (Boolean(help)) {
             return 0;
@@ -67,7 +60,7 @@ Environment variables:
             env.HTTP_PASSWORD = options["http-password"];
         }
 
-        await require("./index")(url, webUrl, options["slack-channel"], skip, options["test-nested"]);
+        await require("./index")(url, skip, options["with-nested"], options["with-images"]);
     } catch (e) {
         console.error(e);
         return 1;

package/src/uxf-sitemap-check/index.js

@@ -1,23 +1,43 @@
-const Slack = require("../Slack");
 const Sitemap = require("../Sitemap");
 const { performance } = require("perf_hooks");
-const { env, stdout } = require("process");
-const { axios } = require("../Sitemap");
+const { stdout } = require("process");
 const cheerio = require("cheerio");
 const GoogleChat = require("../GoogleChat");
+const robotsTxtParser = require("robots-txt-parser");
+
+const { HTTP_USERNAME, HTTP_PASSWORD } = process.env;
 
 /**
- *
- * @typedef {{parentUrl: (string | undefined), isImg: boolean, time: number, ttl: number, url: string, status: number, message: (string | undefined), shouldIgnoreError: (boolean | undefined)}} UrlCheckResponse
+ * @typedef {{parentUrl: (string | undefined), isImg: boolean, ttl: number, url: string, status: number, message: (string | undefined), skipped: boolean, html: (string | null)}} UrlCheckResponse
  */
 
 const MAX_TTL = 3;
-const TESTED_URLS = [];
 const IMAGES_LABEL = "🏞 Images:";
 const URLS_LABEL = "🔗 Links:";
 
+const TESTED_URLS = [];
+const URLS_TO_CHECK = new Set();
+
+const robotsParser = robotsTxtParser({ userAgent: "uxf-bot", allowOnNeutral: false });
+
+/**
+ * @param url {string}
+ * @param options {{redirect: boolean, isExternal: boolean}}
+ * @returns {Promise<Response>}
+ */
+function fetcher(url, options) {
+    return fetch(url, {
+        credentials: "include",
+        headers:
+            !options.isExternal && HTTP_USERNAME && HTTP_PASSWORD
+                ? new Headers({ Authorization: "Basic " + btoa(`${HTTP_USERNAME}:${HTTP_PASSWORD}`) })
+                : undefined,
+        redirect: options.redirect ? "follow" : "manual",
+        signal: AbortSignal.timeout(20_000),
+    });
+}
+
 /**
- *
  * @param length {number}
  * @return {string}
  */
@@ -26,7 +46,6 @@ function createTabSpace(length = 1) {
 }
 
 /**
- *
  * @param url {string}
  * @return {boolean}
  */
@@ -35,37 +54,19 @@ function isImageUrl(url) {
 }
 
 /**
- *
- * @param url {string}
- * @param status {number}
- * @param e {Error}
- * @returns {boolean}
- */
-function shouldIgnoreError(url, status, e) {
-    if (status === 999 && url.startsWith("https://www.linkedin.com")) {
-        return true;
-    }
-    if ((status === -1 || status === 302) && url.startsWith("https://www.facebook.com/sharer/")) {
-        return true;
-    }
-    if (status === -3) {
-        return true;
-    }
-
-    return false;
-}
-
-/**
- *
  * @param errors {UrlCheckResponse[]}
  * @return {string}
  */
 function createErrorList(errors) {
-    return errors.map(err => `${createTabSpace(3)}${err.url}${createTabSpace()}${err.status}${err.message ? ` – ${err.message}` : ""}`).join("\n");
+    return errors
+        .map(
+            (err) =>
+                `${createTabSpace(3)}${err.url}${createTabSpace()}${err.status}${err.message ? ` – ${err.message}` : ""}`,
+        )
+        .join("\n");
 }
 
 /**
- *
  * @param errors {UrlCheckResponse[]}
  * @return {string}
  */
@@ -73,13 +74,13 @@ function createErrorResult(errors) {
     let parentPages = "";
     let nestedPages = "";
 
-    const parentPagesErrors = errors.filter(url => url.parentUrl === undefined);
+    const parentPagesErrors = errors.filter((url) => url.parentUrl === undefined);
     if (parentPagesErrors.length > 0) {
         parentPages = `${createTabSpace()}Pages from sitemap:\n${createErrorList(parentPagesErrors)}\n`;
     }
 
     const nestedPagesErrors = errors
-        .filter(url => url.parentUrl !== undefined)
+        .filter((url) => url.parentUrl !== undefined)
         .sort((prev, curr) => prev.parentUrl.localeCompare(curr.parentUrl));
     for (let i = 0; i < nestedPagesErrors.length; i++) {
         if (i === 0) {
@@ -92,8 +93,8 @@ function createErrorResult(errors) {
                 nestedPages += `${createTabSpace(1)}Page: ${nestedPagesErrors[i].parentUrl}\n`;
             }
         }
-        const images = nestedPagesErrors.filter(err => err.parentUrl === nestedPagesErrors[i].parentUrl && err.isImg);
-        const links = nestedPagesErrors.filter(err => err.parentUrl === nestedPagesErrors[i].parentUrl && !err.isImg);
+        const images = nestedPagesErrors.filter((err) => err.parentUrl === nestedPagesErrors[i].parentUrl && err.isImg);
+        const links = nestedPagesErrors.filter((err) => err.parentUrl === nestedPagesErrors[i].parentUrl && !err.isImg);
         if (images.length > 0) {
             nestedPages += `${createTabSpace(2)}${IMAGES_LABEL}\n${createErrorList(images)}\n`;
         }
@@ -106,7 +107,21 @@ function createErrorResult(errors) {
 }
 
 /**
- *
+ * @param skippedUrls {UrlCheckResponse[]}
+ * @return {string}
+ */
+function createSkippedResult(skippedUrls) {
+    const flattened = skippedUrls.flatMap((url) => new URL(url.url).origin);
+    const unique = [...new Set(flattened)];
+
+    if (unique.length > 0) {
+        return `${unique.map((err) => `${createTabSpace(2)}${err}`).join("\n")}\n`;
+    }
+
+    return "";
+}
+
+/**
  * @param incorrectLinks {string[]}
  * @param webUrl {string}
  * @return {string[]}
@@ -122,23 +137,83 @@ function createCorrectLinks(incorrectLinks, webUrl) {
 }
 
 /**
- *
  * @param url {string}
+ * @returns {boolean}
+ */
+function validURL(url) {
+    try {
+        new URL(url);
+
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+/**
+ * @param url {string}
+ * @param webUrl {string}
  * @param parentUrl {string | undefined}
  * @param ttl {number}
  * @return {Promise<UrlCheckResponse>}
  */
-async function fetchUrl(url, parentUrl = undefined, ttl = 1) {
+async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
+    if (!validURL(url)) {
+        return {
+            url,
+            parentUrl,
+            isImg: isImageUrl(url),
+            ttl,
+            status: 0,
+            message: "invalid url: " + url,
+            skipped: false,
+            html: null,
+        };
+    }
+
     try {
-        Sitemap.axios.defaults.maxRedirects = parentUrl ? 10 : 0;
-        const encodedUrl = parentUrl ? encodeURI(decodeURI(url)) : url;
+        const origin = new URL(url).origin;
+
+        if (parentUrl && origin !== webUrl) {
+            await robotsParser.useRobotsFor(origin);
+
+            const canCrawl = await robotsParser.canCrawl(url);
+
+            if (!canCrawl) {
+                return {
+                    url,
+                    parentUrl,
+                    isImg: isImageUrl(url),
+                    ttl,
+                    status: 0,
+                    message: "blocked by robots.txt",
+                    skipped: true,
+                    html: null,
+                };
+            }
+        }
+    } catch (e) {
+        const errorStatus = await fetcher(new URL(url).origin + "/robots.txt", { isExternal: true, redirect: true })
+            .then((res) => (res.status === 200 ? -1 : res.status))
+            .catch((e) => e.response?.status);
 
-        const t0 = performance.now();
-        const { status } = await Sitemap.axios.get(encodedUrl);
-        const t1 = performance.now();
+        return {
+            url,
+            parentUrl,
+            isImg: isImageUrl(url),
+            ttl,
+            status: errorStatus,
+            message: e.message,
+            skipped: errorStatus !== undefined,
+            html: null,
+        };
+    }
 
-        if (status !== 200 && ttl < MAX_TTL) {
-            return await fetchUrl(encodedUrl, parentUrl, ttl + 1);
+    try {
+        const response = await fetcher(url, { redirect: !!parentUrl, isExternal: !url.includes(webUrl) });
+
+        if (response.status !== 200 && ttl < MAX_TTL) {
+            return await fetchUrl(url, webUrl, parentUrl, ttl + 1);
         }
 
         return {
@@ -146,46 +221,36 @@ async function fetchUrl(url, parentUrl = undefined, ttl = 1) {
             parentUrl,
             isImg: isImageUrl(url),
             ttl,
-            status,
-            time: Math.ceil(t1 - t0),
+            status: response.status,
+            skipped: false,
+            html: await response.text(),
         };
     } catch (e) {
         const status = Number.parseInt((e && e.response && e.response.status) || -1, 10);
 
-        if (shouldIgnoreError(url, status, e)) {
-            return {
-                url,
-                parentUrl,
-                isImg: isImageUrl(url),
-                ttl,
-                status,
-                shouldIgnoreError: true,
-                time: 0,
-            };
-        }
         return {
             url,
             parentUrl,
             isImg: isImageUrl(url),
             ttl,
             status,
-            time: 0,
             message: e.message,
-            shouldIgnoreError: false,
+            skipped: false,
+            html: null,
         };
     }
 }
 
 /**
- *
  * @param url {string}
+ * @param webUrl {string}
  * @param parentUrl {string | undefined}
  * @return {UrlCheckResponse}
  */
-async function testUrl(url, parentUrl = undefined) {
-    const indexInChecked = TESTED_URLS.findIndex(result => result.url === url);
+async function testUrl(url, webUrl, parentUrl = undefined) {
+    const indexInChecked = TESTED_URLS.findIndex((result) => result.url === url);
     if (indexInChecked === -1) {
-        const result = await fetchUrl(url, parentUrl);
+        const result = await fetchUrl(url, webUrl, parentUrl);
         TESTED_URLS.push(result);
         return result;
     }
@@ -193,99 +258,109 @@ async function testUrl(url, parentUrl = undefined) {
 }
 
 /**
- *
  * @param urls {string[]}
  * @param webUrl {string}
  * @param sitemapUrl {string}
  * @param skip {number}
- * @param testNested {boolean}
+ * @param withNested {boolean}
+ * @param withImages {boolean}
  * @return {Promise<void>}
  */
-async function testSitemapUrls(urls, webUrl, sitemapUrl, skip, testNested) {
+async function testSitemapUrls(urls, webUrl, sitemapUrl, skip, withNested, withImages) {
+    for (let i = skip || 0; i < urls.length; i++) {
+        URLS_TO_CHECK.add(urls[i]);
+    }
+
     for (let i = skip || 0; i < urls.length; i++) {
         const url = urls[i];
-        const changedUrl = webUrl ? `${webUrl}${new URL(url).pathname}` : null;
+        const changedUrl = webUrl ? `${webUrl}${new URL(url).pathname}` : url;
 
-        printUrlInfo(changedUrl ?? url, i, urls.length);
+        const result = await testUrl(changedUrl, webUrl);
+        printProgress();
 
-        const result = await testUrl(changedUrl ?? url);
-        printUrlResult(result);
+        if (withNested && result.status === 200) {
+            await testNestedUrls(result.html, changedUrl, i, webUrl ?? sitemapUrl.split("/").slice(0, 3).join("/"));
+        }
 
-        if (testNested && result.status === 200) {
-            await testAllNestedUrls(changedUrl ?? url, i, webUrl ?? sitemapUrl.split("/").slice(0, 3).join("/"));
+        if (withImages && result.status === 200) {
+            await testNestedImages(result.html, changedUrl, i, webUrl ?? sitemapUrl.split("/").slice(0, 3).join("/"));
         }
     }
 }
 
 /**
- *
+ * @param html {string}
  * @param parentUrl {string}
  * @param parentIndex {number}
  * @param webUrl {string}
  * @return {Promise<void>}
  */
-async function testAllNestedUrls(parentUrl, parentIndex, webUrl) {
-    const { data } = await axios.get(parentUrl);
-    const $ = cheerio.load(data);
+async function testNestedUrls(html, parentUrl, parentIndex, webUrl) {
+    const $ = cheerio.load(html);
     const urls = createCorrectLinks(
         $("a[href]").map((i, node) => $(node).attr("href")),
         webUrl,
     );
+
+    await testNested(urls, parentIndex, parentUrl, createTabSpace() + URLS_LABEL, webUrl);
+}
+
+/**
+ * @param html {string}
+ * @param parentUrl {string}
+ * @param parentIndex {number}
+ * @param webUrl {string}
+ * @return {Promise<void>}
+ */
+async function testNestedImages(html, parentUrl, parentIndex, webUrl) {
+    const $ = cheerio.load(html);
     const images = createCorrectLinks(
         $("img[src]").map((i, node) => $(node).attr("src")),
         webUrl,
     );
 
-    await testNested(images, parentIndex, parentUrl, createTabSpace() + IMAGES_LABEL);
-    await testNested(urls, parentIndex, parentUrl, createTabSpace() + URLS_LABEL);
+    await testNested(images, parentIndex, parentUrl, createTabSpace() + IMAGES_LABEL, webUrl);
 }
 
 /**
- *
  * @param urls {string[]}
  * @param parentIndex {number}
  * @param parentUrl {string}
  * @param label {string}
+ * @param webUrl {string}
  * @return {Promise<void>}
  */
-async function testNested(urls, parentIndex, parentUrl, label) {
+async function testNested(urls, parentIndex, parentUrl, label, webUrl) {
     if (urls.length === 0) {
         return;
     }
 
-    stdout.write(label + "\n");
+    for (let i = 0 || 0; i < urls.length; i++) {
+        URLS_TO_CHECK.add(urls[i]);
+    }
+
     for (let i = 0; i < urls.length; i++) {
-        if (TESTED_URLS.findIndex(result => result.url === urls[i]) !== -1) {
+        if (TESTED_URLS.findIndex((result) => result.url === urls[i]) !== -1) {
             continue;
         }
-        printUrlInfo(urls[i], i, urls.length, `${createTabSpace(2)}(${parentIndex + 1}) `);
-        const result = await testUrl(urls[i], parentUrl);
-        printUrlResult(result);
+        await testUrl(urls[i], webUrl, parentUrl);
+        printProgress();
     }
 }
 
-/**
- *
- * @param url {string}
- * @param urlIndex {number}
- * @param allUrlsCount {number}
- * @param prefix {string}
- */
-function printUrlInfo(url, urlIndex, allUrlsCount, prefix = "") {
-    stdout.write(`${prefix}${urlIndex + 1} / ${allUrlsCount}${createTabSpace()}${url}`);
-}
-
-/**
- *
- * @param result {UrlCheckResponse}
- */
-function printUrlResult(result) {
-    const { ttl, status, time, message } = result;
-    stdout.write(`${createTabSpace()}${status}${message ? " – " + message : ""} (${time}ms) ttl=${ttl} ${status === 200 ? "✅ " : "❌ "}\n`);
+let lastLogTime = 0;
+function printProgress() {
+    const now = Date.now();
+    // Only log every 5 seconds in non-TTY
+    if (now - lastLogTime > 5000) {
+        stdout.write(
+            `Progress: ${TESTED_URLS.length}/${URLS_TO_CHECK.size} completed, ${TESTED_URLS.filter((u) => u.skipped).length} skipped, ${TESTED_URLS.filter((u) => u.status !== 200 && u.skipped === false).length} errors\n`,
+        );
+        lastLogTime = now;
+    }
 }
 
 /**
- *
  * @param errorText {string}
  * @param title {string}
  */
@@ -295,7 +370,6 @@ function logErrors(errorText, title) {
 }
 
 /**
- *
  * @param millis {number}
  * @return {string}
  */
@@ -306,105 +380,82 @@ function convertTime(millis) {
 }
 
 /**
- *
  * @param okResults {UrlCheckResponse[]}
  * @param time {number}
  */
 function logStatistics(okResults, time) {
-    const avgTime = Math.round(okResults.reduce((prev, curr) => prev + curr.time, 0) / TESTED_URLS.length);
-    const maxTime = okResults.reduce((prev, curr) => (curr.time > prev.time ? curr : prev), []);
-    const minTime = okResults.reduce((prev, curr) => (curr.time < prev.time ? curr : prev), []);
-
     stdout.write("\nSummary:\n");
     stdout.write(createTabSpace() + `Time ${convertTime(time)}\n`);
     stdout.write(
-        createTabSpace() + "Images tested:" + createTabSpace() + TESTED_URLS.filter(url => url.isImg).length + "\n",
+        createTabSpace() + "Images tested:" + createTabSpace() + TESTED_URLS.filter((url) => url.isImg).length + "\n",
     );
     stdout.write(
-        createTabSpace() + "Links tested:" + createTabSpace() + TESTED_URLS.filter(url => !url.isImg).length + "\n",
+        createTabSpace() + "Links tested:" + createTabSpace() + TESTED_URLS.filter((url) => !url.isImg).length + "\n",
     );
-    stdout.write(createTabSpace() + "Avg time:" + createTabSpace() + avgTime + "ms\n");
     stdout.write(
-        createTabSpace() + "Min time:" + createTabSpace() + minTime.time + "ms" + createTabSpace() + minTime.url + "\n",
+        createTabSpace() + "Skipped:" + createTabSpace() + TESTED_URLS.filter((url) => url.skipped).length + "\n",
     );
     stdout.write(
-        createTabSpace() + "Max time" + createTabSpace() + maxTime.time + "ms" + createTabSpace() + maxTime.url + "\n",
+        createTabSpace() +
+            "Errors:" +
+            createTabSpace() +
+            TESTED_URLS.filter((url) => url.status !== 200 && url.skipped === false).length +
+            "\n",
     );
 }
 
 /**
- *
- * @param errorText {string}
- * @param slackChannel {string}
- * @return {Promise<void>}
- */
-async function sendSlackMessage(errorText, slackChannel) {
-    await Slack.chatPostMessage(slackChannel, {
-        text: ":warning: Odkazy uvedené v sitemap.xml nejsou dostupné",
-        attachments: [
-            {
-                text: errorText,
-            },
-        ],
-    });
-}
-
-/**
- *
  * @param resultErrors {string}
  * @return {Promise<void>}
  */
 async function sendGoogleChatMessage(resultErrors) {
     await GoogleChat.chatPostMessage({
-        text: resultErrors
+        text: resultErrors,
     });
 }
 
 /**
- *
  * @param sitemapUrl {string}
- * @param webUrl {string}
- * @param slackChannel {string}
  * @param skip {number}
- * @param testNested {boolean}
+ * @param withNested {boolean}
+ * @param withImages {boolean}
  * @return {Promise<*>}
  */
-module.exports = async function run(sitemapUrl, webUrl, slackChannel, skip, testNested) {
+module.exports = async function run(sitemapUrl, skip, withNested, withImages) {
     if (!sitemapUrl) {
         stdout.write("⛔ Required parameter --url is empty.\n");
         return process.exit(1);
     }
 
-    if (slackChannel && !env.SLACK_TOKEN) {
-        stdout.write("⛔ Environment variable SLACK_TOKEN is empty.\n");
-        return process.exit(1);
-    }
+    const url = new URL(sitemapUrl);
+    const webUrl = url.origin;
 
-    if (webUrl) {
-        stdout.write(`${createTabSpace()}Sitemap url: ${sitemapUrl}\n`);
-        stdout.write(`❗${createTabSpace()}Web url is defined: ${webUrl}\n\n`);
+    stdout.write(`${createTabSpace()}Sitemap url: ${sitemapUrl}\n`);
+    stdout.write(`${createTabSpace()}Web url: ${webUrl}\n\n`);
+    if (withNested) {
+        stdout.write(`${createTabSpace()}Will test nested links\n`);
+    }
+    if (withImages) {
+        stdout.write(`${createTabSpace()}Will test images\n\n`);
     }
 
     const startTime = performance.now();
-    await testSitemapUrls((await Sitemap.getSitemap(sitemapUrl)), webUrl, sitemapUrl, skip, testNested);
+    await testSitemapUrls(await Sitemap.getSitemap(sitemapUrl), webUrl, sitemapUrl, skip, withNested, withImages);
     const finishTime = performance.now();
 
-    const errors = TESTED_URLS.filter(r => r.status !== 200 && r.shouldIgnoreError === false);
-    const ignoredErrors = TESTED_URLS.filter(r => r.status !== 200 && r.shouldIgnoreError === true);
-    const ok = TESTED_URLS.filter(r => r.status === 200);
+    const errors = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === false);
+    const skippedUrls = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === true);
+    const ok = TESTED_URLS.filter((r) => r.status === 200);
 
     if (errors.length > 0) {
         const errorText = createErrorResult(errors);
-        logErrors(errorText, "\nErrors:\n");
-        const ignoredErrorText = createErrorResult(ignoredErrors);
-        logErrors(ignoredErrorText, "\nIngored errors:\n");
-        await sendSlackMessage(errorText, slackChannel);
+        logErrors(errorText, "\n\n\nErrors:\n");
         await sendGoogleChatMessage(errorText);
     }
 
-    if (ignoredErrors.length > 0) {
-        const ignoredErrorText = createErrorResult(ignoredErrors);
-        logErrors(ignoredErrorText, "\nIngored errors:\n");
+    if (skippedUrls.length > 0) {
+        const skippedUrlsText = createSkippedResult(skippedUrls);
+        logErrors(skippedUrlsText, "\nSkipped origins:\n");
     }
 
     logStatistics(ok, Math.ceil(finishTime - startTime));