@uxf/scripts 11.62.1 → 11.62.3

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@uxf/scripts",
-    "version": "11.62.1",
+    "version": "11.62.3",
     "description": "",
     "main": "index.js",
     "bin": {

package/src/uxf-sitemap-check/cli.js

@@ -43,6 +43,11 @@ Environment variables:
             type: "boolean",
             group: "Options",
         })
+        .option("check-missing", {
+            describe: "If indexable pages missing in sitemap should be reported. (--with-nested must be used)",
+            type: "boolean",
+            group: "Options",
+        })
         .option("google-chat-webhook-url", {
             describe: "Webhook URL of Google Chat where to send the results.",
             type: "string",
@@ -70,6 +75,7 @@ Environment variables:
             skip,
             options["with-nested"],
             options["with-images"],
+            options["check-missing"],
             options["google-chat-webhook-url"],
         );
     } catch (e) {

package/src/uxf-sitemap-check/index.js

@@ -9,15 +9,13 @@ const { HTTP_USERNAME, HTTP_PASSWORD } = process.env;
 
 /**
  * @typedef {{parentUrl: (string | undefined), isImg: boolean, isWebPage: boolean, ttl: number, url: string, status: number, message: (string | undefined), skipped: boolean, html: (string | null), redirected: boolean}} UrlCheckResponse
- */
-
-/**
  * @typedef {{url: string, parentUrl: (string | undefined), canonicalUrl: (string | null), isImg: boolean, isWebPage: boolean, ttl: number, status: number, message: (string | undefined), skipped: boolean, indexable: boolean, redirected: boolean}} TestedUrlDto
+ * @typedef {{errors: TestedUrlDto[], duplicates: unknown[], missingInSitemap: (*|*[]), ok: TestedUrlDto[], skippedUrls: TestedUrlDto[], errorsSum: number}} ResultDto
  */
 
 const DUPLICATES_TITLE = "\n\n\nDuplicated pages in sitemap:\n";
 const MISSING_TITLE = "\n\n\nMissing pages in sitemap:\n";
-const ERROR_TITLE = "\n\n\nErrors:\n"
+const ERROR_TITLE = "\n\n\nErrors:\n";
 
 const MAX_TTL = 3;
 const IMAGES_LABEL = "🏞 Images:";
@@ -28,10 +26,27 @@ const URLS_LABEL = "🔗 Links:";
  */
 const TESTED_URLS = [];
 const URLS_TO_CHECK = new Set();
-const ERRORS = [];
 
 const robotsParser = robotsTxtParser({ userAgent: "uxf-bot", allowOnNeutral: false });
 
+const HOSTNAME_ROBOTS_MAP = {
+    "fb.me": "facebook.com",
+};
+
+/**
+ * @param url {string}
+ * @returns {string}
+ */
+function getUrlOrigin(url) {
+    const urlObject = new URL(url);
+
+    if (urlObject.hostname in HOSTNAME_ROBOTS_MAP) {
+        return new URL(url.replace(urlObject.hostname, HOSTNAME_ROBOTS_MAP[urlObject.hostname])).origin;
+    }
+
+    return urlObject.origin;
+}
+
 /**
  * @param url {string}
  * @param options {{redirect: boolean, isExternal: boolean}}
@@ -116,13 +131,6 @@ function createErrorResult(errors) {
         }
     }
 
-    if (ERRORS.length > 0) {
-        generalErrors = `\n\nGeneral errors:\n`;
-        for (const error of ERRORS) {
-            generalErrors += `${createTabSpace(1)}${error}\n`;
-        }
-    }
-
     return parentPages + nestedPages + generalErrors;
 }
 
@@ -194,7 +202,7 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
     }
 
     try {
-        const origin = new URL(url).origin;
+        const origin = getUrlOrigin(url);
 
         if (parentUrl && origin !== webUrl) {
             await robotsParser.useRobotsFor(origin);
@@ -344,17 +352,13 @@ async function testSitemapUrls(urls, webUrl, sitemapUrl, skip, withNested, withI
  * @return {Promise<void>}
  */
 async function testNestedUrls(html, parentUrl, parentIndex, webUrl) {
-    try {
-        const $ = cheerio.load(html.toString());
-        let urls = createCorrectLinks(
-            $("a[href]").map((i, node) => $(node).attr("href")),
-            webUrl,
-        );
+    const $ = cheerio.load(html.toString());
+    let urls = createCorrectLinks(
+        $("a[href]").map((i, node) => $(node).attr("href")),
+        webUrl,
+    );
 
-        await testNested(urls, parentIndex, parentUrl, createTabSpace() + URLS_LABEL, webUrl);
-    } catch (e) {
-        ERRORS.push(`Can't test all nested pages for ${parentUrl} - ${e.message}`);
-    }
+    await testNested(urls, parentIndex, parentUrl, createTabSpace() + URLS_LABEL, webUrl);
 }
 
 /**
@@ -365,17 +369,13 @@ async function testNestedUrls(html, parentUrl, parentIndex, webUrl) {
  * @return {Promise<void>}
  */
 async function testNestedImages(html, parentUrl, parentIndex, webUrl) {
-    try {
-        const $ = cheerio.load(html);
-        const images = createCorrectLinks(
-            $("img[src]").map((i, node) => $(node).attr("src")),
-            webUrl,
-        );
+    const $ = cheerio.load(html);
+    const images = createCorrectLinks(
+        $("img[src]").map((i, node) => $(node).attr("src")),
+        webUrl,
+    );
 
-        await testNested(images, parentIndex, parentUrl, createTabSpace() + IMAGES_LABEL, webUrl);
-    } catch (e) {
-        ERRORS.push(`Can't test all nested images for ${parentUrl} - ${e.message}`);
-    }
+    await testNested(images, parentIndex, parentUrl, createTabSpace() + IMAGES_LABEL, webUrl);
 }
 
 /**
@@ -436,7 +436,7 @@ function convertTime(millis) {
 }
 
 /**
- * @param okResults {UrlCheckResponse[]}
+ * @param okResults {TestedUrlDto[]}
  * @param time {number}
  */
 function logStatistics(okResults, time) {
@@ -484,6 +484,7 @@ function getPagesShouldBeInSitemap(webUrl) {
             (dto) =>
                 (dto.canonicalUrl ?? dto.url).startsWith(webUrl) &&
                 !(dto.canonicalUrl ?? dto.url).includes("?") &&
+                !(dto.canonicalUrl ?? dto.url).includes("#") &&
                 dto.status === 200 &&
                 dto.ttl <= 1 &&
                 !dto.redirected &&
@@ -492,15 +493,91 @@ function getPagesShouldBeInSitemap(webUrl) {
         .map((url) => (url.canonicalUrl ?? url.url).toLowerCase());
 }
 
+/**
+ * @param sitemapUrl {string}
+ * @param webUrl {string}
+ * @param withNested {boolean}
+ * @param withImages {boolean}
+ * @param checkMissing {boolean}
+ * @param shouldReportMissing {boolean}
+ * @return {void}
+ */
+function logInitialInfo(sitemapUrl, webUrl, withNested, withImages, checkMissing, shouldReportMissing) {
+    stdout.write(`${createTabSpace()}Sitemap url: ${sitemapUrl}\n`);
+    stdout.write(`${createTabSpace()}Web url: ${webUrl}\n\n`);
+    if (withNested) {
+        stdout.write(`${createTabSpace()}Will test nested links\n`);
+    }
+    if (withImages) {
+        stdout.write(`${createTabSpace()}Will test images\n\n`);
+    }
+    if (checkMissing) {
+        if (!shouldReportMissing) {
+            stdout.write(`${createTabSpace()}--check-missing option is only available with --with-nested option!\n`);
+            process.exit(1);
+        }
+        stdout.write(`${createTabSpace()}Will look for pages missing in sitemap\n\n`);
+    }
+}
+
+/**
+ * @param webUrl {string}
+ * @param result {ResultDto}
+ * @returns {string}
+ */
+function logResultErrors(webUrl, result) {
+    let chatMessage = `Result for ${webUrl}:\n\n`;
+    const duplicatesText = result.duplicates.map((url) => `${createTabSpace()}${url}`).join("\n");
+    const missingText = result.missingInSitemap.map((url) => `${createTabSpace()}${url}`).join("\n");
+    const errorText = createErrorResult(result.errors);
+
+    if (duplicatesText) {
+        logErrors(duplicatesText, DUPLICATES_TITLE);
+        chatMessage += DUPLICATES_TITLE + duplicatesText;
+    }
+    if (missingText) {
+        logErrors(missingText, MISSING_TITLE);
+        chatMessage += MISSING_TITLE + missingText;
+    }
+    if (errorText) {
+        logErrors(errorText, ERROR_TITLE);
+        chatMessage += ERROR_TITLE + errorText;
+    }
+
+    return chatMessage;
+}
+
+/**
+ * @param webUrl {string}
+ * @param sitemapUrls {string[]}
+ * @param shouldReportMissing {boolean}
+ * @returns {ResultDto}
+ */
+function getResult(webUrl, sitemapUrls, shouldReportMissing) {
+    const shouldBeInSitemap = getPagesShouldBeInSitemap(webUrl);
+    const errors = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === false);
+    const duplicates = [...new Set(sitemapUrls.filter((item, index, self) => self.indexOf(item) !== index))];
+    const missingInSitemap = shouldReportMissing
+        ? shouldBeInSitemap.filter((testedUrl) => !sitemapUrls.includes(testedUrl))
+        : [];
+    const ok = TESTED_URLS.filter((r) => r.status === 200);
+    const skippedUrls = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === true);
+
+    const errorsSum = missingInSitemap.length + duplicates.length + errors.length;
+
+    return { errors, duplicates, missingInSitemap, ok, skippedUrls, errorsSum };
+}
+
 /**
  * @param sitemapUrl {string}
  * @param skip {number}
  * @param withNested {boolean}
  * @param withImages {boolean}
+ * @param checkMissing {boolean}
  * @param googleWebhookUrl {string|undefined}
  * @return {Promise<*>}
  */
-module.exports = async function run(sitemapUrl, skip, withNested, withImages, googleWebhookUrl) {
+module.exports = async function run(sitemapUrl, skip, withNested, withImages, checkMissing, googleWebhookUrl) {
     if (!sitemapUrl) {
         stdout.write("⛔ Required parameter --url is empty.\n");
         return process.exit(1);
@@ -509,56 +586,29 @@ module.exports = async function run(sitemapUrl, skip, withNested, withImages, go
     const url = new URL(sitemapUrl);
     const webUrl = url.origin;
 
-    stdout.write(`${createTabSpace()}Sitemap url: ${sitemapUrl}\n`);
-    stdout.write(`${createTabSpace()}Web url: ${webUrl}\n\n`);
-    if (withNested) {
-        stdout.write(`${createTabSpace()}Will test nested links\n`);
-    }
-    if (withImages) {
-        stdout.write(`${createTabSpace()}Will test images\n\n`);
-    }
+    const shouldReportMissing = checkMissing && withNested;
+
+    logInitialInfo(sitemapUrl, webUrl, withNested, withImages, checkMissing, shouldReportMissing);
 
     const startTime = performance.now();
     const sitemapUrls = await Sitemap.getSitemap(sitemapUrl);
     await testSitemapUrls(sitemapUrls, webUrl, sitemapUrl, skip, withNested, withImages);
     const finishTime = performance.now();
 
-    const shouldBeInSitemap = getPagesShouldBeInSitemap(webUrl);
-
-    const errors = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === false);
-    const duplicates = [...new Set(sitemapUrls.filter((item, index, self) => self.indexOf(item) !== index))];
-    const skippedUrls = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === true);
-    const ok = TESTED_URLS.filter((r) => r.status === 200);
-    const missingInSitemap = shouldBeInSitemap.filter((testedUrl) => !sitemapUrls.includes(testedUrl));
-
-    if (missingInSitemap.length > 0 || duplicates.length > 0 || errors.length > 0 || ERRORS.length > 0) {
-        let chatMessage = "";
-        const duplicatesText = duplicates.map((url) => `${createTabSpace()}${url}`).join("\n");
-        const missingText = missingInSitemap.map((url) => `${createTabSpace()}${url}`).join("\n");
-        const errorText = createErrorResult(errors);
+    const result = getResult(webUrl, sitemapUrls, shouldReportMissing);
 
-        if (duplicatesText) {
-            logErrors(duplicatesText, DUPLICATES_TITLE);
-            chatMessage += DUPLICATES_TITLE + duplicatesText;
-        }
-        if (missingText) {
-            logErrors(missingText, MISSING_TITLE);
-            chatMessage += MISSING_TITLE + missingText;
-        }
-        if (errorText) {
-            logErrors(errorText, ERROR_TITLE);
-            chatMessage += ERROR_TITLE + errorText;
-        }
+    if (result.errorsSum > 0) {
+        const chatMessage = logResultErrors(webUrl, result);
 
         await sendGoogleChatMessage(chatMessage, googleWebhookUrl);
     }
 
-    if (skippedUrls.length > 0) {
-        const skippedUrlsText = createSkippedResult(skippedUrls);
+    if (result.skippedUrls.length > 0) {
+        const skippedUrlsText = createSkippedResult(result.skippedUrls);
         logErrors(skippedUrlsText, "\nSkipped origins:\n");
     }
 
-    logStatistics(ok, Math.ceil(finishTime - startTime));
+    logStatistics(result.ok, Math.ceil(finishTime - startTime));
 
-    process.exit(errors.length > 0 ? 1 : 0);
+    process.exit(result.errorsSum > 0 ? 1 : 0);
 };