@uxf/scripts 11.62.2 → 11.62.3

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@uxf/scripts",
-    "version": "11.62.2",
+    "version": "11.62.3",
     "description": "",
     "main": "index.js",
     "bin": {

package/src/uxf-sitemap-check/index.js

@@ -15,7 +15,7 @@ const { HTTP_USERNAME, HTTP_PASSWORD } = process.env;
 
 const DUPLICATES_TITLE = "\n\n\nDuplicated pages in sitemap:\n";
 const MISSING_TITLE = "\n\n\nMissing pages in sitemap:\n";
-const ERROR_TITLE = "\n\n\nErrors:\n"
+const ERROR_TITLE = "\n\n\nErrors:\n";
 
 const MAX_TTL = 3;
 const IMAGES_LABEL = "🏞 Images:";
@@ -29,6 +29,24 @@ const URLS_TO_CHECK = new Set();
 
 const robotsParser = robotsTxtParser({ userAgent: "uxf-bot", allowOnNeutral: false });
 
+const HOSTNAME_ROBOTS_MAP = {
+    "fb.me": "facebook.com",
+};
+
+/**
+ * @param url {string}
+ * @returns {string}
+ */
+function getUrlOrigin(url) {
+    const urlObject = new URL(url);
+
+    if (urlObject.hostname in HOSTNAME_ROBOTS_MAP) {
+        return new URL(url.replace(urlObject.hostname, HOSTNAME_ROBOTS_MAP[urlObject.hostname])).origin;
+    }
+
+    return urlObject.origin;
+}
+
 /**
  * @param url {string}
  * @param options {{redirect: boolean, isExternal: boolean}}
@@ -184,7 +202,7 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
     }
 
     try {
-        const origin = new URL(url).origin;
+        const origin = getUrlOrigin(url);
 
         if (parentUrl && origin !== webUrl) {
             await robotsParser.useRobotsFor(origin);
@@ -539,13 +557,15 @@ function getResult(webUrl, sitemapUrls, shouldReportMissing) {
     const shouldBeInSitemap = getPagesShouldBeInSitemap(webUrl);
     const errors = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === false);
     const duplicates = [...new Set(sitemapUrls.filter((item, index, self) => self.indexOf(item) !== index))];
-    const missingInSitemap = shouldReportMissing ? shouldBeInSitemap.filter((testedUrl) => !sitemapUrls.includes(testedUrl)) : [];
+    const missingInSitemap = shouldReportMissing
+        ? shouldBeInSitemap.filter((testedUrl) => !sitemapUrls.includes(testedUrl))
+        : [];
     const ok = TESTED_URLS.filter((r) => r.status === 200);
     const skippedUrls = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === true);
 
     const errorsSum = missingInSitemap.length + duplicates.length + errors.length;
 
-    return {errors, duplicates, missingInSitemap, ok, skippedUrls, errorsSum};
+    return { errors, duplicates, missingInSitemap, ok, skippedUrls, errorsSum };
 }
 
 /**