@uxf/scripts 11.61.5 → 11.62.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@uxf/scripts",
-    "version": "11.61.5",
+    "version": "11.62.0",
     "description": "",
     "main": "index.js",
     "bin": {

package/src/uxf-sitemap-check/index.js

@@ -8,13 +8,20 @@ const robotsTxtParser = require("robots-txt-parser");
 const { HTTP_USERNAME, HTTP_PASSWORD } = process.env;
 
 /**
- * @typedef {{parentUrl: (string | undefined), isImg: boolean, ttl: number, url: string, status: number, message: (string | undefined), skipped: boolean, html: (string | null)}} UrlCheckResponse
+ * @typedef {{parentUrl: (string | undefined), isImg: boolean, isWebPage: boolean, ttl: number, url: string, status: number, message: (string | undefined), skipped: boolean, html: (string | null), redirected: boolean}} UrlCheckResponse
+ */
+
+/**
+ * @typedef {{url: string, parentUrl: (string | undefined), canonicalUrl: (string | null), isImg: boolean, isWebPage: boolean, ttl: number, status: number, message: (string | undefined), skipped: boolean, indexable: boolean, redirected: boolean}} TestedUrlDto
  */
 
 const MAX_TTL = 3;
 const IMAGES_LABEL = "🏞 Images:";
 const URLS_LABEL = "🔗 Links:";
 
+/**
+ * @type TestedUrlDto[]
+ */
 const TESTED_URLS = [];
 const URLS_TO_CHECK = new Set();
 const ERRORS = [];
@@ -172,11 +179,13 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
             url,
             parentUrl,
             isImg: isImageUrl(url),
+            isWebPage: false,
             ttl,
             status: 0,
             message: "invalid url: " + url,
             skipped: false,
             html: null,
+            redirected: false,
         };
     }
 
@@ -193,11 +202,13 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
                     url,
                     parentUrl,
                     isImg: isImageUrl(url),
+                    isWebPage: true,
                     ttl,
                     status: 0,
                     message: "blocked by robots.txt",
                     skipped: true,
                     html: null,
+                    redirected: false,
                 };
             }
         }
@@ -210,11 +221,13 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
             url,
             parentUrl,
             isImg: isImageUrl(url),
+            isWebPage: true,
             ttl,
             status: errorStatus,
             message: e.message,
             skipped: errorStatus !== undefined,
             html: null,
+            redirected: false,
         };
     }
 
@@ -229,10 +242,12 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
             url,
             parentUrl,
             isImg: isImageUrl(url),
+            isWebPage: response.headers.get("content-type").includes("text/html"),
             ttl,
             status: response.status,
             skipped: false,
             html: await response.text(),
+            redirected: response.redirected,
         };
     } catch (e) {
         const status = Number.parseInt((e && e.response && e.response.status) || -1, 10);
@@ -241,11 +256,13 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
             url,
             parentUrl,
             isImg: isImageUrl(url),
+            isWebPage: true,
             ttl,
             status,
             message: e.message,
             skipped: false,
             html: null,
+            redirected: false,
         };
     }
 }
@@ -254,7 +271,7 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
  * @param url {string}
  * @param webUrl {string}
  * @param parentUrl {string | undefined}
- * @return {UrlCheckResponse}
+ * @return {result: UrlCheckResponse|TestedUrlDto, fromCache: boolean}
  */
 async function testUrl(url, webUrl, parentUrl = undefined) {
     const indexInChecked = TESTED_URLS.findIndex((result) => result.url === url);
@@ -262,16 +279,26 @@ async function testUrl(url, webUrl, parentUrl = undefined) {
         const result = await fetchUrl(url, webUrl, parentUrl);
         TESTED_URLS.push({
             isImg: result.isImg,
+            isWebPage: result.isWebPage,
             message: result.message,
             parentUrl: result.parentUrl,
             skipped: result.skipped,
             status: result.status,
             ttl: result.ttl,
             url: result.url,
+            redirected: result.redirected,
+            indexable:
+                result.isWebPage && typeof result.html === "string"
+                    ? cheerio.load(result.html)("meta[name='robots']").attr("content") !== "noindex"
+                    : false,
+            canonicalUrl:
+                result.isWebPage && typeof result.html === "string"
+                    ? (cheerio.load(result.html)("link[rel='canonical']").attr("href") ?? null)
+                    : null,
         });
-        return result;
+        return { result, fromCache: false };
     }
-    return TESTED_URLS[indexInChecked];
+    return { result: TESTED_URLS[indexInChecked], fromCache: true };
 }
 
 /**
@@ -292,14 +319,14 @@ async function testSitemapUrls(urls, webUrl, sitemapUrl, skip, withNested, withI
         const url = urls[i];
         const changedUrl = webUrl ? `${webUrl}${new URL(url).pathname}` : url;
 
-        const result = await testUrl(changedUrl, webUrl);
+        const { result, fromCache } = await testUrl(changedUrl, webUrl);
         printProgress();
 
-        if (withNested && result.status === 200) {
+        if (withNested && !fromCache && result.status === 200) {
             await testNestedUrls(result.html, changedUrl, i, webUrl ?? sitemapUrl.split("/").slice(0, 3).join("/"));
         }
 
-        if (withImages && result.status === 200) {
+        if (withImages && !fromCache && result.status === 200) {
             await testNestedImages(result.html, changedUrl, i, webUrl ?? sitemapUrl.split("/").slice(0, 3).join("/"));
         }
     }
@@ -314,12 +341,15 @@ async function testSitemapUrls(urls, webUrl, sitemapUrl, skip, withNested, withI
  */
 async function testNestedUrls(html, parentUrl, parentIndex, webUrl) {
     try {
-        const $ = cheerio.load(html);
-        const urls = createCorrectLinks(
+        const $ = cheerio.load(html.toString());
+        let urls = createCorrectLinks(
             $("a[href]").map((i, node) => $(node).attr("href")),
             webUrl,
         );
 
+        // FIXME
+        urls = urls.filter((url) => url.startsWith(webUrl) || url.startsWith("/"));
+
         await testNested(urls, parentIndex, parentUrl, createTabSpace() + URLS_LABEL, webUrl);
     } catch (e) {
         ERRORS.push(`Can't test all nested pages for ${parentUrl} - ${e.message}`);
@@ -443,6 +473,24 @@ async function sendGoogleChatMessage(resultErrors, webhookUrl) {
     );
 }
 
+/**
+ * @param webUrl {string}
+ * @return {string[]}
+ */
+function getPagesShouldBeInSitemap(webUrl) {
+    return TESTED_URLS.filter((u) => u.indexable)
+        .filter(
+            (dto) =>
+                (dto.canonicalUrl ?? dto.url).startsWith(webUrl) &&
+                !(dto.canonicalUrl ?? dto.url).includes("?") &&
+                dto.status === 200 &&
+                dto.ttl <= 1 &&
+                !dto.redirected &&
+                dto.isWebPage,
+        )
+        .map((url) => (url.canonicalUrl ?? url.url).toLowerCase());
+}
+
 /**
  * @param sitemapUrl {string}
  * @param skip {number}
@@ -470,17 +518,32 @@ module.exports = async function run(sitemapUrl, skip, withNested, withImages, go
     }
 
     const startTime = performance.now();
-    await testSitemapUrls(await Sitemap.getSitemap(sitemapUrl), webUrl, sitemapUrl, skip, withNested, withImages);
+    const sitemapUrls = await Sitemap.getSitemap(sitemapUrl);
+    await testSitemapUrls(sitemapUrls, webUrl, sitemapUrl, skip, withNested, withImages);
     const finishTime = performance.now();
 
+    const shouldBeInSitemap = getPagesShouldBeInSitemap(webUrl);
+
     const errors = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === false);
+    const duplicates = [...new Set(sitemapUrls.filter((item, index, self) => self.indexOf(item) !== index))];
     const skippedUrls = TESTED_URLS.filter((r) => r.status !== 200 && r.skipped === true);
     const ok = TESTED_URLS.filter((r) => r.status === 200);
+    const missingInSitemap = shouldBeInSitemap.filter((testedUrl) => !sitemapUrls.includes(testedUrl));
 
-    if (errors.length > 0 || ERRORS.length > 0) {
+    if (missingInSitemap.length > 0 || duplicates.length > 0 || errors.length > 0 || ERRORS.length > 0) {
+        const duplicatesText = duplicates.map((url) => `${createTabSpace()}${url}`).join("\n");
+        const missingText = missingInSitemap.map((url) => `${createTabSpace()}${url}`).join("\n");
         const errorText = createErrorResult(errors);
-        logErrors(errorText, "\n\n\nErrors:\n");
-        await sendGoogleChatMessage(errorText, googleWebhookUrl);
+        if (duplicatesText) {
+            logErrors(duplicatesText, "\n\n\nDuplicated pages in sitemap:\n");
+        }
+        if (missingText) {
+            logErrors(missingText, "\n\n\nMissing pages in sitemap:\n");
+        }
+        if (errorText) {
+            logErrors(errorText, "\n\n\nErrors:\n");
+        }
+        await sendGoogleChatMessage(duplicatesText + missingText + errorText, googleWebhookUrl);
     }
 
     if (skippedUrls.length > 0) {