@uxf/scripts 11.63.0 → 11.64.1

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@uxf/scripts",
-    "version": "11.63.0",
+    "version": "11.64.1",
     "description": "",
     "main": "index.js",
     "bin": {
@@ -33,6 +33,7 @@
         "cheerio": "1.0.0",
         "dayjs": "1.11.13",
         "fast-glob": "3.3.2",
+        "got": "14.4.7",
         "madge": "8.0.0",
         "robots-txt-parser": "2.0.3",
         "yargs": "17.7.2"

package/src/uxf-sitemap-check/index.js

@@ -5,6 +5,8 @@ const cheerio = require("cheerio");
 const GoogleChat = require("../GoogleChat");
 const robotsTxtParser = require("robots-txt-parser");
 
+const got = (url, init) => import("got").then((mod) => mod.default(url, init));
+
 const { HTTP_USERNAME, HTTP_PASSWORD } = process.env;
 
 /**
@@ -17,7 +19,7 @@ const DUPLICATES_TITLE = "\n\n\nDuplicated pages in sitemap:\n";
 const MISSING_TITLE = "\n\n\nMissing pages in sitemap:\n";
 const ERROR_TITLE = "\n\n\nErrors:\n";
 
-const MAX_TTL = 3;
+const MAX_TTL = 1;
 const IMAGES_LABEL = "🏞 Images:";
 const URLS_LABEL = "🔗 Links:";
 
@@ -50,17 +52,42 @@ function getUrlOrigin(url) {
 /**
  * @param url {string}
  * @param options {{redirect: boolean, isExternal: boolean}}
- * @returns {Promise<Response>}
+ * @returns {Promise<import('got').Response<string>>}
  */
 function fetcher(url, options) {
-    return fetch(url, {
-        credentials: "include",
-        headers:
-            !options.isExternal && HTTP_USERNAME && HTTP_PASSWORD
-                ? new Headers({ Authorization: "Basic " + btoa(`${HTTP_USERNAME}:${HTTP_PASSWORD}`) })
-                : undefined,
-        redirect: options.redirect ? "follow" : "manual",
-        signal: AbortSignal.timeout(20_000),
+    const shouldUseBasicAuth = !options.isExternal && HTTP_USERNAME && HTTP_PASSWORD;
+
+    const headers = new Headers({
+        "User-Agent":
+            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+        "Accept-Encoding": "gzip, deflate, br",
+        "Accept-Language": "en-US,en;q=0.9,cs-CZ;q=0.8,cs;q=0.7,de;q=0.6",
+        "Cache-Control": "no-cache",
+        Connection: "keep-alive",
+        Pragma: "no-cache",
+        "Sec-Ch-Ua": '"Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"',
+        "Sec-Ch-Ua-Arch": '"x86"',
+        "Sec-Ch-Ua-Mobile": "?0",
+        "Sec-Ch-Ua-Platform": '"Windows"',
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "cross-site",
+        "Sec-Fetch-User": "?1",
+        "Sec-Fetch-User-Agent": "?1",
+    });
+
+    return got(url, {
+        throwHttpErrors: false,
+        decompress: false,
+        https: {
+            rejectUnauthorized: false,
+        },
+        headers,
+        username: shouldUseBasicAuth ? HTTP_USERNAME : undefined,
+        password: shouldUseBasicAuth ? HTTP_PASSWORD : undefined,
+        followRedirect: options.redirect,
+        signal: AbortSignal.timeout(30_000),
     });
 }
 
@@ -226,7 +253,7 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
         }
     } catch (e) {
         const errorStatus = await fetcher(new URL(url).origin + "/robots.txt", { isExternal: true, redirect: true })
-            .then((res) => (res.status === 200 ? -1 : res.status))
+            .then((res) => (res.statusCode === 200 ? -1 : res.statusCode))
             .catch((e) => e.response?.status);
 
         return {
@@ -237,7 +264,7 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
             ttl,
             status: errorStatus,
             message: e.message,
-            skipped: errorStatus !== undefined,
+            skipped: true,
             html: null,
             redirected: false,
         };
@@ -246,7 +273,22 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
     try {
         const response = await fetcher(url, { redirect: !!parentUrl, isExternal: !url.includes(webUrl) });
 
-        if (response.status !== 200 && ttl < MAX_TTL) {
+        if (response.statusCode === 403 && response.headers["server"] === "cloudflare") {
+            return {
+                url,
+                parentUrl,
+                isImg: isImageUrl(url),
+                isWebPage: true,
+                ttl,
+                status: 0,
+                message: "blocked by server",
+                skipped: true,
+                html: null,
+                redirected: false,
+            };
+        }
+
+        if (response.statusCode !== 200 && ttl < MAX_TTL) {
             return await fetchUrl(url, webUrl, parentUrl, ttl + 1);
         }
 
@@ -254,12 +296,12 @@ async function fetchUrl(url, webUrl, parentUrl = undefined, ttl = 1) {
             url,
             parentUrl,
             isImg: isImageUrl(url),
-            isWebPage: response.headers.get("content-type")?.includes("text/html") ?? true,
+            isWebPage: response.headers["content-type"]?.includes("text/html") ?? true,
             ttl,
-            status: response.status,
+            status: response.statusCode,
             skipped: false,
-            html: await response.text(),
-            redirected: response.redirected,
+            html: response.body,
+            redirected: response.redirectUrls.length > 0,
         };
     } catch (e) {
         const status = Number.parseInt((e && e.response && e.response.status) || -1, 10);
@@ -301,7 +343,7 @@ async function testUrl(url, webUrl, parentUrl = undefined) {
             redirected: result.redirected,
             indexable:
                 result.isWebPage && typeof result.html === "string"
-                    ? cheerio.load(result.html)("meta[name='robots']").attr("content") !== "noindex"
+                    ? !cheerio.load(result.html)("meta[name='robots']").attr("content")?.includes("noindex")
                     : false,
             canonicalUrl:
                 result.isWebPage && typeof result.html === "string"
@@ -590,25 +632,33 @@ module.exports = async function run(sitemapUrl, skip, withNested, withImages, ch
 
     logInitialInfo(sitemapUrl, webUrl, withNested, withImages, checkMissing, shouldReportMissing);
 
-    const startTime = performance.now();
-    const sitemapUrls = await Sitemap.getSitemap(sitemapUrl);
-    await testSitemapUrls(sitemapUrls, webUrl, sitemapUrl, skip, withNested, withImages);
-    const finishTime = performance.now();
+    try {
+        const startTime = performance.now();
+        const sitemapUrls = await Sitemap.getSitemap(sitemapUrl);
+        await testSitemapUrls(sitemapUrls, webUrl, sitemapUrl, skip, withNested, withImages);
+        const finishTime = performance.now();
 
-    const result = getResult(webUrl, sitemapUrls, shouldReportMissing);
+        const result = getResult(webUrl, sitemapUrls, shouldReportMissing);
 
-    if (result.errorsSum > 0) {
-        const chatMessage = logResultErrors(webUrl, result);
+        if (result.errorsSum > 0) {
+            const chatMessage = logResultErrors(webUrl, result);
 
-        await sendGoogleChatMessage(chatMessage, googleWebhookUrl);
-    }
+            await sendGoogleChatMessage(chatMessage, googleWebhookUrl);
+        }
 
-    if (result.skippedUrls.length > 0) {
-        const skippedUrlsText = createSkippedResult(result.skippedUrls);
-        logErrors(skippedUrlsText, "\nSkipped origins:\n");
-    }
+        if (result.skippedUrls.length > 0) {
+            const skippedUrlsText = createSkippedResult(result.skippedUrls);
+            logErrors(skippedUrlsText, "\nSkipped origins:\n");
+        }
 
-    logStatistics(result.ok, Math.ceil(finishTime - startTime));
+        logStatistics(result.ok, Math.ceil(finishTime - startTime));
 
-    process.exit(result.errorsSum > 0 ? 1 : 0);
+        process.exit(result.errorsSum > 0 ? 1 : 0);
+    } catch (e) {
+        stdout.write("⛔ Error: " + e.message + "\n");
+
+        await sendGoogleChatMessage(`Sitemap check failed completely:\n\n${e.message}`, googleWebhookUrl);
+
+        process.exit(1);
+    }
 };