@uxf/scripts 1.5.5 → 1.6.1

package/.gitlab-ci.yml

@@ -2,7 +2,7 @@ stages:
   - release
 
 release:
-  image: node:16.9.1-buster-slim
+  image: node:18-buster-slim
   stage: release
   tags:
     - pluto-docker

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uxf/scripts",
-  "version": "1.5.5",
+  "version": "1.6.1",
   "description": "",
   "main": "index.js",
   "bin": {

package/src/Sitemap.js

@@ -27,6 +27,28 @@ const axios = create({
     withCredentials: true,
     maxRedirects: 0,
     timeout: 20000,
+    headers: {
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+        'DNT': '1',
+        'Pragma': 'no-cache',
+        'Sec-Ch-Ua': '"Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"',
+        'Sec-Ch-Ua-Arch': '"x86"',
+        'Sec-Ch-Ua-Mobile': '?0',
+        'Sec-Ch-Ua-Platform': '"Windows"',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'cross-site',
+        'Sec-Fetch-User': '?1',
+        'Sec-Fetch-User-Agent': '?1',
+        'TE': 'Trailers',
+        'Upgrade-Insecure-Requests': '1',
+        'X-Requested-With': 'XMLHttpRequest',
+    },
 });
 
 module.exports = {

package/src/uxf-sitemap-check/index.js

@@ -8,7 +8,7 @@ const GoogleChat = require("../GoogleChat");
 
 /**
  *
- * @typedef {{parentUrl: (string | undefined), isImg: boolean, time: number, ttl: number, url: string, status: number, message: (string | undefined)}} UrlCheckResponse
+ * @typedef {{parentUrl: (string | undefined), isImg: boolean, time: number, ttl: number, url: string, status: number, message: (string | undefined), shouldIgnoreError: (boolean | undefined)}} UrlCheckResponse
  */
 
 const MAX_TTL = 3;
@@ -34,6 +34,24 @@ function isImageUrl(url) {
     return new RegExp("[a-z].*(\\.jpg|\\.png|\\.webp|\\.avif|\\.gif)$", "gim").test(url);
 }
 
+/**
+ *
+ * @param url {string}
+ * @param status {number}
+ * @param e {Error}
+ * @returns {boolean}
+ */
+function shouldIgnoreError(url, status, e) {
+    if (status === 999 && url.startsWith("https://www.linkedin.com")) {
+        return true;
+    }
+    if ((status === -1 || status === 302) && url.startsWith("https://www.facebook.com/sharer/")) {
+        return true;
+    }
+
+    return false;
+}
+
 /**
  *
  * @param errors {UrlCheckResponse[]}
@@ -109,14 +127,15 @@ function createCorrectLinks(incorrectLinks, webUrl) {
  */
 async function fetchUrl(url, parentUrl = undefined, ttl = 1) {
     try {
-        Sitemap.axios.defaults.maxRedirects = parentUrl ? 1 : 0;
+        Sitemap.axios.defaults.maxRedirects = parentUrl ? 3 : 0;
+        const encodedUrl = parentUrl ? encodeURI(url) : url;
 
         const t0 = performance.now();
-        const { status } = await Sitemap.axios.get(url);
+        const { status } = await Sitemap.axios.get(encodedUrl);
         const t1 = performance.now();
 
         if (status !== 200 && ttl < MAX_TTL) {
-            return await fetchUrl(url, parentUrl, ttl + 1);
+            return await fetchUrl(encodedUrl, parentUrl, ttl + 1);
         }
 
         return {
@@ -128,14 +147,28 @@ async function fetchUrl(url, parentUrl = undefined, ttl = 1) {
             time: Math.ceil(t1 - t0),
         };
     } catch (e) {
+        const status = Number.parseInt((e && e.response && e.response.status) || -1, 10);
+
+        if (shouldIgnoreError(url, status, e)) {
+            return {
+                url,
+                parentUrl,
+                isImg: isImageUrl(url),
+                ttl,
+                status,
+                shouldIgnoreError: true,
+                time: 0,
+            };
+        }
         return {
             url,
             parentUrl,
             isImg: isImageUrl(url),
             ttl,
-            status: Number.parseInt((e && e.response && e.response.status) || -1, 10),
+            status,
             time: 0,
             message: e.message,
+            shouldIgnoreError: false,
         };
     }
 }
@@ -251,9 +284,10 @@ function printUrlResult(result) {
 /**
  *
  * @param errorText {string}
+ * @param title {string}
  */
-function logErrors(errorText) {
-    stdout.write("\nErrors:\n");
+function logErrors(errorText, title) {
+    stdout.write(title);
     stdout.write(`${errorText}\n\n`);
 }
 
@@ -275,8 +309,8 @@ function convertTime(millis) {
  */
 function logStatistics(okResults, time) {
     const avgTime = Math.round(okResults.reduce((prev, curr) => prev + curr.time, 0) / TESTED_URLS.length);
-    const maxTime = okResults.reduce((prev, curr) => (curr.time > prev.time ? curr : prev));
-    const minTime = okResults.reduce((prev, curr) => (curr.time < prev.time ? curr : prev));
+    const maxTime = okResults.reduce((prev, curr) => (curr.time > prev.time ? curr : prev), []);
+    const minTime = okResults.reduce((prev, curr) => (curr.time < prev.time ? curr : prev), []);
 
     stdout.write("\nSummary:\n");
     stdout.write(createTabSpace() + `Time ${convertTime(time)}\n`);
@@ -350,15 +384,24 @@ module.exports = async function run(sitemapUrl, webUrl, slackChannel, skip, test
     await testSitemapUrls((await Sitemap.getSitemap(sitemapUrl)), webUrl, sitemapUrl, skip, testNested);
     const finishTime = performance.now();
 
-    const errors = TESTED_URLS.filter(r => r.status !== 200);
+    const errors = TESTED_URLS.filter(r => r.status !== 200 && r.shouldIgnoreError === false);
+    const ignoredErrors = TESTED_URLS.filter(r => r.status !== 200 && r.shouldIgnoreError === true);
     const ok = TESTED_URLS.filter(r => r.status === 200);
 
     if (errors.length > 0) {
         const errorText = createErrorResult(errors);
-        logErrors(errorText);
+        logErrors(errorText, "\nErrors:\n");
+        const ignoredErrorText = createErrorResult(ignoredErrors);
+        logErrors(ignoredErrorText, "\nIngored errors:\n");
         await sendSlackMessage(errorText, slackChannel);
-        await sendGoogleChatMessage(errors);
+        await sendGoogleChatMessage(errorText);
     }
+
+    if (ignoredErrors.length > 0) {
+        const ignoredErrorText = createErrorResult(ignoredErrors);
+        logErrors(ignoredErrorText, "\nIngored errors:\n");
+    }
+
     logStatistics(ok, Math.ceil(finishTime - startTime));
 
     process.exit(errors.length > 0 ? 1 : 0);