@uxf/scripts 1.6.2 → 1.6.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uxf/scripts",
-  "version": "1.6.2",
+  "version": "1.6.4",
   "description": "",
   "main": "index.js",
   "bin": {

package/src/Sitemap.js

@@ -33,6 +33,7 @@ const axios = create({
     maxRedirects: 0,
     timeout: 20000,
     httpsAgent: agent,
+    decompress: false,
     headers: {
         'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',

package/src/uxf-sitemap-check/index.js

@@ -48,6 +48,9 @@ function shouldIgnoreError(url, status, e) {
     if ((status === -1 || status === 302) && url.startsWith("https://www.facebook.com/sharer/")) {
         return true;
     }
+    if (status === -3) {
+        return true;
+    }
 
     return false;
 }
@@ -127,8 +130,8 @@ function createCorrectLinks(incorrectLinks, webUrl) {
  */
 async function fetchUrl(url, parentUrl = undefined, ttl = 1) {
     try {
-        Sitemap.axios.defaults.maxRedirects = parentUrl ? 3 : 0;
-        const encodedUrl = parentUrl ? encodeURI(url) : url;
+        Sitemap.axios.defaults.maxRedirects = parentUrl ? 10 : 0;
+        const encodedUrl = parentUrl ? encodeURI(decodeURI(url)) : url;
 
         const t0 = performance.now();
         const { status } = await Sitemap.axios.get(encodedUrl);
@@ -225,11 +228,11 @@ async function testAllNestedUrls(parentUrl, parentIndex, webUrl) {
     const { data } = await axios.get(parentUrl);
     const $ = cheerio.load(data);
     const urls = createCorrectLinks(
-        $("a").map((i, node) => $(node).attr("href")),
+        $("a[href]").map((i, node) => $(node).attr("href")),
         webUrl,
     );
     const images = createCorrectLinks(
-        $("img").map((i, node) => $(node).attr("src")),
+        $("img[src]").map((i, node) => $(node).attr("src")),
         webUrl,
     );