@uxf/scripts 1.6.2 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -48,6 +48,9 @@ function shouldIgnoreError(url, status, e) {
|
|
|
48
48
|
if ((status === -1 || status === 302) && url.startsWith("https://www.facebook.com/sharer/")) {
|
|
49
49
|
return true;
|
|
50
50
|
}
|
|
51
|
+
if (status === -3) {
|
|
52
|
+
return true;
|
|
53
|
+
}
|
|
51
54
|
|
|
52
55
|
return false;
|
|
53
56
|
}
|
|
@@ -127,8 +130,8 @@ function createCorrectLinks(incorrectLinks, webUrl) {
|
|
|
127
130
|
*/
|
|
128
131
|
async function fetchUrl(url, parentUrl = undefined, ttl = 1) {
|
|
129
132
|
try {
|
|
130
|
-
Sitemap.axios.defaults.maxRedirects = parentUrl ?
|
|
131
|
-
const encodedUrl = parentUrl ? encodeURI(url) : url;
|
|
133
|
+
Sitemap.axios.defaults.maxRedirects = parentUrl ? 10 : 0;
|
|
134
|
+
const encodedUrl = parentUrl ? encodeURI(decodeURI(url)) : url;
|
|
132
135
|
|
|
133
136
|
const t0 = performance.now();
|
|
134
137
|
const { status } = await Sitemap.axios.get(encodedUrl);
|
|
@@ -225,11 +228,11 @@ async function testAllNestedUrls(parentUrl, parentIndex, webUrl) {
|
|
|
225
228
|
const { data } = await axios.get(parentUrl);
|
|
226
229
|
const $ = cheerio.load(data);
|
|
227
230
|
const urls = createCorrectLinks(
|
|
228
|
-
$("a").map((i, node) => $(node).attr("href")),
|
|
231
|
+
$("a[href]").map((i, node) => $(node).attr("href")),
|
|
229
232
|
webUrl,
|
|
230
233
|
);
|
|
231
234
|
const images = createCorrectLinks(
|
|
232
|
-
$("img").map((i, node) => $(node).attr("src")),
|
|
235
|
+
$("img[src]").map((i, node) => $(node).attr("src")),
|
|
233
236
|
webUrl,
|
|
234
237
|
);
|
|
235
238
|
|