@crawlee/utils 4.0.0-beta.190 → 4.0.0-beta.192
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/sitemap.js +5 -0
- package/package.json +4 -4
package/internals/sitemap.js
CHANGED
|
@@ -423,6 +423,11 @@ export async function* discoverValidSitemaps(urls, options = {}) {
|
|
|
423
423
|
}
|
|
424
424
|
else {
|
|
425
425
|
const firstUrl = new URL(domainUrls[0]);
|
|
426
|
+
// The guessed candidates are well-known paths on the same origin, so the query string
|
|
427
|
+
// and fragment of the input URL are not part of them - carrying them over probes (and
|
|
428
|
+
// yields) URLs like `/sitemap.xml?page=2`, which servers may well answer with a 404.
|
|
429
|
+
firstUrl.search = '';
|
|
430
|
+
firstUrl.hash = '';
|
|
426
431
|
const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
|
|
427
432
|
const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
|
|
428
433
|
firstUrl.pathname = pathname;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.192",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -44,8 +44,8 @@
|
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@apify/ps-tree": "^1.2.0",
|
|
46
46
|
"@apify/validations": "^1.1.0",
|
|
47
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
48
|
-
"@crawlee/types": "4.0.0-beta.
|
|
47
|
+
"@crawlee/http-client": "4.0.0-beta.192",
|
|
48
|
+
"@crawlee/types": "4.0.0-beta.192",
|
|
49
49
|
"@types/sax": "^1.2.7",
|
|
50
50
|
"cheerio": "^1.0.0",
|
|
51
51
|
"domhandler": "^5.0.3",
|
|
@@ -64,5 +64,5 @@
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
},
|
|
67
|
-
"gitHead": "
|
|
67
|
+
"gitHead": "729823422c7168238d8b5fcf38dcb798cbf0af0b"
|
|
68
68
|
}
|