@crawlee/utils 3.18.3-beta.0 → 3.18.3-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/sitemap.js +5 -3
- package/package.json +3 -3
package/internals/sitemap.js
CHANGED
|
@@ -275,6 +275,10 @@ async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
275
275
|
// URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
|
|
276
276
|
// the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
|
|
277
277
|
let droppedUrlEntries = 0;
|
|
278
|
+
// The same for every entry of this source, so compute it once.
|
|
279
|
+
const originSitemapUrl = source.type === 'url'
|
|
280
|
+
? source.url
|
|
281
|
+
: `raw://${(0, node_crypto_1.createHash)('sha256').update(source.content).digest('base64')}`;
|
|
278
282
|
for await (const item of items) {
|
|
279
283
|
if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
|
|
280
284
|
if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
|
|
@@ -307,9 +311,7 @@ async function* parseSitemap(initialSources, proxyUrl, options) {
|
|
|
307
311
|
}
|
|
308
312
|
yield {
|
|
309
313
|
...item,
|
|
310
|
-
originSitemapUrl
|
|
311
|
-
? source.url
|
|
312
|
-
: `raw://${(0, node_crypto_1.createHash)('sha256').update(source.content).digest('base64')}`,
|
|
314
|
+
originSitemapUrl,
|
|
313
315
|
};
|
|
314
316
|
}
|
|
315
317
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "3.18.3-beta.
|
|
3
|
+
"version": "3.18.3-beta.2",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/log": "^2.4.0",
|
|
51
51
|
"@apify/ps-tree": "^1.2.0",
|
|
52
|
-
"@crawlee/types": "3.18.3-beta.
|
|
52
|
+
"@crawlee/types": "3.18.3-beta.2",
|
|
53
53
|
"@types/sax": "^1.2.7",
|
|
54
54
|
"cheerio": "1.0.0-rc.12",
|
|
55
55
|
"file-type": "^21.3.1",
|
|
@@ -71,5 +71,5 @@
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
},
|
|
74
|
-
"gitHead": "
|
|
74
|
+
"gitHead": "1a8eeb14772ec02c1e01d93b73ea66e07977715b"
|
|
75
75
|
}
|