@crawlee/core 3.17.1-beta.60 → 3.17.1-beta.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.17.1-beta.
|
|
3
|
+
"version": "3.17.1-beta.61",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.4.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.17.1-beta.
|
|
63
|
-
"@crawlee/types": "3.17.1-beta.
|
|
64
|
-
"@crawlee/utils": "3.17.1-beta.
|
|
62
|
+
"@crawlee/memory-storage": "3.17.1-beta.61",
|
|
63
|
+
"@crawlee/types": "3.17.1-beta.61",
|
|
64
|
+
"@crawlee/utils": "3.17.1-beta.61",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@standard-schema/spec": "^1.0.0",
|
|
67
67
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
@@ -84,5 +84,5 @@
|
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
},
|
|
87
|
-
"gitHead": "
|
|
87
|
+
"gitHead": "474ef1ca7e3f599d2159c67abbd26817d472bc11"
|
|
88
88
|
}
|
|
@@ -410,14 +410,21 @@ class SitemapRequestList {
|
|
|
410
410
|
}
|
|
411
411
|
// Create a new stream, as we have read all the URLs from the current one.
|
|
412
412
|
// Pushing the urls back to the original stream might not be possible if it has been ended.
|
|
413
|
-
const
|
|
413
|
+
const previousStream = this.urlQueueStream;
|
|
414
|
+
const newStream = this.createNewStream(previousStream.readableHighWaterMark);
|
|
414
415
|
for (const url of urlQueue) {
|
|
415
416
|
newStream.push(url);
|
|
416
417
|
}
|
|
417
|
-
if (
|
|
418
|
+
if (previousStream.writableEnded) {
|
|
418
419
|
newStream.end();
|
|
419
420
|
}
|
|
420
421
|
this.urlQueueStream = newStream;
|
|
422
|
+
// A `pushNextUrl()` call may be blocked on backpressure, waiting for a `readdata` event on the
|
|
423
|
+
// previous stream. That event is only ever emitted by `readNextUrl()` on the current stream, so
|
|
424
|
+
// after the swap the waiter would never be notified and the background sitemap loading would hang.
|
|
425
|
+
// Re-emit `readdata` on the previous stream to release any such pending waiter (its URL has already
|
|
426
|
+
// been transferred to the new stream above).
|
|
427
|
+
previousStream.emit('readdata');
|
|
421
428
|
await this.store.setValue(this.persistStateKey, {
|
|
422
429
|
sitemapParsingProgress: {
|
|
423
430
|
pendingSitemapUrls: Array.from(this.sitemapParsingProgress.pendingSitemapUrls),
|