@crawlee/basic 4.0.0-beta.205 → 4.0.0-beta.207
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/basic-crawler.d.ts +10 -1
- package/package.json +7 -7
|
@@ -222,7 +222,16 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
|
|
|
222
222
|
/**
|
|
223
223
|
* Allows to keep the crawler alive even if the {@link RequestQueue} gets empty.
|
|
224
224
|
* By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
|
|
225
|
-
* waiting for more requests to come.
|
|
225
|
+
* waiting for more requests to come.
|
|
226
|
+
*
|
|
227
|
+
* `crawler.stop()` only stops the crawler from dispatching new requests — it does not wait for or drain
|
|
228
|
+
* requests already sitting in the queue. If this crawler is fed by another crawler or an external producer,
|
|
229
|
+
* calling `stop()` as soon as the producer finishes can abandon requests the producer already enqueued but
|
|
230
|
+
* this crawler had not yet picked up.
|
|
231
|
+
*
|
|
232
|
+
* To shut down only once the queue is genuinely empty and the producer has stopped, override
|
|
233
|
+
* {@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`} to check both
|
|
234
|
+
* `(await requestManager.checkReadiness()).status === 'finished'` and the producer's own running/finished flag.
|
|
226
235
|
*/
|
|
227
236
|
keepAlive?: boolean;
|
|
228
237
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.207",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.0.1",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.207",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.207",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.207",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.207",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.0.0",
|
|
51
51
|
"csv-stringify": "^6.5.2",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.5.4"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.207"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "8d4e54dfe0343e33f40a90d12abfb1840340c520"
|
|
70
70
|
}
|