@crawlee/basic 4.0.0-beta.127 → 4.0.0-beta.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/basic-crawler.js +11 -4
- package/package.json +7 -7
|
@@ -833,8 +833,8 @@ export class BasicCrawler {
|
|
|
833
833
|
await this.#autoscaledPool.run();
|
|
834
834
|
}
|
|
835
835
|
finally {
|
|
836
|
-
await this.teardown();
|
|
837
836
|
await this.statistics.stopCapturing();
|
|
837
|
+
await this.teardown();
|
|
838
838
|
process.off('SIGINT', sigintHandler);
|
|
839
839
|
eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
|
|
840
840
|
eventManager.off(EventType.ABORTING, boundPauseOnMigration);
|
|
@@ -1150,7 +1150,9 @@ export class BasicCrawler {
|
|
|
1150
1150
|
// Only log the limit message when an explicit `limit` was passed (not the internal
|
|
1151
1151
|
// `maxRequestsPerCrawl`-derived one), and only once per call.
|
|
1152
1152
|
if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
|
|
1153
|
-
this.log.info(
|
|
1153
|
+
this.log.info(requestLimit === options.limit
|
|
1154
|
+
? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
|
|
1155
|
+
: `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
|
|
1154
1156
|
}
|
|
1155
1157
|
await Promise.all(allSkipped.map(async ({ url, reason }) => {
|
|
1156
1158
|
await this.handleSkippedRequest({ url, reason });
|
|
@@ -1706,11 +1708,16 @@ export class BasicCrawler {
|
|
|
1706
1708
|
* To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
|
|
1707
1709
|
*/
|
|
1708
1710
|
async teardown() {
|
|
1709
|
-
|
|
1711
|
+
// When this crawler initialized the event manager, its close() call emits
|
|
1712
|
+
// the final persistence event after the crawler-specific state has been
|
|
1713
|
+
// saved. External event managers still need an explicit event here.
|
|
1714
|
+
if (!this.#closeEvents) {
|
|
1715
|
+
serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
|
|
1716
|
+
}
|
|
1717
|
+
await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this.#closeEvents ?? false }));
|
|
1710
1718
|
if (this.#closeEvents) {
|
|
1711
1719
|
await serviceLocator.getEventManager().close();
|
|
1712
1720
|
}
|
|
1713
|
-
await this.#sessionPoolDep.ifOwned((pool) => pool.teardown());
|
|
1714
1721
|
await this.#autoscaledPool?.abort();
|
|
1715
1722
|
await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
|
|
1716
1723
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.129",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^2.0.0",
|
|
43
43
|
"@apify/timeout": "^0.4.4",
|
|
44
44
|
"@apify/utilities": "^2.15.5",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.129",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.129",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.129",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.129",
|
|
49
49
|
"csv-stringify": "^6.5.2",
|
|
50
50
|
"tldts": "^7.0.6",
|
|
51
51
|
"tslib": "^2.8.1",
|
|
@@ -53,7 +53,7 @@
|
|
|
53
53
|
"zod": "^4.4.3"
|
|
54
54
|
},
|
|
55
55
|
"optionalDependencies": {
|
|
56
|
-
"@crawlee/impit-client": "^4.0.0-beta.
|
|
56
|
+
"@crawlee/impit-client": "^4.0.0-beta.129"
|
|
57
57
|
},
|
|
58
58
|
"lerna": {
|
|
59
59
|
"command": {
|
|
@@ -62,5 +62,5 @@
|
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
64
|
},
|
|
65
|
-
"gitHead": "
|
|
65
|
+
"gitHead": "b8a2e854bb8620679ca8e9f774fce4932c4e3b51"
|
|
66
66
|
}
|