@crawlee/basic 4.0.0-beta.127 → 4.0.0-beta.129

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -833,8 +833,8 @@ export class BasicCrawler {
833
833
  await this.#autoscaledPool.run();
834
834
  }
835
835
  finally {
836
- await this.teardown();
837
836
  await this.statistics.stopCapturing();
837
+ await this.teardown();
838
838
  process.off('SIGINT', sigintHandler);
839
839
  eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
840
840
  eventManager.off(EventType.ABORTING, boundPauseOnMigration);
@@ -1150,7 +1150,9 @@ export class BasicCrawler {
1150
1150
  // Only log the limit message when an explicit `limit` was passed (not the internal
1151
1151
  // `maxRequestsPerCrawl`-derived one), and only once per call.
1152
1152
  if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
1153
- this.log.info(`Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`);
1153
+ this.log.info(requestLimit === options.limit
1154
+ ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
1155
+ : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
1154
1156
  }
1155
1157
  await Promise.all(allSkipped.map(async ({ url, reason }) => {
1156
1158
  await this.handleSkippedRequest({ url, reason });
@@ -1706,11 +1708,16 @@ export class BasicCrawler {
1706
1708
  * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
1707
1709
  */
1708
1710
  async teardown() {
1709
- serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
1711
+ // When this crawler initialized the event manager, its close() call emits
1712
+ // the final persistence event after the crawler-specific state has been
1713
+ // saved. External event managers still need an explicit event here.
1714
+ if (!this.#closeEvents) {
1715
+ serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
1716
+ }
1717
+ await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this.#closeEvents ?? false }));
1710
1718
  if (this.#closeEvents) {
1711
1719
  await serviceLocator.getEventManager().close();
1712
1720
  }
1713
- await this.#sessionPoolDep.ifOwned((pool) => pool.teardown());
1714
1721
  await this.#autoscaledPool?.abort();
1715
1722
  await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
1716
1723
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.127",
3
+ "version": "4.0.0-beta.129",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^2.0.0",
43
43
  "@apify/timeout": "^0.4.4",
44
44
  "@apify/utilities": "^2.15.5",
45
- "@crawlee/core": "4.0.0-beta.127",
46
- "@crawlee/http-client": "4.0.0-beta.127",
47
- "@crawlee/types": "4.0.0-beta.127",
48
- "@crawlee/utils": "4.0.0-beta.127",
45
+ "@crawlee/core": "4.0.0-beta.129",
46
+ "@crawlee/http-client": "4.0.0-beta.129",
47
+ "@crawlee/types": "4.0.0-beta.129",
48
+ "@crawlee/utils": "4.0.0-beta.129",
49
49
  "csv-stringify": "^6.5.2",
50
50
  "tldts": "^7.0.6",
51
51
  "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
53
53
  "zod": "^4.4.3"
54
54
  },
55
55
  "optionalDependencies": {
56
- "@crawlee/impit-client": "^4.0.0-beta.127"
56
+ "@crawlee/impit-client": "^4.0.0-beta.129"
57
57
  },
58
58
  "lerna": {
59
59
  "command": {
@@ -62,5 +62,5 @@
62
62
  }
63
63
  }
64
64
  },
65
- "gitHead": "4aa4a4d8d105bb530649ac5cd9197a3169106bd9"
65
+ "gitHead": "b8a2e854bb8620679ca8e9f774fce4932c4e3b51"
66
66
  }