@crawlee/basic 4.0.0-beta.126 → 4.0.0-beta.128

package/internals/basic-crawler.d.ts

@@ -282,7 +282,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * implementing {@link IStatistics}.
      *
      * Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
-     * {@link BasicCrawler.stats|`crawler.stats.state`}:
+     * {@link BasicCrawler.statistics|`crawler.statistics.state`}:
      *
      * ```ts
      * const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
@@ -295,8 +295,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * });
      *
      * await crawler.run();
-     * // the custom fields are typed on `crawler.stats` too
-     * console.log(crawler.stats.state.productsFound);
+     * // the custom fields are typed on `crawler.statistics` too
+     * console.log(crawler.statistics.state.productsFound);
      * ```
      */
     statistics?: IStatistics<StatisticStateExtension>;
@@ -464,7 +464,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    get stats(): IStatistics<StatisticStateExtension>;
+    get statistics(): IStatistics<StatisticStateExtension>;
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily

package/internals/basic-crawler.js

@@ -84,14 +84,14 @@ export class BasicCrawler {
      * Used to detect and warn about multiple crawlers sharing the same state.
      */
     static #useStateAnonymousIndices = new Set();
-    /** Backs the {@link BasicCrawler.stats|`stats`} getter. */
-    #statsDep;
+    /** Backs the {@link BasicCrawler.statistics|`statistics`} getter. */
+    #statisticsDep;
     /**
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    get stats() {
-        return this.#statsDep.value;
+    get statistics() {
+        return this.#statisticsDep.value;
     }
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
@@ -201,7 +201,7 @@ export class BasicCrawler {
     #sameDomainDelaySecs;
     maxRequestsPerCrawl;
     get handledRequestsCount() {
-        return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
+        return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;
     }
     #statusMessageLoggingInterval;
     #statusMessageCallback;
@@ -387,7 +387,7 @@ export class BasicCrawler {
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
             this.#sameDomainDelaySecs = sameDomainDelaySecs;
-            this.#statsDep = OwnedOrInjected.resolve(statistics, 
+            this.#statisticsDep = OwnedOrInjected.resolve(statistics, 
             // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
             // only be satisfied by an injected instance carrying the matching `state`, so this branch does
             // not run in that case - hence the cast.
@@ -436,7 +436,7 @@ export class BasicCrawler {
                     // Started here, rather than in `handleRequest`, so that a failure during context pipeline
                     // initialization (e.g. a browser page timing out before the request handler ever runs) is
                     // still accounted for by `failJob` below - which is a no-op without a matching `startJob`.
-                    this.stats.startJob(request.id || request.uniqueKey);
+                    this.statistics.startJob(request.id || request.uniqueKey);
                     const crawlingContext = { request };
                     try {
                         // The transaction spans the whole pipeline call, covering the navigation hooks
@@ -453,7 +453,7 @@ export class BasicCrawler {
                         // ContextPipelineInterruptedError means the request was intentionally skipped
                         // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
                         if (error instanceof ContextPipelineInterruptedError) {
-                            this.stats.discardJob(request.id || request.uniqueKey);
+                            this.statistics.discardJob(request.id || request.uniqueKey);
                             await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
                             return;
                         }
@@ -714,11 +714,11 @@ export class BasicCrawler {
         });
     }
     getPeriodicLogger() {
-        let previousState = { ...this.stats.state };
+        let previousState = { ...this.statistics.state };
         const getOperationMode = () => {
-            const { requestsFailed } = this.stats.state;
+            const { requestsFailed } = this.statistics.state;
             const { requestsFailed: previousRequestsFailed } = previousState;
-            previousState = { ...this.stats.state };
+            previousState = { ...this.statistics.state };
             const failedDelta = requestsFailed - previousRequestsFailed;
             if (failedDelta > 0) {
                 return { mode: 'ERROR', failedDelta };
@@ -733,12 +733,12 @@ export class BasicCrawler {
             }
             else {
                 const total = await this.requestManager?.getTotalCount();
-                message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
+                message = `Crawled ${this.statistics.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
             if (this.#statusMessageCallback) {
                 await this.#statusMessageCallback({
                     crawler: this,
-                    state: this.stats.state,
+                    state: this.statistics.state,
                     previousState,
                     message,
                 });
@@ -785,7 +785,7 @@ export class BasicCrawler {
                 }
             }
             // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
-            await this.#statsDep.ifOwned(async (stats) => {
+            await this.#statisticsDep.ifOwned(async (stats) => {
                 stats.reset();
                 await stats.resetStore();
             });
@@ -804,7 +804,7 @@ export class BasicCrawler {
         }
         try {
             await this.init();
-            await this.stats.startCapturing();
+            await this.statistics.startCapturing();
         }
         catch (error) {
             // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
@@ -833,25 +833,25 @@ export class BasicCrawler {
             await this.#autoscaledPool.run();
         }
         finally {
+            await this.statistics.stopCapturing();
             await this.teardown();
-            await this.stats.stopCapturing();
             process.off('SIGINT', sigintHandler);
             eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
             eventManager.off(EventType.ABORTING, boundPauseOnMigration);
-            const finalStats = this.stats.calculate();
+            const finalStats = this.statistics.calculate();
             stats = {
-                requestsFinished: this.stats.state.requestsFinished,
-                requestsFailed: this.stats.state.requestsFailed,
-                retryHistogram: this.stats.requestRetryHistogram,
+                requestsFinished: this.statistics.state.requestsFinished,
+                requestsFailed: this.statistics.state.requestsFailed,
+                retryHistogram: this.statistics.requestRetryHistogram,
                 ...finalStats,
             };
             this.log.info('Final request statistics:', stats);
-            if (this.stats.errorTracker.total !== 0) {
+            if (this.statistics.errorTracker.total !== 0) {
                 const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
                 this.log.info(`Error analysis:`, {
-                    totalErrors: this.stats.errorTracker.total,
-                    uniqueErrors: this.stats.errorTracker.getUniqueErrorCount(),
-                    mostCommonErrors: this.stats.errorTracker.getMostPopularErrors(3).map(prettify),
+                    totalErrors: this.statistics.errorTracker.total,
+                    uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
+                    mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
                 });
             }
             const client = serviceLocator.getStorageBackend();
@@ -866,7 +866,7 @@ export class BasicCrawler {
                 finished = true;
             }
             periodicLogger.stop();
-            this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
+            this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
             this.running = false;
             this.hasFinishedBefore = true;
         }
@@ -1150,7 +1150,9 @@ export class BasicCrawler {
             // Only log the limit message when an explicit `limit` was passed (not the internal
             // `maxRequestsPerCrawl`-derived one), and only once per call.
             if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
-                this.log.info(`Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`);
+                this.log.info(requestLimit === options.limit
+                    ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
+                    : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
             }
             await Promise.all(allSkipped.map(async ({ url, reason }) => {
                 await this.handleSkippedRequest({ url, reason });
@@ -1447,7 +1449,7 @@ export class BasicCrawler {
                 });
             }
         })();
-        await Promise.all([requestManagerPersistPromise, this.stats.persistState?.()]);
+        await Promise.all([requestManagerPersistPromise, this.statistics.persistState?.()]);
     }
     /**
      * Fetches the next request to process from the underlying request provider.
@@ -1478,7 +1480,7 @@ export class BasicCrawler {
             await transaction?.commit();
             await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
             isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
-            this.stats.finishJob(statisticsId, request.retryCount);
+            this.statistics.finishJob(statisticsId, request.retryCount);
             // reclaim session if request finishes successfully
             request.state = RequestState.DONE;
             crawlingContext.session.markGood();
@@ -1608,7 +1610,7 @@ export class BasicCrawler {
         }
         const shouldRetryRequest = this.canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
-            await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
+            await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
             await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
             error);
             if (error instanceof SessionError) {
@@ -1637,16 +1639,16 @@ export class BasicCrawler {
         // This is to make sure the error snapshot is not duplicated in the errorTrackerRetry and errorTracker objects.
         const { noRetry, maxRetries } = request;
         if (noRetry || !maxRetries) {
-            await this.stats.errorTracker.addAsync(error, crawlingContext);
+            await this.statistics.errorTracker.addAsync(error, crawlingContext);
         }
         else {
-            this.stats.errorTracker.add(error);
+            this.statistics.errorTracker.add(error);
         }
         // If we get here, the request is either not retryable
         // or failed more than retryCount times and will not be retried anymore.
         // Mark the request as failed and do not retry.
         await source.markRequestAsHandled(request);
-        this.stats.failJob(request.id || request.uniqueKey, request.retryCount);
+        this.statistics.failJob(request.id || request.uniqueKey, request.retryCount);
         await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
     async handleFailedRequestHandler(crawlingContext, error) {
@@ -1706,11 +1708,16 @@ export class BasicCrawler {
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     async teardown() {
-        serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
+        // When this crawler initialized the event manager, its close() call emits
+        // the final persistence event after the crawler-specific state has been
+        // saved. External event managers still need an explicit event here.
+        if (!this.#closeEvents) {
+            serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
+        }
+        await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this.#closeEvents ?? false }));
         if (this.#closeEvents) {
             await serviceLocator.getEventManager().close();
         }
-        await this.#sessionPoolDep.ifOwned((pool) => pool.teardown());
         await this.#autoscaledPool?.abort();
         await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.126",
+    "version": "4.0.0-beta.128",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.126",
-        "@crawlee/http-client": "4.0.0-beta.126",
-        "@crawlee/types": "4.0.0-beta.126",
-        "@crawlee/utils": "4.0.0-beta.126",
+        "@crawlee/core": "4.0.0-beta.128",
+        "@crawlee/http-client": "4.0.0-beta.128",
+        "@crawlee/types": "4.0.0-beta.128",
+        "@crawlee/utils": "4.0.0-beta.128",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.126"
+        "@crawlee/impit-client": "^4.0.0-beta.128"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
+    "gitHead": "2c017f04d564e8fa13855bf19f31f43cb65f4f44"
 }