@crawlee/basic 4.0.0-beta.101 → 4.0.0-beta.103

package/internals/basic-crawler.d.ts

@@ -1,5 +1,5 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
-import { ConcurrencySystem, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
+import { ConcurrencySystem, ContextPipeline, Dataset, RequestQueue } from '@crawlee/core';
 import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
@@ -267,10 +267,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     onSkippedRequest?: SkippedRequestCallback;
     /**
-     * Customize the way statistics collecting works, such as logging interval or
-     * whether to output them to the Key-Value store.
+     * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
+     * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} (subclass it to track
+     * extra fields) or any object implementing {@link IStatistics}.
      */
-    statisticsOptions?: StatisticsOptions;
+    statistics?: IStatistics;
     /**
      * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
      * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
@@ -422,10 +423,13 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Used to detect and warn about multiple crawlers sharing the same state.
      */
     private static useStateAnonymousIndices;
+    /** Backs the {@link BasicCrawler.stats|`stats`} getter. */
+    private statsDep;
     /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+     * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    readonly stats: Statistics;
+    get stats(): IStatistics;
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
@@ -608,7 +612,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
 // @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };

package/internals/basic-crawler.js

@@ -52,10 +52,15 @@ export class BasicCrawler {
      * Used to detect and warn about multiple crawlers sharing the same state.
      */
     static useStateAnonymousIndices = new Set();
+    /** Backs the {@link BasicCrawler.stats|`stats`} getter. */
+    statsDep;
     /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+     * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    stats;
+    get stats() {
+        return this.statsDep.value;
+    }
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
@@ -225,7 +230,7 @@ export class BasicCrawler {
         maxConcurrency: ow.optional.number,
         maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
         keepAlive: ow.optional.boolean,
-        statisticsOptions: ow.optional.object,
+        statistics: ow.optional.object,
         id: ow.optional.string,
     };
     /**
@@ -241,7 +246,7 @@ export class BasicCrawler {
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statisticsOptions, httpClient, id, } = options;
+        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statistics, httpClient, id, } = options;
         // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
         // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
         // hammering a site.
@@ -323,12 +328,11 @@ export class BasicCrawler {
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
             this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
-            this.stats = new Statistics({
+            this.statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
                 id: this.identity.id,
-                ...statisticsOptions,
-            });
+            }));
             if (sessionPool && proxyConfiguration) {
                 this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
                     'The `proxyConfiguration` is ignored - sessions from the supplied pool keep whatever ' +
@@ -707,8 +711,11 @@ export class BasicCrawler {
             if (managerToPurge?.purge && shouldPurge) {
                 await managerToPurge.purge();
             }
-            this.stats.reset();
-            await this.stats.resetStore();
+            // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+            await this.statsDep.ifOwned(async (stats) => {
+                stats.reset();
+                await stats.resetStore();
+            });
             await this.sessionPoolDep.ifOwned((pool) => pool.resetStore());
         }
         this.unexpectedStop = false;
@@ -1230,7 +1237,7 @@ export class BasicCrawler {
                 });
             }
         })();
-        await Promise.all([requestManagerPersistPromise, this.stats.persistState()]);
+        await Promise.all([requestManagerPersistPromise, this.stats.persistState?.()]);
     }
     /**
      * Fetches the next request to process from the underlying request provider.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.101",
+    "version": "4.0.0-beta.103",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.101",
-        "@crawlee/http-client": "4.0.0-beta.101",
-        "@crawlee/types": "4.0.0-beta.101",
-        "@crawlee/utils": "4.0.0-beta.101",
+        "@crawlee/core": "4.0.0-beta.103",
+        "@crawlee/http-client": "4.0.0-beta.103",
+        "@crawlee/types": "4.0.0-beta.103",
+        "@crawlee/utils": "4.0.0-beta.103",
         "csv-stringify": "^6.5.2",
         "ow": "^2.0.0",
         "tldts": "^7.0.6",
@@ -53,7 +53,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.101"
+        "@crawlee/impit-client": "^4.0.0-beta.103"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "1fd886ee601c053d6fa8cada24b751f9cbe0a539"
+    "gitHead": "fe5d0ae11067683e27a2d17b4edd226ccf112cf5"
 }