@crawlee/basic 4.0.0-beta.124 → 4.0.0-beta.126

package/internals/basic-crawler.d.ts

@@ -29,7 +29,7 @@ export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlin
 export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
     contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
 };
-export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
@@ -278,10 +278,28 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     onSkippedRequest?: SkippedRequestCallback;
     /**
      * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
-     * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} (subclass it to track
-     * extra fields) or any object implementing {@link IStatistics}.
+     * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
+     * implementing {@link IStatistics}.
+     *
+     * Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
+     * {@link BasicCrawler.stats|`crawler.stats.state`}:
+     *
+     * ```ts
+     * const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
+     *
+     * const crawler = new BasicCrawler({
+     *     statistics,
+     *     requestHandler: async () => {
+     *         statistics.state.productsFound++;
+     *     },
+     * });
+     *
+     * await crawler.run();
+     * // the custom fields are typed on `crawler.stats` too
+     * console.log(crawler.stats.state.productsFound);
+     * ```
      */
-    statistics?: IStatistics;
+    statistics?: IStatistics<StatisticStateExtension>;
     /**
      * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
      * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
@@ -433,7 +451,7 @@ interface CrawlerIdentity {
     /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */
     readonly hasExplicitId: boolean;
 }
-export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     #private;
     protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
     /**
@@ -446,7 +464,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    get stats(): IStatistics;
+    get stats(): IStatistics<StatisticStateExtension>;
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
@@ -609,7 +627,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> & RequireContextPipeline<CrawlingContext, Context>);
+    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
      * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a

package/internals/basic-crawler.js

@@ -387,7 +387,11 @@ export class BasicCrawler {
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
             this.#sameDomainDelaySecs = sameDomainDelaySecs;
-            this.#statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
+            this.#statsDep = OwnedOrInjected.resolve(statistics, 
+            // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
+            // only be satisfied by an injected instance carrying the matching `state`, so this branch does
+            // not run in that case - hence the cast.
+            () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
                 id: this.identity.id,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.124",
+    "version": "4.0.0-beta.126",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.124",
-        "@crawlee/http-client": "4.0.0-beta.124",
-        "@crawlee/types": "4.0.0-beta.124",
-        "@crawlee/utils": "4.0.0-beta.124",
+        "@crawlee/core": "4.0.0-beta.126",
+        "@crawlee/http-client": "4.0.0-beta.126",
+        "@crawlee/types": "4.0.0-beta.126",
+        "@crawlee/utils": "4.0.0-beta.126",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.124"
+        "@crawlee/impit-client": "^4.0.0-beta.126"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "0694ee1b94c755b98141671baa93cc363f2bf8e3"
+    "gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
 }