@crawlee/playwright 4.0.0-beta.102 → 4.0.0-beta.104

package/internals/adaptive-playwright-crawler.d.ts

@@ -25,7 +25,7 @@ declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
     trackBrowserRequestHandlerRun(): void;
     trackRenderingTypeMisprediction(): void;
 }
-export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
+export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
     request: LoadedRequest<Request<UserData>>;
     /**
      * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
@@ -186,7 +186,7 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
     private staticContextPipeline;
     private browserContextPipeline;
     private individualRequestHandlerTimeoutMillis;
-    readonly stats: AdaptivePlaywrightCrawlerStatistics;
+    get stats(): AdaptivePlaywrightCrawlerStatistics;
     private resultObjects;
     private inFlightRenderingTypeDetections;
     private teardownHooks;

package/internals/adaptive-playwright-crawler.js

@@ -90,17 +90,30 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
     staticContextPipeline;
     browserContextPipeline;
     individualRequestHandlerTimeoutMillis;
+    // The constructor always injects an `AdaptivePlaywrightCrawlerStatistics`, so narrowing the cast is sound.
+    get stats() {
+        return super.stats;
+    }
     resultObjects = new WeakMap();
     inFlightRenderingTypeDetections = 0;
     teardownHooks = [];
     constructor(options = {}) {
-        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statisticsOptions, preventDirectStorageAccess = true, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, ...rest } = options;
+        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, preventDirectStorageAccess = true, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, ...rest } = options;
+        if (statistics !== undefined && !(statistics instanceof AdaptivePlaywrightCrawlerStatistics)) {
+            throw new Error('AdaptivePlaywrightCrawler tracks extra fields on its own Statistics subclass and cannot use a ' +
+                'plain `statistics` instance. Omit the option to let the crawler build its own.');
+        }
         super({
             ...rest,
             errorHandler,
             failedRequestHandler,
             requestHandler,
             requestHandlerTimeoutSecs,
+            // Inject our subclass so the base tracks the extra adaptive fields instead of building a plain `Statistics`.
+            statistics: statistics ??
+                new AdaptivePlaywrightCrawlerStatistics({
+                    logMessage: `${AdaptivePlaywrightCrawler.name} request statistics:`,
+                }),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
         this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
@@ -134,18 +147,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         // each hook's overrides at runtime regardless of the static type.
         const staticCrawler = new CheerioCrawler({
             ...rest,
-            statisticsOptions: {
-                persistenceOptions: { enable: false },
-            },
+            statistics: new Statistics({ persistenceOptions: { enable: false } }),
             preNavigationHooks,
             postNavigationHooks,
             extendContext,
         });
         const browserCrawler = new PlaywrightCrawler({
             ...rest,
-            statisticsOptions: {
-                persistenceOptions: { enable: false },
-            },
+            statistics: new Statistics({ persistenceOptions: { enable: false } }),
             preNavigationHooks: preNavigationHooks,
             postNavigationHooks: postNavigationHooks,
             extendContext,
@@ -157,10 +166,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         this.browserContextPipeline = browserCrawler.contextPipeline.compose({
             action: this.adaptPlaywrightContext.bind(this),
         });
-        this.stats = new AdaptivePlaywrightCrawlerStatistics({
-            logMessage: `${this.log.getOptions().prefix} request statistics:`,
-            ...statisticsOptions,
-        });
         this.preventDirectStorageAccess = preventDirectStorageAccess;
     }
     async _init() {

package/internals/playwright-crawler.d.ts

@@ -8,11 +8,9 @@ import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/clic
 import type { PlaywrightLaunchContext } from './playwright-launcher.js';
 import type { BlockRequestsOptions, DirectNavigationOptions, HandleCloudflareChallengeOptions, InfiniteScrollOptions, InjectFileOptions, PlaywrightContextUtils, SaveSnapshotOptions } from './utils/playwright-utils.js';
 export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
-export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
-}
-// @ts-ignore optional peer dependency or compatibility with es2022
-export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {
+export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
 }
+export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
 export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
     browserPlugins: [PlaywrightPlugin];
 }, Routes> {
@@ -59,7 +57,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
      * ]
      * ```
      */
-    preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
+    preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
@@ -76,7 +74,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
      * ]
      * ```
      */
-    postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
+    postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
 }
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -238,7 +236,7 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
 // @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.102",
+    "version": "4.0.0-beta.104",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.102",
-        "@crawlee/browser": "4.0.0-beta.102",
-        "@crawlee/browser-pool": "4.0.0-beta.102",
-        "@crawlee/cheerio": "4.0.0-beta.102",
-        "@crawlee/core": "4.0.0-beta.102",
-        "@crawlee/types": "4.0.0-beta.102",
-        "@crawlee/utils": "4.0.0-beta.102",
+        "@crawlee/basic": "4.0.0-beta.104",
+        "@crawlee/browser": "4.0.0-beta.104",
+        "@crawlee/browser-pool": "4.0.0-beta.104",
+        "@crawlee/cheerio": "4.0.0-beta.104",
+        "@crawlee/core": "4.0.0-beta.104",
+        "@crawlee/types": "4.0.0-beta.104",
+        "@crawlee/utils": "4.0.0-beta.104",
         "cheerio": "^1.0.0",
         "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
@@ -85,5 +85,5 @@
             }
         }
     },
-    "gitHead": "19eec2f388401dc75cf82d8cdcad5dd27143b1aa"
+    "gitHead": "1c761e689e56ac06b67fc4f1dc091f693e024d12"
 }