@crawlee/playwright 4.0.0-beta.81 → 4.0.0-beta.82

package/internals/adaptive-playwright-crawler.d.ts

@@ -79,13 +79,11 @@ interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id
     request: Request;
     gotoOptions?: PlaywrightGotoOptions;
 }
-interface AdaptiveHook extends BrowserHook<AdaptiveHookContext> {
-}
-interface AdaptivePostNavigationHook extends BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
+type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHookContext, ContextExtension>;
+type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
     request: LoadedRequest<Request>;
-}> {
-}
-export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
+}, ContextExtension>;
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -94,7 +92,7 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
      * A hook may optionally return a partial object whose properties are merged into the crawling context,
      * allowing the hook to override context members for subsequent hooks and pipeline stages.
      */
-    preNavigationHooks?: AdaptiveHook[];
+    preNavigationHooks?: AdaptiveHook<ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -103,7 +101,7 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
      * A hook may optionally return a partial object whose properties are merged into the crawling context
      * (e.g. to override `response` after solving a challenge).
      */
-    postNavigationHooks?: AdaptivePostNavigationHook[];
+    postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];
     /**
      * Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
      * Defaults to 0.1 (so 10%).
@@ -177,7 +175,7 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
  *
  * @experimental
  */
-export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {
+export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext> {
     private renderingTypePredictor;
     private resultChecker;
     private shouldPropagateError;
@@ -190,7 +188,7 @@ export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptiveP
     private resultObjects;
     private inFlightRenderingTypeDetections;
     private teardownHooks;
-    constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>);
+    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext>);
     protected _init(): Promise<void>;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
         readonly request: LoadedRequest<Request<Dictionary>>;

package/internals/adaptive-playwright-crawler.js

@@ -122,10 +122,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     }));
             };
         }
-        // Each adaptive hook is registered as its own static/browser hook so the underlying
-        // `ContextPipeline` handles override merging between hooks for free. The hook signatures
-        // are structurally compatible with the underlying crawlers' contexts (subset of fields);
-        // the casts just relax the nominal type difference.
+        // `extendContext` is forwarded to the inner crawlers, which run it *before* navigation (see
+        // `BasicCrawler`), keeping the behavior consistent with the non-adaptive crawlers: the
+        // extension is visible to the pre/post-navigation hooks and the request handler, but cannot
+        // access navigation-dependent members (`page`, `response`, `$`, ...).
+        //
+        // The adaptive hooks target a subset context (`AdaptiveHookContext`); the casts to the inner
+        // crawlers' `PlaywrightHook` type relax that nominal difference. The `ContextPipeline` merges
+        // each hook's overrides at runtime regardless of the static type.
         const staticCrawler = new CheerioCrawler({
             ...rest,
             statisticsOptions: {
@@ -133,6 +137,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
             preNavigationHooks,
             postNavigationHooks,
+            extendContext,
         });
         const browserCrawler = new PlaywrightCrawler({
             ...rest,
@@ -141,21 +146,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
             preNavigationHooks: preNavigationHooks,
             postNavigationHooks: postNavigationHooks,
+            extendContext,
         });
         this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
-        this.staticContextPipeline = staticCrawler.contextPipeline
-            .compose({
+        this.staticContextPipeline = staticCrawler.contextPipeline.compose({
             action: this.adaptCheerioContext.bind(this),
-        })
-            .compose({
-            action: async (context) => extendContext ? await extendContext(context) : context,
         });
-        this.browserContextPipeline = browserCrawler.contextPipeline
-            .compose({
+        this.browserContextPipeline = browserCrawler.contextPipeline.compose({
             action: this.adaptPlaywrightContext.bind(this),
-        })
-            .compose({
-            action: async (context) => extendContext ? await extendContext(context) : context,
         });
         this.stats = new AdaptivePlaywrightCrawlerStatistics({
             logMessage: `${this.log.getOptions().prefix} request statistics:`,

package/internals/playwright-crawler.d.ts

@@ -59,7 +59,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
      * ]
      * ```
      */
-    preNavigationHooks?: PlaywrightHook[];
+    preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
@@ -76,7 +76,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
      * ]
      * ```
      */
-    postNavigationHooks?: PlaywrightHook[];
+    postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
 }
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -243,7 +243,7 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options?: PlaywrightCrawlerOptions<ExtendedContext>);
+    constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext>);
 // @ts-ignore optional peer dependency or compatibility with es2022
     protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response, Dictionary, Dictionary> & {
         injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.81",
+    "version": "4.0.0-beta.82",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.81",
-        "@crawlee/browser": "4.0.0-beta.81",
-        "@crawlee/browser-pool": "4.0.0-beta.81",
-        "@crawlee/cheerio": "4.0.0-beta.81",
-        "@crawlee/core": "4.0.0-beta.81",
-        "@crawlee/types": "4.0.0-beta.81",
-        "@crawlee/utils": "4.0.0-beta.81",
+        "@crawlee/basic": "4.0.0-beta.82",
+        "@crawlee/browser": "4.0.0-beta.82",
+        "@crawlee/browser-pool": "4.0.0-beta.82",
+        "@crawlee/cheerio": "4.0.0-beta.82",
+        "@crawlee/core": "4.0.0-beta.82",
+        "@crawlee/types": "4.0.0-beta.82",
+        "@crawlee/utils": "4.0.0-beta.82",
         "cheerio": "^1.0.0",
         "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
@@ -85,5 +85,5 @@
             }
         }
     },
-    "gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
+    "gitHead": "eb1096f7c7743d124375ef011fbbadb19476822e"
 }