@crawlee/playwright 4.0.0-beta.7 → 4.0.0-beta.71

package/internals/playwright-crawler.js

@@ -1,7 +1,7 @@
-import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';
+import { BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';
 import ow from 'ow';
 import { PlaywrightLauncher } from './playwright-launcher.js';
-import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.js';
+import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless Chromium, Firefox and Webkit browsers with [Playwright](https://github.com/microsoft/playwright).
@@ -13,13 +13,15 @@ import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.j
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
- * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
@@ -66,19 +68,19 @@ import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.j
  * @category Crawlers
  */
 export class PlaywrightCrawler extends BrowserCrawler {
-    options;
-    config;
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
         browserPoolOptions: ow.optional.object,
         launcher: ow.optional.object,
+        ignoreIframes: ow.optional.boolean,
+        ignoreShadowRoots: ow.optional.boolean,
     };
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options = {}) {
         ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
-        const { launchContext = {}, headless, ...browserCrawlerOptions } = options;
+        const { launchContext = {}, headless, contextPipelineBuilder, ...browserCrawlerOptions } = options;
         const browserPoolOptions = {
             ...options.browserPoolOptions,
         };
@@ -95,19 +97,80 @@ export class PlaywrightCrawler extends BrowserCrawler {
             launchContext.launchOptions ??= {};
             launchContext.launchOptions.headless = headless;
         }
-        const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
+        const playwrightLauncher = new PlaywrightLauncher(launchContext, options.configuration);
         browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
-        super({ ...browserCrawlerOptions, launchContext, browserPoolOptions }, config);
-        this.options = options;
-        this.config = config;
+        super({
+            ...browserCrawlerOptions,
+            launchContext,
+            browserPoolOptions,
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+        });
     }
-    async _runRequestHandler(context) {
-        registerUtilsToContext(context, this.options);
-        await super._runRequestHandler(context);
+    buildContextPipeline() {
+        return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });
     }
     async _navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
+    async enhanceContext(context) {
+        const waitForSelector = async (selector, timeoutMs = 5_000) => {
+            const locator = context.page.locator(selector).first();
+            await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+        };
+        return {
+            injectFile: async (filePath, options) => playwrightUtils.injectFile(context.page, filePath, options),
+            injectJQuery: async () => {
+                if (context.request.state === RequestState.BEFORE_NAV) {
+                    context.log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
+                    await playwrightUtils.injectJQuery(context.page);
+                    return;
+                }
+                await playwrightUtils.injectJQuery(context.page, { surviveNavigations: false });
+            },
+            blockRequests: async (options) => playwrightUtils.blockRequests(context.page, options),
+            waitForSelector,
+            parseWithCheerio: async (selector, timeoutMs = 5_000) => {
+                if (selector) {
+                    await waitForSelector(selector, timeoutMs);
+                }
+                return playwrightUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
+            },
+            infiniteScroll: async (options) => playwrightUtils.infiniteScroll(context.page, options),
+            saveSnapshot: async (options) => playwrightUtils.saveSnapshot(context.page, { ...options, config: serviceLocator.getConfiguration() }),
+            enqueueLinksByClickingElements: async (options) => playwrightUtils.enqueueLinksByClickingElements({
+                ...options,
+                page: context.page,
+                requestManager: this.requestManager,
+            }),
+            compileScript: (scriptString, ctx) => playwrightUtils.compileScript(scriptString, ctx),
+            closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page),
+            handleCloudflareChallenge: async (options) => {
+                return playwrightUtils.handleCloudflareChallenge(context.page, context.request.url, options);
+            },
+        };
+    }
+}
+/**
+ * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
+ * and propagates the post-challenge {@link Response} back into the crawling context via its return value.
+ *
+ * **Example usage**
+ * ```ts
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
+ *
+ * const crawler = new PlaywrightCrawler({
+ *     postNavigationHooks: [handleCloudflareChallengeHook()],
+ * });
+ * ```
+ */
+export function handleCloudflareChallengeHook(options) {
+    return async (context) => {
+        const response = await context.handleCloudflareChallenge(options);
+        if (response !== undefined) {
+            return { response };
+        }
+        return undefined;
+    };
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/playwright-crawler.js.map

@@ -1 +1 @@
-{"version":3,"file":"playwright-crawler.js","sourceRoot":"","sources":["../../src/internals/playwright-crawler.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAGzE,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,EAAE,YAAY,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AA4GnF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8DG;AACH,MAAM,OAAO,iBAAkB,SAAQ,cAItC;IAWwB;IACC;IAXZ,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,cAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC;IAEF;;OAEG;IACH,YACqB,UAAoC,EAAE,EACrC,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,0BAA0B,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE9F,MAAM,EAAE,aAAa,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,qBAAqB,EAAE,GAAG,OAAO,CAAC;QAE3E,MAAM,kBAAkB,GAAG;YACvB,GAAG,OAAO,CAAC,kBAAkB;SACV,CAAC;QAExB,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACX,sFAAsF;gBAClF,iDAAiD,CACxD,CAAC;QACN,CAAC;QAED,2EAA2E;QAC3E,uFAAuF;QACvF,IAAI,kBAAkB,CAAC,cAAc,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;QAC5G,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACpD,aAAa,CAAC,aAAa,CAAC,QAAQ,GAAG,QAAmB,CAAC;QAC/D,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAEzE,kBAAkB,CAAC,cAAc,GAAG,CAAC,kBAAkB,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAE/E,KAAK,CAAC,EAAE,GAAG,qBAAqB,EAAE,aAAa,EAAE,kBAAkB,EAAE,EAAE,MAAM,CAAC,CAAC;QAjC9D,YAAO,GAAP,OAAO,CAA+B;QACrC,WAAM,GAAN,MAAM,CAAkC;IAiC9D,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,OAAkC;QAC1E,sBAAsB,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CACvC,eAA0C,EAC1C,WAAoC;QAEpC,OAAO,YAAY,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;;AAGL;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,sBAAsB,CAGpC,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"playwright-crawler.js","sourceRoot":"","sources":["../../src/internals/playwright-crawler.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAGxF,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAU9D,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAqF5E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,OAAO,iBAGX,SAAQ,cAQT;IACa,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,cAAc,CAAC,YAAY;QAC9B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAClC,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACzC,CAAC;IAEF;;OAEG;IACH,YAAY,UAAqD,EAAE;QAC/D,EAAE,CAAC,OAAO,EAAE,0BAA0B,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAE9F,MAAM,EAAE,aAAa,GAAG,EAAE,EAAE,QAAQ,EAAE,sBAAsB,EAAE,GAAG,qBAAqB,EAAE,GAAG,OAAO,CAAC;QAEnG,MAAM,kBAAkB,GAAG;YACvB,GAAG,OAAO,CAAC,kBAAkB;SACV,CAAC;QAExB,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACX,sFAAsF;gBAClF,iDAAiD,CACxD,CAAC;QACN,CAAC;QAED,2EAA2E;QAC3E,uFAAuF;QACvF,IAAI,kBAAkB,CAAC,cAAc,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;QAC5G,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACpD,aAAa,CAAC,aAAa,CAAC,QAAQ,GAAG,QAAmB,CAAC;QAC/D,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAExF,kBAAkB,CAAC,cAAc,GAAG,CAAC,kBAAkB,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAE/E,KAAK,CAAC;YACF,GAAI,qBAAmE;YACvE,aAAa;YACb,kBAAkB;YAClB,sBAAsB,EAAE,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;SACxF,CAAC,CAAC;IACP,CAAC;IAEkB,oBAAoB;QACnC,OAAO,KAAK,CAAC,oBAAoB,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5F,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CACvC,eAA0C,EAC1C,WAAoC;QAEpC,OAAO,YAAY,CAAC,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACpF,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,OAA2D;QACpF,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE;YAClE,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC;YACvD,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;QACrE,CAAC,CAAC;QAEF,OAAO;YACH,UAAU,EAAE,KAAK,EAAE,QAAgB,EAAE,OAA2B,EAAE,EAAE,CAChE,eAAe,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC;YAC/D,YAAY,EAAE,KAAK,IAAI,EAAE;gBACrB,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,KAAK,YAAY,CAAC,UAAU,EAAE,CAAC;oBACpD,OAAO,CAAC,GAAG,CAAC,OAAO,CACf,mIAAmI,CACtI,CAAC;oBACF,MAAM,eAAe,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBACjD,OAAO;gBACX,CAAC;gBACD,MAAM,eAAe,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC;YACpF,CAAC;YACD,aAAa,EAAE,KAAK,EAAE,OAA8B,EAAE,EAAE,CACpD,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;YACxD,eAAe;YACf,gBAAgB,EAAE,KAAK,EAAE,QAAiB,EAAE,SAAS,GAAG,KAAK,EAAE,EAAE;gBAC7D,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAC/C,CAAC;gBAED,OAAO,eAAe,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YACtG,CAAC;YACD,cAAc,EAAE,KAAK,EAAE,OAA+B,EAAE,EAAE,CACtD,eAAe,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;YACzD,YAAY,EAAE,KAAK,EAAE,OAA6B,EAAE,EAAE,CAClD,eAAe,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE,EAAE,CAAC;YACzG,8BAA8B,EAAE,KAAK,EACjC,OAA+E,EACjF,EAAE,CACA,eAAe,CAAC,8BAA8B,CAAC;gBAC3C,GAAG,OAAO;gBACV,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,cAAc,EAAE,IAAI,CAAC,cAAe;aACvC,CAAC;YACN,aAAa,EAAE,CAAC,YAAoB,EAAE,GAAgB,EAAE,EAAE,CAAC,eAAe,CAAC,aAAa,CAAC,YAAY,EAAE,GAAG,CAAC;YAC3G,iBAAiB,EAAE,KAAK,IAAI,EAAE,CAAC,eAAe,CAAC,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC;YAC9E,yBAAyB,EAAE,KAAK,EAAE,OAA0C,EAAE,EAAE;gBAC5E,OAAO,eAAe,CAAC,yBAAyB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACjG,CAAC;SACJ,CAAC;IACN,CAAC;;AAGL;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,6BAA6B,CAAC,OAA0C;IACpF,OAAO,KAAK,EAAE,OAAO,EAAE,EAAE;QACrB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAClE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,sBAAsB,CAGpC,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}

package/internals/playwright-launcher.d.ts

@@ -84,6 +84,8 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
         useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022

package/internals/playwright-launcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"playwright-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/playwright-launcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,OAAO,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEtE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,uBAAwB,SAAQ,oBAAoB,CAAC,aAAa,EAAE,WAAW,CAAC;IAC7F;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,GAAG,UAAU,CAAC,WAAW,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,WAAW,CAAC;CAC1B;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,eAAe,CAAC,gBAAgB,CAAC;aAY/C,MAAM;IAX5B,iBAA0B,YAAY;;;;;;;;;;MAIpC;IAEF;;OAEG;gBAEC,aAAa,GAAE,uBAA4B,EACzB,MAAM,gBAAkC;CA2BjE;AA0BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAsB,gBAAgB,CAClC,aAAa,CAAC,EAAE,uBAAuB,EACvC,MAAM,gBAAkC,GACzC,OAAO,CAAC,OAAO,CAAC,CAIlB"}
+{"version":3,"file":"playwright-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/playwright-launcher.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,OAAO,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEtE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,uBAAwB,SAAQ,oBAAoB,CAAC,aAAa,EAAE,WAAW,CAAC;IAC7F;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,GAAG,UAAU,CAAC,WAAW,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtF;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,WAAW,CAAC;CAC1B;AAED;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,eAAe,CAAC,gBAAgB,CAAC;aAY/C,MAAM;IAX5B,iBAA0B,YAAY;;;;;;;;;;;MAIpC;IAEF;;OAEG;gBAEC,aAAa,GAAE,uBAA4B,EACzB,MAAM,gBAAkC;CA2BjE;AA0BD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAsB,gBAAgB,CAClC,aAAa,CAAC,EAAE,uBAAuB,EACvC,MAAM,gBAAkC,GACzC,OAAO,CAAC,OAAO,CAAC,CAIlB"}

package/internals/playwright-launcher.js

@@ -37,7 +37,7 @@ export class PlaywrightLauncher extends BrowserLauncher {
  * @ignore
  */
 function getDefaultExecutablePath(launchContext, config) {
-    const pathFromPlaywrightImage = config.get('defaultBrowserPath');
+    const pathFromPlaywrightImage = config.defaultBrowserPath;
     const { launchOptions = {} } = launchContext;
     if (launchOptions.executablePath) {
         return launchOptions.executablePath;

package/internals/playwright-launcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"playwright-launcher.js","sourceRoot":"","sources":["../../src/internals/playwright-launcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,MAAM,IAAI,CAAC;AAuEpB;;;GAGG;AACH,MAAM,OAAO,kBAAmB,SAAQ,eAAiC;IAY/C;IAXZ,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,eAAe,CAAC,YAAY;QAC/B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC3C,CAAC;IAEF;;OAEG;IACH,YACI,gBAAyC,EAAE,EACzB,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,aAAa,EAAE,2BAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC,CAAC;QAEtG,MAAM,EACF,QAAQ,GAAG,eAAe,CAAC,sBAAsB,CAC7C,YAAY,EACZ,+BAA+B,CAClC,CAAC,QAAQ,GACb,GAAG,aAAa,CAAC;QAElB,MAAM,EAAE,aAAa,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,aAAa,CAAC;QAEtD,KAAK,CACD;YACI,GAAG,IAAI;YACP,aAAa,EAAE;gBACX,GAAG,aAAa;gBAChB,cAAc,EAAE,wBAAwB,CAAC,aAAa,EAAE,MAAM,CAAC;aAClE;YACD,QAAQ;SACX,EACD,MAAM,CACT,CAAC;QAvBgB,WAAM,GAAN,MAAM,CAAkC;QAyB1D,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC;IACnC,CAAC;;AAGL;;;;GAIG;AACH,SAAS,wBAAwB,CAAC,aAAsC,EAAE,MAAqB;IAC3F,MAAM,uBAAuB,GAAG,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACjE,MAAM,EAAE,aAAa,GAAG,EAAE,EAAE,GAAG,aAAa,CAAC;IAE7C,IAAI,aAAa,CAAC,cAAc,EAAE,CAAC;QAC/B,OAAO,aAAa,CAAC,cAAc,CAAC;IACxC,CAAC;IAED,IAAI,aAAa,CAAC,SAAS,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,IAAI,uBAAuB,EAAE,CAAC;QAC1B,OAAO,uBAAuB,CAAC;IACnC,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAClC,aAAuC,EACvC,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;IAExC,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAEzE,OAAO,kBAAkB,CAAC,MAAM,EAAE,CAAC;AACvC,CAAC"}
+{"version":3,"file":"playwright-launcher.js","sourceRoot":"","sources":["../../src/internals/playwright-launcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,MAAM,IAAI,CAAC;AAuEpB;;;GAGG;AACH,MAAM,OAAO,kBAAmB,SAAQ,eAAiC;IAY/C;IAXZ,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,eAAe,CAAC,YAAY;QAC/B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAC3C,CAAC;IAEF;;OAEG;IACH,YACI,gBAAyC,EAAE,EACzB,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,aAAa,EAAE,2BAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC,CAAC;QAEtG,MAAM,EACF,QAAQ,GAAG,eAAe,CAAC,sBAAsB,CAC7C,YAAY,EACZ,+BAA+B,CAClC,CAAC,QAAQ,GACb,GAAG,aAAa,CAAC;QAElB,MAAM,EAAE,aAAa,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,aAAa,CAAC;QAEtD,KAAK,CACD;YACI,GAAG,IAAI;YACP,aAAa,EAAE;gBACX,GAAG,aAAa;gBAChB,cAAc,EAAE,wBAAwB,CAAC,aAAa,EAAE,MAAM,CAAC;aAClE;YACD,QAAQ;SACX,EACD,MAAM,CACT,CAAC;QAvBgB,WAAM,GAAN,MAAM,CAAkC;QAyB1D,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC;IACnC,CAAC;;AAGL;;;;GAIG;AACH,SAAS,wBAAwB,CAAC,aAAsC,EAAE,MAAqB;IAC3F,MAAM,uBAAuB,GAAG,MAAM,CAAC,kBAAkB,CAAC;IAC1D,MAAM,EAAE,aAAa,GAAG,EAAE,EAAE,GAAG,aAAa,CAAC;IAE7C,IAAI,aAAa,CAAC,cAAc,EAAE,CAAC;QAC/B,OAAO,aAAa,CAAC,cAAc,CAAC;IACxC,CAAC;IAED,IAAI,aAAa,CAAC,SAAS,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,IAAI,uBAAuB,EAAE,CAAC;QAC1B,OAAO,uBAAuB,CAAC;IACnC,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAClC,aAAuC,EACvC,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;IAExC,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAEzE,OAAO,kBAAkB,CAAC,MAAM,EAAE,CAAC;AACvC,CAAC"}

package/internals/utils/playwright-utils.d.ts

@@ -17,14 +17,13 @@
  * ```
  * @module playwrightUtils
  */
-import { Configuration, type Request, type Session } from '@crawlee/browser';
+import { Configuration, type Request } from '@crawlee/browser';
 import type { BatchAddRequestsResult } from '@crawlee/types';
 import { type CheerioRoot, type Dictionary } from '@crawlee/utils';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext } from '../playwright-crawler.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
 export interface InjectFileOptions {
     /**
@@ -290,7 +289,7 @@ export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions):
  */
 export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
 export declare function closeCookieModals(page: Page): Promise<void>;
-interface HandleCloudflareChallengeOptions {
+export interface HandleCloudflareChallengeOptions {
     /** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */
     verbose?: boolean;
     /** How long should we wait after the challenge is completed for the final page to load. */
@@ -304,6 +303,13 @@ interface HandleCloudflareChallengeOptions {
     isChallengeCallback?: (page: Page) => Promise<boolean>;
     /** Allows overriding the detection of Cloudflare "blocked page". */
     isBlockedCallback?: (page: Page) => Promise<boolean>;
+    /** Allows overriding how the checkbox click position is calculated. */
+    clickPositionCallback?: (page: Page) => Promise<{
+        x: number;
+        y: number;
+    } | null>;
+    /** Optional delay (in seconds) before the first click attempt on the challenge checkbox. Defaults to 1s. */
+    preChallengeSleepSecs?: number;
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -312,23 +318,24 @@ interface HandleCloudflareChallengeOptions {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-declare function handleCloudflareChallenge(page: Page, url: string, session?: Session, options?: HandleCloudflareChallengeOptions): Promise<void>;
+declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
 /** @internal */
 export interface PlaywrightContextUtils {
     /**
@@ -480,7 +487,7 @@ export interface PlaywrightContextUtils {
      *
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
     /**
      * Compiles a Playwright script into an async function that may be executed at any time
      * by providing it with the following object:
@@ -510,6 +517,15 @@ export interface PlaywrightContextUtils {
     compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
     /**
      * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+     *
+     * Note that this method requires the idcac-playwright package to be installed.
+     * Crawlee does not include it by default due to licensing issues.
+     *
+     * To use this method, please install the package manually by running:
+     *
+     * ```bash
+     * npm install idcac-playwright
+     * ```
      */
     closeCookieModals(): Promise<void>;
     /**
@@ -519,22 +535,21 @@ export interface PlaywrightContextUtils {
      * result in a SessionError which will be automatically retried, so only successful requests will get
      * into the `requestHandler`.
      *
-     * Works best with camoufox.
+     * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned,
+     * which can be returned from the hook to update the crawling context's `response`. For the common case,
+     * prefer the pre-wrapped {@link handleCloudflareChallengeHook} hook.
      *
      * **Example usage**
      * ```ts
      * postNavigationHooks: [
-     *     async ({ handleCloudflareChallenge }) => {
-     *         await handleCloudflareChallenge();
-     *     },
+     *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
      * ],
      * ```
      *
      * @param [options]
      */
-    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<void>;
+    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
 }
-export declare function registerUtilsToContext(context: PlaywrightCrawlingContext, crawlerOptions: PlaywrightCrawlerOptions): void;
 export { enqueueLinksByClickingElements };
 /** @internal */
 export declare const playwrightUtils: {

package/internals/utils/playwright-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"playwright-utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,OAAO,EACH,aAAa,EAEb,KAAK,OAAO,EAEZ,KAAK,OAAO,EAGf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,UAAU,EAA4B,MAAM,gBAAgB,CAAC;AAI7F,OAAO,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAS,MAAM,YAAY,CAAC;AAKxD,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,oCAAoC,CAAC;AAChG,OAAO,EAAE,8BAA8B,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,KAAK,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AACpG,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAUxE,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CA0BhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE;IAAE,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3G;AAED,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAC9B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,OAAO,EAChB,WAAW,GAAE,uBAA4B,GAC1C,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAiD1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBjG;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAExF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+C/F;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,gBAAgB,CAClC,IAAI,EAAE,IAAI,EACV,iBAAiB,UAAQ,EACzB,aAAa,UAAQ,GACtB,OAAO,CAAC,WAAW,CAAC,CAmCtB;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAIjE;AAED,UAAU,gCAAgC;IACtC,2FAA2F;IAC3F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,2FAA2F;IAC3F,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uKAAuK;IACvK,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACrF,sEAAsE;IACtE,mBAAmB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACxD;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,iBAAe,yBAAyB,CACpC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,OAAO,EACjB,OAAO,GAAE,gCAAqC,GAC/C,OAAO,CAAC,IAAI,CAAC,CAgGf;AAED,gBAAgB;AAChB,MAAM,WAAW,sBAAsB;IACnC;;;;;;OAMG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5E;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9E;;;OAGG;IACH,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/D;;;OAGG;IACH,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,8BAA8B,CAC1B,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,cAAc,CAAC,GAC9E,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,GAAG,sBAAsB,CAAC;IAE9E;;OAEG;IACH,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,yBAAyB,CAAC,OAAO,CAAC,EAAE,gCAAgC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxF;AAED,wBAAgB,sBAAsB,CAClC,OAAO,EAAE,yBAAyB,EAClC,cAAc,EAAE,wBAAwB,GACzC,IAAI,CAyCN;AAED,OAAO,EAAE,8BAA8B,EAAE,CAAC;AAE1C,gBAAgB;AAChB,eAAO,MAAM,eAAe;;;;;;;;;;;;;CAa3B,CAAC"}
+{"version":3,"file":"playwright-utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/playwright-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAMH,OAAO,EAAE,aAAa,EAAiB,KAAK,OAAO,EAA4C,MAAM,kBAAkB,CAAC;AACxH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,UAAU,EAA4B,MAAM,gBAAgB,CAAC;AAG7F,OAAO,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAS,MAAM,YAAY,CAAC;AAIxD,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,oCAAoC,CAAC;AAChG,OAAO,EAAE,8BAA8B,EAAE,MAAM,oCAAoC,CAAC;AACpF,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAUxE,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAOD;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CA0BhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE;IAAE,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3G;AAED,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAC9B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,OAAO,EAChB,WAAW,GAAE,uBAA4B,GAC1C,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAiD1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBjG;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAExF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+C/F;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,gBAAgB,CAClC,IAAI,EAAE,IAAI,EACV,iBAAiB,UAAQ,EACzB,aAAa,UAAQ,GACtB,OAAO,CAAC,WAAW,CAAC,CA2CtB;AAwBD,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAOjE;AAED,MAAM,WAAW,gCAAgC;IAC7C,2FAA2F;IAC3F,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,2FAA2F;IAC3F,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uKAAuK;IACvK,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACrF,sEAAsE;IACtE,mBAAmB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACvD,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACrD,uEAAuE;IACvE,qBAAqB,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACjF,4GAA4G;IAC5G,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,iBAAe,yBAAyB,CACpC,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,gCAAqC,GAC/C,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAsG/B;AAED,gBAAgB;AAChB,MAAM,WAAW,sBAAsB;IACnC;;;;;;OAMG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5E;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;IACH,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9E;;;OAGG;IACH,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/D;;;OAGG;IACH,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,8BAA8B,CAC1B,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,gBAAgB,CAAC,GAChF,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,GAAG,sBAAsB,CAAC;IAE9E;;;;;;;;;;;OAWG;IACH,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,yBAAyB,CAAC,OAAO,CAAC,EAAE,gCAAgC,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;CACxG;AAED,OAAO,EAAE,8BAA8B,EAAE,CAAC;AAE1C,gBAAgB;AAChB,eAAO,MAAM,eAAe;;;;;;;;;;;;;CAa3B,CAAC"}

package/internals/utils/playwright-utils.js

@@ -20,16 +20,14 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, RequestState, SessionError, validators, } from '@crawlee/browser';
+import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
 import { expandShadowRoots, sleep } from '@crawlee/utils';
 import * as cheerio from 'cheerio';
-import { getInjectableScript as getCookieClosingScript } from 'idcac-playwright';
 import ow from 'ow';
 import { LruCache } from '@apify/datastructures';
-import log_ from '@apify/log';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
-const log = log_.child({ prefix: 'Playwright Utils' });
+const getLog = () => serviceLocator.getChildLog('Playwright Utils');
 const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
@@ -64,7 +62,7 @@ export async function injectFile(page, filePath, options = {}) {
     if (options.surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
-            .catch((error) => log.warning('An error occurred during the script injection!', { error })));
+            .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
     }
     return evalP;
 }
@@ -123,7 +121,7 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
     if (method !== 'GET' || payload || !isEmpty(headers)) {
         // This is not deprecated, we use it to log only once.
-        log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
+        getLog().deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
             'in recent versions of Playwright. Use only when necessary.');
         let wasCalled = false;
         const interceptRequestHandler = async (route) => {
@@ -144,7 +142,7 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
                 await route.continue(overrides);
             }
             catch (error) {
-                log.debug('Error inside request interceptor', { error });
+                getLog().debug('Error inside request interceptor', { error });
             }
             return undefined;
         };
@@ -213,7 +211,7 @@ export async function blockRequests(page, options = {}) {
         await client.send('Network.setBlockedURLs', { urls: patternsToBlock });
     }
     catch {
-        log.warning('blockRequests() helper is incompatible with non-Chromium browsers.');
+        getLog().warning('blockRequests() helper is incompatible with non-Chromium browsers.');
     }
 }
 /**
@@ -249,7 +247,7 @@ export function compileScript(scriptString, context = Object.create(null)) {
         func = vm.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
     }
     catch (err) {
-        log.exception(err, 'Cannot compile script!');
+        getLog().exception(err, 'Cannot compile script!');
         throw err;
     }
     if (typeof func !== 'function')
@@ -360,7 +358,7 @@ export async function saveSnapshot(page, options = {}) {
     }));
     const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, config, } = options;
     try {
-        const store = await KeyValueStore.open(keyValueStoreName, {
+        const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
             config: config ?? Configuration.getGlobalConfig(),
         });
         if (saveScreenshot) {
@@ -403,7 +401,15 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
             try {
                 const iframe = await frame.contentFrame();
                 if (iframe) {
-                    const contents = await iframe.content();
+                    const getIframeHTML = async () => {
+                        try {
+                            return iframe.locator('body').first().innerHTML();
+                        }
+                        catch {
+                            return iframe.content();
+                        }
+                    };
+                    const contents = await getIframeHTML();
                     await frame.evaluate((f, c) => {
                         const replacementNode = document.createElement('div');
                         replacementNode.innerHTML = c;
@@ -413,7 +419,7 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
                 }
             }
             catch (error) {
-                log.warning(`Failed to extract iframe content: ${error}`);
+                getLog().warning(`Failed to extract iframe content: ${error}`);
             }
         }));
     }
@@ -423,9 +429,34 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
     const pageContent = html || (await page.content());
     return cheerio.load(pageContent);
 }
+let idcacPlaywright = null;
+async function getIdcacPlaywright() {
+    if (idcacPlaywright)
+        return idcacPlaywright;
+    try {
+        idcacPlaywright = await import('idcac-playwright');
+    }
+    catch (error) {
+        getLog().warning(`Failed to import 'idcac-playwright'.
+
+We recently made idcac-playwright an optional dependency due to licensing issues.
+To use this feature, please install it manually by running
+
+npm install idcac-playwright
+
+Original error message follows:
+
+${error.message}
+`);
+    }
+    return idcacPlaywright;
+}
 export async function closeCookieModals(page) {
     ow(page, ow.object.validate(validators.browserPage));
-    await page.evaluate(getCookieClosingScript());
+    const idcac = await getIdcacPlaywright();
+    if (idcac?.getInjectableScript()) {
+        await page.evaluate(idcac.getInjectableScript());
+    }
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -434,30 +465,24 @@ export async function closeCookieModals(page) {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-async function handleCloudflareChallenge(page, url, session, options = {}) {
-    // eslint-disable-next-line dot-notation
-    const blockedStatusCodes = session?.['sessionPool']['blockedStatusCodes'];
-    // Cloudflare pages are 403, which are blocked by default
-    if (blockedStatusCodes?.includes(403)) {
-        const idx = blockedStatusCodes.indexOf(403);
-        blockedStatusCodes.splice(idx, 1);
-    }
+async function handleCloudflareChallenge(page, url, options = {}) {
     options.isBlockedCallback ??= async () => {
         const isBlocked = await page.evaluate(() => {
             return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
@@ -481,10 +506,10 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     if (!(await isChallenge())) {
         await retryBlocked();
-        return;
+        return undefined;
     }
     const logLevel = options.verbose ? 'info' : 'debug';
-    log[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
+    getLog()[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
     const bb = await page
         .evaluate(() => {
         const div = document.querySelector('.main-content div');
@@ -492,20 +517,27 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     })
         .catch(() => undefined);
     if (!bb) {
-        return;
+        return undefined;
     }
     const randomOffset = (range) => {
         return Math.round(100 * range * Math.random()) / 100;
     };
-    const x = bb.x + 30;
-    const y = bb.y + 25;
+    let x = bb.x + 30;
+    let y = bb.y + 25;
     // try to click the checkbox every second
     for (let i = 0; i < 10; i++) {
-        await sleep(1000);
+        await sleep((options.preChallengeSleepSecs ?? 1) * 1000);
         // break early if we are no longer on the CF challenge page
         if (!(await isChallenge())) {
             break;
         }
+        if (options.clickPositionCallback) {
+            const pos = await options.clickPositionCallback(page);
+            if (pos) {
+                x = pos.x;
+                y = pos.y;
+            }
+        }
         if (options.clickCallback) {
             await options.clickCallback(page, { x, y });
             continue;
@@ -513,7 +545,10 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         // we can click on the text too, so X can be a bit larger
         const xRandomized = x + randomOffset(10);
         const yRandomized = y + randomOffset(10);
-        log[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, { x: xRandomized, y: yRandomized });
+        getLog()[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, {
+            x: xRandomized,
+            y: yRandomized,
+        });
         await page.mouse.click(xRandomized, yRandomized);
         // sometimes the checkbox is lower (could be caused by a lag when rendering the logo)
         await page.mouse.click(xRandomized, yRandomized + 35);
@@ -523,40 +558,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
     }
     await retryBlocked();
-}
-export function registerUtilsToContext(context, crawlerOptions) {
-    context.injectFile = async (filePath, options) => injectFile(context.page, filePath, options);
-    context.injectJQuery = async () => {
-        if (context.request.state === RequestState.BEFORE_NAV) {
-            log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
-            await injectJQuery(context.page);
-            return;
-        }
-        await injectJQuery(context.page, { surviveNavigations: false });
-    };
-    context.blockRequests = async (options) => blockRequests(context.page, options);
-    context.waitForSelector = async (selector, timeoutMs = 5_000) => {
-        const locator = context.page.locator(selector).first();
-        await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-    };
-    context.parseWithCheerio = async (selector, timeoutMs = 5_000) => {
-        if (selector) {
-            await context.waitForSelector(selector, timeoutMs);
-        }
-        return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes);
-    };
-    context.infiniteScroll = async (options) => infiniteScroll(context.page, options);
-    context.saveSnapshot = async (options) => saveSnapshot(context.page, { ...options, config: context.crawler.config });
-    context.enqueueLinksByClickingElements = async (options) => enqueueLinksByClickingElements({
-        ...options,
-        page: context.page,
-        requestQueue: context.crawler.requestQueue,
-    });
-    context.compileScript = (scriptString, ctx) => compileScript(scriptString, ctx);
-    context.closeCookieModals = async () => closeCookieModals(context.page);
-    context.handleCloudflareChallenge = async (options) => {
-        return handleCloudflareChallenge(context.page, context.request.url, context.session, options);
-    };
+    // Reload to obtain a fresh Response without the challenge interstitial, which the caller can
+    // propagate back into the crawling context so downstream status-code checks see the new value.
+    return (await page.reload()) ?? undefined;
 }
 export { enqueueLinksByClickingElements };
 /** @internal */