@crawlee/playwright 4.0.0-beta.21 → 4.0.0-beta.210

package/internals/playwright-crawler.js

@@ -1,7 +1,8 @@
-import { BrowserCrawler, Configuration, RequestState, Router } from '@crawlee/browser';
-import ow from 'ow';
-import { PlaywrightLauncher } from './playwright-launcher.js';
-import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
+import { BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';
+import { assertBrowserPoolNotConfigured, parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
+import { playwrightBrowserPool, remotePlaywrightBrowserPool } from './playwright-browser-pool.js';
+import { blockRequests, compileScript, enqueueLinksByClickingElements, gotoExtended, handleCloudflareChallenge, infiniteScroll, injectFile, injectJQuery, parseWithCheerio, saveSnapshot, } from './utils/playwright-utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless Chromium, Firefox and Webkit browsers with [Playwright](https://github.com/microsoft/playwright).
@@ -13,24 +14,26 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
- * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
  * `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
  * and then calls the function provided by user as the {@link PlaywrightCrawlerOptions.requestHandler} option.
  *
- * New pages are only opened when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link PlaywrightCrawlerOptions.autoscaledPoolOptions}
- * parameter of the `PlaywrightCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPoolOptions} are available directly in the `PlaywrightCrawler` constructor.
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * Note that the pool of Playwright instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
  *
@@ -66,47 +69,46 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
  * @category Crawlers
  */
 export class PlaywrightCrawler extends BrowserCrawler {
-    config;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
-        browserPoolOptions: ow.optional.object,
-        launcher: ow.optional.object,
-        ignoreIframes: ow.optional.boolean,
-        ignoreShadowRoots: ow.optional.boolean,
+        launchContext: schemas.anyObject.default(() => ({})),
+        headless: z.boolean().optional(),
     };
+    /** @internal */
+    static optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
-        const { launchContext = {}, headless, ...browserCrawlerOptions } = options;
-        const browserPoolOptions = {
-            ...options.browserPoolOptions,
-        };
+    constructor(options = {}) {
+        const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');
+        const { launchContext, headless, configuration, ...browserCrawlerOptions } = parsedOptions;
         if (launchContext.proxyUrl) {
             throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.' +
                 'Use PlaywrightCrawlerOptions.proxyConfiguration');
         }
-        // `browserPlugins` is working when it's not overridden by `launchContext`,
-        // which for crawlers it is always overridden. Hence the error to use the other option.
-        if (browserPoolOptions.browserPlugins) {
-            throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
-        }
-        if (headless != null) {
-            launchContext.launchOptions ??= {};
-            launchContext.launchOptions.headless = headless;
+        if (options.browserPool) {
+            // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
+            assertBrowserPoolNotConfigured(new.target.name, {
+                launchContext: options.launchContext,
+                headless: options.headless,
+            });
         }
-        const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
-        browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
         super({
             ...browserCrawlerOptions,
-            launchContext,
-            browserPoolOptions,
-            contextPipelineBuilder: () => this.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) }),
-        }, config);
-        this.config = config;
+            configuration,
+            browserPoolBuilder: (remoteBrowser) => remoteBrowser
+                ? remotePlaywrightBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
+                : playwrightBrowserPool({ launchContext, headless, configuration }),
+            contextPipelineBuilder: () => this.#buildContextPipeline(),
+        });
     }
-    async _navigationHandler(crawlingContext, gotoOptions) {
+    #buildContextPipeline() {
+        return this.buildContextPipeline().compose(this.enhanceContext.bind(this));
+    }
+    async navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
     async enhanceContext(context) {
@@ -114,64 +116,66 @@ export class PlaywrightCrawler extends BrowserCrawler {
             const locator = context.page.locator(selector).first();
             await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
         };
+        const downloads = [];
+        context.page.on('download', (download) => downloads.push(download));
         return {
-            injectFile: async (filePath, options) => playwrightUtils.injectFile(context.page, filePath, options),
+            injectFile: async (filePath, options) => injectFile(context.page, filePath, options),
             injectJQuery: async () => {
                 if (context.request.state === RequestState.BEFORE_NAV) {
                     context.log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
-                    await playwrightUtils.injectJQuery(context.page);
+                    await injectJQuery(context.page);
                     return;
                 }
-                await playwrightUtils.injectJQuery(context.page, { surviveNavigations: false });
+                await injectJQuery(context.page, { surviveNavigations: false });
             },
-            blockRequests: async (options) => playwrightUtils.blockRequests(context.page, options),
+            blockRequests: async (options) => blockRequests(context.page, options),
             waitForSelector,
             parseWithCheerio: async (selector, timeoutMs = 5_000) => {
                 if (selector) {
                     await waitForSelector(selector, timeoutMs);
                 }
-                return playwrightUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
+                return parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
             },
-            infiniteScroll: async (options) => playwrightUtils.infiniteScroll(context.page, options),
-            saveSnapshot: async (options) => playwrightUtils.saveSnapshot(context.page, { ...options, config: this.config }),
-            enqueueLinksByClickingElements: async (options) => playwrightUtils.enqueueLinksByClickingElements({
+            infiniteScroll: async (options) => infiniteScroll(context.page, options),
+            listDownloads: async () => downloads,
+            saveSnapshot: async (options) => saveSnapshot(context.page, {
+                ...options,
+                configuration: serviceLocator.getConfiguration(),
+            }),
+            enqueueLinksByClickingElements: async (options) => enqueueLinksByClickingElements({
                 ...options,
                 page: context.page,
-                requestQueue: this.requestQueue,
+                requestManager: this.requestManager,
             }),
-            compileScript: (scriptString, ctx) => playwrightUtils.compileScript(scriptString, ctx),
-            closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page),
+            compileScript: (scriptString, ctx) => compileScript(scriptString, ctx),
             handleCloudflareChallenge: async (options) => {
-                return playwrightUtils.handleCloudflareChallenge(context.page, context.request.url, context.session, options);
+                return handleCloudflareChallenge(context.page, context.request.url, options);
             },
         };
     }
 }
 /**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
- * Defaults to the {@link PlaywrightCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<PlaywrightCrawlingContext>()`.
+ * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
+ * and propagates the post-challenge {@link Response} back into the crawling context via its return value.
  *
+ * **Example usage**
  * ```ts
- * import { PlaywrightCrawler, createPlaywrightRouter } from 'crawlee';
- *
- * const router = createPlaywrightRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
  *
  * const crawler = new PlaywrightCrawler({
- *     requestHandler: router,
+ *     postNavigationHooks: [handleCloudflareChallengeHook()],
  * });
- * await crawler.run();
  * ```
  */
-export function createPlaywrightRouter(routes) {
-    return Router.create(routes);
+export function handleCloudflareChallengeHook(options) {
+    return async (context) => {
+        const response = await context.handleCloudflareChallenge(options);
+        if (response !== undefined) {
+            return { response };
+        }
+        return undefined;
+    };
+}
+export function createPlaywrightRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=playwright-crawler.js.map

package/internals/playwright-launcher.d.ts

@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PlaywrightPlugin } from '@crawlee/browser-pool';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Browser, BrowserType, LaunchOptions } from 'playwright';
+import { z } from 'zod';
 /**
  * Apify extends the launch options of Playwright.
  * You can use any of the Playwright compatible
@@ -70,31 +71,41 @@ export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOpti
  * @ignore
  */
 export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin> {
-    readonly config: Configuration;
+    readonly configuration: Configuration;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        launchContextOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+    };
+    /** @internal */
+    protected static optionsSchema: z.ZodObject<{
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-    };
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+    }, z.core.$strict>;
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.
      */
-    constructor(launchContext?: PlaywrightLaunchContext, config?: Configuration);
+    constructor(launchContext?: PlaywrightLaunchContext, configuration?: Configuration);
 }
 /**
  * Launches headless browsers using Playwright pre-configured to work within the Apify platform.
@@ -125,9 +136,8 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
  *   Optional settings passed to `browserType.launch()`. In addition to
  *   [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)
  *   the object may contain our own  {@link PlaywrightLaunchContext} that enable additional features.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Playwright's `Browser` instance.
  */
-export declare function launchPlaywright(launchContext?: PlaywrightLaunchContext, config?: Configuration): Promise<Browser>;
-//# sourceMappingURL=playwright-launcher.d.ts.map
+export declare function launchPlaywright(launchContext?: PlaywrightLaunchContext, configuration?: Configuration): Promise<Browser>;

package/internals/playwright-launcher.js

@@ -1,33 +1,39 @@
 import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PlaywrightPlugin } from '@crawlee/browser-pool';
-import ow from 'ow';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 /**
  * `PlaywrightLauncher` is based on the `BrowserLauncher`. It launches `playwright` browser instance.
  * @ignore
  */
 export class PlaywrightLauncher extends BrowserLauncher {
-    config;
+    configuration;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
-        launcher: ow.optional.object,
-        launchContextOptions: ow.optional.object,
+        // Passthrough schema — the launcher module object must keep its prototype through parsing.
+        launcher: schemas.anyObject.optional(),
     };
+    /** @internal */
+    static optionsSchema = z.strictObject(PlaywrightLauncher.optionsShape);
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.
      */
-    constructor(launchContext = {}, config = Configuration.getGlobalConfig()) {
-        ow(launchContext, 'PlaywrightLauncherOptions', ow.object.exactShape(PlaywrightLauncher.optionsShape));
-        const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = launchContext;
-        const { launchOptions = {}, ...rest } = launchContext;
+    constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
+        const parsedContext = parseArgument(launchContext, PlaywrightLauncher.optionsSchema, 'PlaywrightLaunchContext');
+        const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = parsedContext;
+        const { launchOptions = {}, ...rest } = parsedContext;
         super({
             ...rest,
             launchOptions: {
                 ...launchOptions,
-                executablePath: getDefaultExecutablePath(launchContext, config),
+                executablePath: getDefaultExecutablePath(parsedContext, configuration),
             },
             launcher,
-        }, config);
-        this.config = config;
+        }, configuration);
+        this.configuration = configuration;
         this.Plugin = PlaywrightPlugin;
     }
 }
@@ -36,8 +42,8 @@ export class PlaywrightLauncher extends BrowserLauncher {
  * @returns default path to browser.
  * @ignore
  */
-function getDefaultExecutablePath(launchContext, config) {
-    const pathFromPlaywrightImage = config.get('defaultBrowserPath');
+function getDefaultExecutablePath(launchContext, configuration) {
+    const pathFromPlaywrightImage = configuration.defaultBrowserPath;
     const { launchOptions = {} } = launchContext;
     if (launchOptions.executablePath) {
         return launchOptions.executablePath;
@@ -79,12 +85,11 @@ function getDefaultExecutablePath(launchContext, config) {
  *   Optional settings passed to `browserType.launch()`. In addition to
  *   [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)
  *   the object may contain our own  {@link PlaywrightLaunchContext} that enable additional features.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Playwright's `Browser` instance.
  */
-export async function launchPlaywright(launchContext, config = Configuration.getGlobalConfig()) {
-    const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
+export async function launchPlaywright(launchContext, configuration = Configuration.getGlobalConfiguration()) {
+    const playwrightLauncher = new PlaywrightLauncher(launchContext, configuration);
     return playwrightLauncher.launch();
 }
-//# sourceMappingURL=playwright-launcher.js.map

package/internals/utils/playwright-utils.d.ts

@@ -17,14 +17,13 @@
  * ```
  * @module playwrightUtils
  */
-import { Configuration, type Request, type Session } from '@crawlee/browser';
-import type { BatchAddRequestsResult } from '@crawlee/types';
-import { type CheerioRoot, type Dictionary } from '@crawlee/utils';
+import { Configuration, type Request } from '@crawlee/browser';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import type { CheerioAPI } from 'cheerio';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { Page, Response } from 'playwright';
+import type { Download, Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import { RenderingTypePredictor } from './rendering-type-prediction.js';
 export interface InjectFileOptions {
     /**
      * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
@@ -265,9 +264,9 @@ export interface SaveSnapshotOptions {
     keyValueStoreName?: string | null;
     /**
      * Configuration of the crawler that will be used to save the snapshot.
-     * @default Configuration.getGlobalConfig()
+     * @default Configuration.getGlobalConfiguration()
      */
-    config?: Configuration;
+    configuration?: Configuration;
 }
 /**
  * Saves a full screenshot and HTML of the current page into a Key-Value store.
@@ -287,8 +286,7 @@ export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions):
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object.
  * @param ignoreShadowRoots
  */
-export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
-export declare function closeCookieModals(page: Page): Promise<void>;
+export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
 export interface HandleCloudflareChallengeOptions {
     /** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */
     verbose?: boolean;
@@ -303,6 +301,13 @@ export interface HandleCloudflareChallengeOptions {
     isChallengeCallback?: (page: Page) => Promise<boolean>;
     /** Allows overriding the detection of Cloudflare "blocked page". */
     isBlockedCallback?: (page: Page) => Promise<boolean>;
+    /** Allows overriding how the checkbox click position is calculated. */
+    clickPositionCallback?: (page: Page) => Promise<{
+        x: number;
+        y: number;
+    } | null>;
+    /** Optional delay (in seconds) before the first click attempt on the challenge checkbox. Defaults to 1s. */
+    preChallengeSleepSecs?: number;
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -311,24 +316,24 @@ export interface HandleCloudflareChallengeOptions {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-declare function handleCloudflareChallenge(page: Page, url: string, session?: Session, options?: HandleCloudflareChallengeOptions): Promise<void>;
-/** @internal */
+export declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
 export interface PlaywrightContextUtils {
     /**
      * Injects a JavaScript file into current `page`.
@@ -427,7 +432,7 @@ export interface PlaywrightContextUtils {
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
     /**
      * Scrolls to the bottom of a page, or until it times out.
      * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.
@@ -447,8 +452,7 @@ export interface PlaywrightContextUtils {
      * in `href` elements, but rather navigations are triggered in click handlers.
      * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
      * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -469,9 +473,9 @@ export interface PlaywrightContextUtils {
      * async requestHandler({ enqueueLinksByClickingElements }) {
      *     await enqueueLinksByClickingElements({
      *         selector: 'a.product-detail',
-     *         globs: [
-     *             'https://www.example.com/handbags/**'
-     *             'https://www.example.com/purses/**'
+     *         include: [
+     *             'https://www.example.com/handbags/**',
+     *             'https://www.example.com/purses/**',
      *         ],
      *     });
      * });
@@ -479,7 +483,7 @@ export interface PlaywrightContextUtils {
      *
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
     /**
      * Compiles a Playwright script into an async function that may be executed at any time
      * by providing it with the following object:
@@ -507,19 +511,6 @@ export interface PlaywrightContextUtils {
      * secured copies beforehand.
      */
     compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
-    /**
-     * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
-     *
-     * Note that this method requires the idcac-playwright package to be installed.
-     * Crawlee does not include it by default due to licensing issues.
-     *
-     * To use this method, please install the package manually by running:
-     *
-     * ```bash
-     * npm install idcac-playwright
-     * ```
-     */
-    closeCookieModals(): Promise<void>;
     /**
      * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
      * It will try to detect the Cloudflare page, click on the checkbox, and wait for 10 seconds (configurable
@@ -527,35 +518,42 @@ export interface PlaywrightContextUtils {
      * result in a SessionError which will be automatically retried, so only successful requests will get
      * into the `requestHandler`.
      *
-     * Works best with camoufox.
+     * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned,
+     * which can be returned from the hook to update the crawling context's `response`. For the common case,
+     * prefer the pre-wrapped {@link handleCloudflareChallengeHook} hook.
      *
      * **Example usage**
      * ```ts
      * postNavigationHooks: [
-     *     async ({ handleCloudflareChallenge }) => {
-     *         await handleCloudflareChallenge();
-     *     },
+     *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
      * ],
      * ```
      *
      * @param [options]
      */
-    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<void>;
+    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
+    /**
+     * Returns the list of {@link https://playwright.dev/docs/api/class-download | Download} objects
+     * collected during the current page navigation and request handler.
+     *
+     * Useful for accessing files that the page downloads automatically.
+     * For most use cases, prefer re-enqueueing the URL to {@link FileDownload}.
+     * Use this only when direct access to the Playwright `Download` object is required.
+     *
+     * **Example usage**
+     * ```ts
+     * requestHandler: async ({ listDownloads }) => {
+     *     for (const download of await listDownloads()) {
+     *         try {
+     *             const stream = await download.createReadStream();
+     *             // stream to storage...
+     *         } catch {
+     *             // download failed or was cancelled
+     *         }
+     *     }
+     * },
+     * ```
+     */
+    listDownloads(): Promise<Download[]>;
 }
 export { enqueueLinksByClickingElements };
-/** @internal */
-export declare const playwrightUtils: {
-    injectFile: typeof injectFile;
-    injectJQuery: typeof injectJQuery;
-    gotoExtended: typeof gotoExtended;
-    blockRequests: typeof blockRequests;
-    enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
-    parseWithCheerio: typeof parseWithCheerio;
-    infiniteScroll: typeof infiniteScroll;
-    saveSnapshot: typeof saveSnapshot;
-    compileScript: typeof compileScript;
-    closeCookieModals: typeof closeCookieModals;
-    RenderingTypePredictor: typeof RenderingTypePredictor;
-    handleCloudflareChallenge: typeof handleCloudflareChallenge;
-};
-//# sourceMappingURL=playwright-utils.d.ts.map