@crawlee/playwright 4.0.0-beta.8 → 4.0.0-beta.80

package/internals/enqueue-links/click-elements.d.ts

@@ -1,4 +1,4 @@
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestProvider, RequestTransform } from '@crawlee/browser';
+import type { GlobInput, IRequestManager, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
@@ -9,9 +9,9 @@ export interface EnqueueLinksByClickingElementsOptions {
      */
     page: Page;
     /**
-     * A request queue to which the URLs will be enqueued.
+     * * A request manager to which the URLs will be enqueued.
      */
-    requestQueue: RequestProvider;
+    requestManager: IRequestManager;
     /**
      * A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
      * value. This is to prevent suboptimal use of this function by using it too broadly.
@@ -40,6 +40,16 @@ export interface EnqueueLinksByClickingElementsOptions {
      * after clicking on elements matching the provided CSS selector.
      */
     globs?: GlobInput[];
+    /**
+     * An array of glob pattern strings, regexp patterns or plain objects
+     * containing patterns matching URLs that will **never** be enqueued.
+     *
+     * The plain objects must include either the `glob` property or the `regexp` property.
+     *
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, provide a regexp.
+     */
+    exclude?: readonly (GlobInput | RegExpInput)[];
     /**
      * An array of regular expressions or plain objects
      * containing regular expressions matching the URLs to be enqueued.
@@ -73,25 +83,28 @@ export interface EnqueueLinksByClickingElementsOptions {
      */
     pseudoUrls?: PseudoUrlInput[];
     /**
-     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
-     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
-     * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
-     * or to dynamically update or create `userData`.
-     *
-     * For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
-     * a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
-     * (POST requests).
+     * After {@link Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
+     * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
+     * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
+     * but differ in methods or payloads, or to dynamically update or create `userData`.
      *
      * **Example:**
      * ```javascript
      * {
      *     transformRequestFunction: (request) => {
      *         request.userData.foo = 'bar';
-     *         request.useExtendedUniqueKey = true;
      *         return request;
      *     }
      * }
      * ```
+     *
+     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     *
+     * The function receives a {@link RequestOptions} object and can return either:
+     * - The modified {@link RequestOptions} object
+     * - `'unchanged'` to keep the original options as-is
+     * - A falsy value or `'skip'` to exclude the request from the queue
      */
     transformRequestFunction?: RequestTransform;
     /**
@@ -131,6 +144,12 @@ export interface EnqueueLinksByClickingElementsOptions {
      * @default false
      */
     skipNavigation?: boolean;
+    /**
+     * When a request is skipped for some reason, you can use this callback to act on it.
+     * This is fired for requests skipped because they don't match enqueueLinks filters
+     * or because they were removed by `transformRequestFunction`.
+     */
+    onSkippedRequest?: SkippedRequestCallback;
 }
 /**
  * The function finds elements matching a specific CSS selector in a Playwright page,
@@ -162,7 +181,7 @@ export interface EnqueueLinksByClickingElementsOptions {
  * ```javascript
  * await playwrightUtils.enqueueLinksByClickingElements({
  *   page,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
  *   pseudoUrls: [
  *       'https://www.example.com/handbags/[.*]'
@@ -200,4 +219,3 @@ export declare function clickElementsAndInterceptNavigationRequests(options: Cli
  */
 export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
 export {};
-//# sourceMappingURL=click-elements.d.ts.map

package/internals/enqueue-links/click-elements.js

@@ -1,9 +1,8 @@
 import { URL } from 'node:url';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, } from '@crawlee/browser';
+import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, Request as CrawleeRequest, serviceLocator, } from '@crawlee/browser';
 import ow from 'ow';
-import log_ from '@apify/log';
 const STARTING_Z_INDEX = 2147400000;
-const log = log_.child({ prefix: 'Playwright Click Elements' });
+const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
 /**
  * The function finds elements matching a specific CSS selector in a Playwright page,
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -34,7 +33,7 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
  * ```javascript
  * await playwrightUtils.enqueueLinksByClickingElements({
  *   page,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
  *   pseudoUrls: [
  *       'https://www.example.com/handbags/[.*]'
@@ -48,26 +47,41 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
 export async function enqueueLinksByClickingElements(options) {
     ow(options, ow.object.exactShape({
         page: ow.object.hasKeys('goto', 'evaluate'),
-        requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
+        requestManager: ow.object.hasKeys('fetchNextRequest', 'addRequestsBatched'),
         selector: ow.string,
         userData: ow.optional.object,
-        clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
+        clickOptions: ow.optional.object,
         pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
         globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
         regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
+        exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
         transformRequestFunction: ow.optional.function,
         waitForPageIdleSecs: ow.optional.number,
         maxWaitForPageIdleSecs: ow.optional.number,
         label: ow.optional.string,
         forefront: ow.optional.boolean,
         skipNavigation: ow.optional.boolean,
+        onSkippedRequest: ow.optional.function,
     }));
-    const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, } = options;
+    const { page, requestManager, selector, clickOptions, 
+    // oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
+    pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
     const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
     const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
+    const urlExcludePatternObjects = [];
     const urlPatternObjects = [];
+    if (exclude?.length) {
+        for (const excl of exclude) {
+            if (typeof excl === 'string' || 'glob' in excl) {
+                urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
+            }
+            else if (excl instanceof RegExp || 'regexp' in excl) {
+                urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
+            }
+        }
+    }
     if (pseudoUrls?.length) {
-        log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
+        serviceLocator.getLogger().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
         urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
     }
     if (globs?.length) {
@@ -83,12 +97,21 @@ export async function enqueueLinksByClickingElements(options) {
         maxWaitForPageIdleMillis,
         clickOptions,
     });
-    let requestOptions = createRequestOptions(interceptedRequests, options);
+    const requestOptions = createRequestOptions(interceptedRequests, options);
+    const skippedByFilters = [];
+    let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
+    if (onSkippedRequest && skippedByFilters.length > 0) {
+        await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
+    }
     if (transformRequestFunction) {
-        requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
+        const skippedByTransform = [];
+        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
+        if (onSkippedRequest && skippedByTransform.length > 0) {
+            await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
+        }
     }
-    const requests = createRequests(requestOptions, urlPatternObjects);
-    const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
+    const requests = filteredOptions.map((opts) => new CrawleeRequest(opts));
+    const { addedRequests } = await requestManager.addRequestsBatched(requests, { forefront });
     return { processedRequests: addedRequests, unprocessedRequests: [] };
 }
 /**
@@ -114,7 +137,7 @@ export async function clickElementsAndInterceptNavigationRequests(options) {
     await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
     // browser.off(BrowserEmittedEvents.TargetCreated, onTargetCreated);
     page.off('framenavigated', onFrameNavigated);
-    await context.unroute('*', onInterceptedRequest);
+    await context.unroute('**', onInterceptedRequest);
     const serializedRequests = Array.from(uniqueRequests);
     return serializedRequests.map((r) => JSON.parse(r));
 }
@@ -150,7 +173,9 @@ function createTargetCreatedHandler(requests) {
             await popup.close();
         }
         catch (err) {
-            log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
+            getLog().debug('enqueueLinksByClickingElements: Could not close spawned page.', {
+                error: err.stack,
+            });
         }
     };
 }
@@ -158,7 +183,16 @@ function createTargetCreatedHandler(requests) {
  * @ignore
  */
 function isTopFrameNavigationRequest(page, req) {
-    return req.isNavigationRequest() && req.frame() === page.mainFrame();
+    try {
+        return req.isNavigationRequest() && req.frame() === page.mainFrame();
+    }
+    catch {
+        // `req.frame()` throws when the owning frame is unavailable - e.g. the request was
+        // issued by a service worker, or before/after its frame existed (see #3216). Such a
+        // request is not a top-frame navigation, so swallow the throw and let it pass through
+        // instead of crashing the route handler (which would leave the route unhandled).
+        return false;
+    }
 }
 /**
  * @ignore
@@ -223,7 +257,7 @@ function updateElementCssToEnableMouseClick(el, zIndex) {
  */
 export async function clickElements(page, selector, clickOptions) {
     const elementHandles = await page.$$(selector);
-    log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
+    getLog().debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
     let clickedElementsCount = 0;
     let zIndex = STARTING_Z_INDEX;
     let shouldLogWarning = true;
@@ -236,15 +270,15 @@ export async function clickElements(page, selector, clickOptions) {
         catch (err) {
             const e = err;
             if (shouldLogWarning && e.stack.includes('is detached from document')) {
-                log.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
+                getLog().warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
                     'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. ' +
                     'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
                 shouldLogWarning = false;
             }
-            log.debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
+            getLog().debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
         }
     }
-    log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
+    getLog().debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
 }
 /**
  * This function tracks whether any requests, frame navigations or targets were emitted
@@ -263,8 +297,6 @@ export async function clickElements(page, selector, clickOptions) {
 async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis, }) {
     return new Promise((resolve) => {
         let timeout;
-        let maxTimeout;
-        page.on('popup', activityHandler);
         function activityHandler() {
             clearTimeout(timeout);
             timeout = setTimeout(() => {
@@ -273,7 +305,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
             }, waitForPageIdleMillis);
         }
         function maxTimeoutHandler() {
-            log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
+            getLog().debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
                 'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
             finish();
         }
@@ -281,7 +313,8 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
             page.off('request', activityHandler).off('framenavigated', activityHandler).off('popup', activityHandler);
             resolve();
         }
-        maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
+        const maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
+        page.on('popup', activityHandler);
         activityHandler(); // We call this once manually in case there would be no requests at all.
         page.on('request', activityHandler);
         page.on('framenavigated', activityHandler);
@@ -304,8 +337,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
             requests.add(JSON.stringify({ url }));
         }
         catch (err) {
-            log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
+            getLog().debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
         }
     });
 }
-//# sourceMappingURL=click-elements.js.map

package/internals/playwright-crawler.d.ts

@@ -1,20 +1,19 @@
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, RouterRoutes } from '@crawlee/browser';
-import { BrowserCrawler, Configuration } from '@crawlee/browser';
-import type { PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import { BrowserCrawler } from '@crawlee/browser';
+import type { PlaywrightPlugin } from '@crawlee/browser-pool';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { LaunchOptions, Page, Response } from 'playwright';
+import type { Download, LaunchOptions, Page, Response } from 'playwright';
+import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
 import type { PlaywrightLaunchContext } from './playwright-launcher.js';
-import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
-export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<PlaywrightCrawler, Page, Response, PlaywrightController, UserData>, PlaywrightContextUtils {
+import type { BlockRequestsOptions, DirectNavigationOptions, HandleCloudflareChallengeOptions, InfiniteScrollOptions, InjectFileOptions, PlaywrightContextUtils, SaveSnapshotOptions } from './utils/playwright-utils.js';
+export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
+export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
 }
 // @ts-ignore optional peer dependency or compatibility with es2022
-export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {
+export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {
 }
-export interface PlaywrightRequestHandler extends BrowserRequestHandler<PlaywrightCrawlingContext> {
-}
-export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
-export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<PlaywrightCrawlingContext, {
+export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
     browserPlugins: [PlaywrightPlugin];
 }> {
     /**
@@ -28,8 +27,6 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
      * - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
      * - `page` is an instance of the `Playwright`
      * [`Page`](https://playwright.dev/docs/api/class-page)
-     * - `browserController` is an instance of the
-     * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
      * - `response` is an instance of the `Playwright`
      * [`Response`](https://playwright.dev/docs/api/class-response),
      * which is the main resource response as returned by `page.goto(request.url)`.
@@ -45,56 +42,28 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
      * The exceptions are logged to the request using the
      * {@link Request.pushErrorMessage} function.
      */
-    requestHandler?: PlaywrightRequestHandler;
-    /**
-     * Function that is called to process each request.
-     *
-     * The function receives the {@link PlaywrightCrawlingContext} as an argument, where:
-     * - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
-     * - `page` is an instance of the `Playwright`
-     * [`Page`](https://playwright.dev/docs/api/class-page)
-     * - `browserController` is an instance of the
-     * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
-     * - `response` is an instance of the `Playwright`
-     * [`Response`](https://playwright.dev/docs/api/class-response),
-     * which is the main resource response as returned by `page.goto(request.url)`.
-     *
-     * The function must return a promise, which is then awaited by the crawler.
-     *
-     * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
-     * If all the retries fail, the crawler calls the function
-     * provided to the `failedRequestHandler` parameter.
-     * To make this work, you should **always**
-     * let your function throw exceptions rather than catch them.
-     * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage} function.
-     *
-     * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.
-     * @ignore
-     */
-    handlePageFunction?: PlaywrightRequestHandler;
+    requestHandler?: RequestHandler<ExtendedContext>;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
-     * which are passed to the `page.goto()` function the crawler calls to navigate.
+     * or browser properties before navigation. The function receives the `crawlingContext`; the options object
+     * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
+     * A hook may optionally return a partial object whose properties are merged into the crawling context
+     * (e.g. to override context members for subsequent hooks and pipeline stages).
      * Example:
      * ```
      * preNavigationHooks: [
-     *     async (crawlingContext, gotoOptions) => {
-     *         const { page } = crawlingContext;
+     *     async ({ page, gotoOptions }) => {
      *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
+     *         gotoOptions.timeout = 60_000;
      *     },
      * ]
      * ```
-     *
-     * Modyfing `pageOptions` is supported only in Playwright incognito.
-     * See {@link PrePageCreateHook}
      */
     preNavigationHooks?: PlaywrightHook[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
-     * The function accepts `crawlingContext` as the only parameter.
+     * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
+     * whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
      * Example:
      * ```
      * postNavigationHooks: [
@@ -120,13 +89,15 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
- * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
@@ -172,16 +143,18 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
  * ```
  * @category Crawlers
  */
-export declare class PlaywrightCrawler extends BrowserCrawler<{
+export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response, {
     browserPlugins: [PlaywrightPlugin];
-}, LaunchOptions, PlaywrightCrawlingContext> {
-    private readonly options;
-    readonly config: Configuration;
+}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext> {
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -193,17 +166,17 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
 // @ts-ignore optional peer dependency or compatibility with es2022
         headless: import("ow").AnyPredicate<string | boolean>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        browserPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        remoteBrowser: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -220,24 +193,40 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        blockedStatusCodes: import("ow").ArrayPredicate<number>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -246,20 +235,50 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
         maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options?: PlaywrightCrawlerOptions, config?: Configuration);
-    protected _runRequestHandler(context: PlaywrightCrawlingContext): Promise<void>;
+    constructor(options?: PlaywrightCrawlerOptions<ExtendedContext>);
+// @ts-ignore optional peer dependency or compatibility with es2022
+    protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response, Dictionary, Dictionary> & {
+        injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
+        injectJQuery: () => Promise<void>;
+        blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
+        waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<import("@crawlee/browser").CheerioAPI>;
+        infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
+        listDownloads: () => Promise<Download[]>;
+        saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        compileScript: (scriptString: string, ctx?: Dictionary) => import("./utils/playwright-utils.js").CompiledScriptFunction;
+        closeCookieModals: () => Promise<void>;
+        handleCloudflareChallenge: (options?: HandleCloudflareChallengeOptions) => Promise<Response | undefined>;
+    }>;
     protected _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
+    private enhanceContext;
 }
+/**
+ * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
+ * and propagates the post-challenge {@link Response} back into the crawling context via its return value.
+ *
+ * **Example usage**
+ * ```ts
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
+ *
+ * const crawler = new PlaywrightCrawler({
+ *     postNavigationHooks: [handleCloudflareChallengeHook()],
+ * });
+ * ```
+ */
+export declare function handleCloudflareChallengeHook(options?: HandleCloudflareChallengeOptions): PlaywrightHook;
 /**
  * Creates new {@link Router} instance that works based on request labels.
  * This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
@@ -284,6 +303,6 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
-//# sourceMappingURL=playwright-crawler.d.ts.map
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;