@crawlee/playwright 4.0.0-beta.10 → 4.0.0-beta.101

package/internals/playwright-crawler.d.ts

@@ -1,22 +1,21 @@
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, RouterRoutes } from '@crawlee/browser';
-import { BrowserCrawler, Configuration } from '@crawlee/browser';
-import type { PlaywrightController, PlaywrightPlugin } from '@crawlee/browser-pool';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import { BrowserCrawler } from '@crawlee/browser';
+import type { PlaywrightPlugin } from '@crawlee/browser-pool';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { LaunchOptions, Page, Response } from 'playwright';
+import type { Download, LaunchOptions, Page, Response } from 'playwright';
+import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
 import type { PlaywrightLaunchContext } from './playwright-launcher.js';
-import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
-export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<PlaywrightCrawler, Page, Response, PlaywrightController, UserData>, PlaywrightContextUtils {
+import type { BlockRequestsOptions, DirectNavigationOptions, HandleCloudflareChallengeOptions, InfiniteScrollOptions, InjectFileOptions, PlaywrightContextUtils, SaveSnapshotOptions } from './utils/playwright-utils.js';
+export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
+export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
 }
 // @ts-ignore optional peer dependency or compatibility with es2022
-export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {
+export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {
 }
-export interface PlaywrightRequestHandler extends BrowserRequestHandler<PlaywrightCrawlingContext> {
-}
-export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];
-export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<PlaywrightCrawlingContext, {
+export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
     browserPlugins: [PlaywrightPlugin];
-}> {
+}, Routes> {
     /**
      * The same options as used by {@link launchPlaywright}.
      */
@@ -28,33 +27,6 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
      * - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
      * - `page` is an instance of the `Playwright`
      * [`Page`](https://playwright.dev/docs/api/class-page)
-     * - `browserController` is an instance of the
-     * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
-     * - `response` is an instance of the `Playwright`
-     * [`Response`](https://playwright.dev/docs/api/class-response),
-     * which is the main resource response as returned by `page.goto(request.url)`.
-     *
-     * The function must return a promise, which is then awaited by the crawler.
-     *
-     * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
-     * If all the retries fail, the crawler calls the function
-     * provided to the `failedRequestHandler` parameter.
-     * To make this work, you should **always**
-     * let your function throw exceptions rather than catch them.
-     * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage} function.
-     */
-    requestHandler?: PlaywrightRequestHandler;
-    /**
-     * Function that is called to process each request.
-     *
-     * The function receives the {@link PlaywrightCrawlingContext} as an argument, where:
-     * - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
-     * - `page` is an instance of the `Playwright`
-     * [`Page`](https://playwright.dev/docs/api/class-page)
-     * - `browserController` is an instance of the
-     * [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
      * - `response` is an instance of the `Playwright`
      * [`Response`](https://playwright.dev/docs/api/class-response),
      * which is the main resource response as returned by `page.goto(request.url)`.
@@ -69,32 +41,29 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
      * let your function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
      * {@link Request.pushErrorMessage} function.
-     *
-     * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.
-     * @ignore
      */
-    handlePageFunction?: PlaywrightRequestHandler;
+    requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
-     * which are passed to the `page.goto()` function the crawler calls to navigate.
+     * or browser properties before navigation. The function receives the `crawlingContext`; the options object
+     * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
+     * A hook may optionally return a partial object whose properties are merged into the crawling context
+     * (e.g. to override context members for subsequent hooks and pipeline stages).
      * Example:
      * ```
      * preNavigationHooks: [
-     *     async (crawlingContext, gotoOptions) => {
-     *         const { page } = crawlingContext;
+     *     async ({ page, gotoOptions }) => {
      *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
+     *         gotoOptions.timeout = 60_000;
      *     },
      * ]
      * ```
-     *
-     * Modyfing `pageOptions` is supported only in Playwright incognito.
-     * See {@link PrePageCreateHook}
      */
-    preNavigationHooks?: PlaywrightHook[];
+    preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
-     * The function accepts `crawlingContext` as the only parameter.
+     * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
+     * whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
      * Example:
      * ```
      * postNavigationHooks: [
@@ -107,7 +76,7 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
      * ]
      * ```
      */
-    postNavigationHooks?: PlaywrightHook[];
+    postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
 }
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -120,24 +89,26 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
- * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
  * `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
  * and then calls the function provided by user as the {@link PlaywrightCrawlerOptions.requestHandler} option.
  *
- * New pages are only opened when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link PlaywrightCrawlerOptions.autoscaledPoolOptions}
- * parameter of the `PlaywrightCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPoolOptions} are available directly in the `PlaywrightCrawler` constructor.
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * Note that the pool of Playwright instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
  *
@@ -172,16 +143,18 @@ export interface PlaywrightCrawlerOptions extends BrowserCrawlerOptions<Playwrig
  * ```
  * @category Crawlers
  */
-export declare class PlaywrightCrawler extends BrowserCrawler<{
+export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawler<Page, Response, {
     browserPlugins: [PlaywrightPlugin];
-}, LaunchOptions, PlaywrightCrawlingContext> {
-    private readonly options;
-    readonly config: Configuration;
+}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes> {
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -193,17 +166,17 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
 // @ts-ignore optional peer dependency or compatibility with es2022
         headless: import("ow").AnyPredicate<string | boolean>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        browserPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        remoteBrowser: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -220,24 +193,42 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        blockedStatusCodes: import("ow").ArrayPredicate<number>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -246,20 +237,50 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
         maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options?: PlaywrightCrawlerOptions, config?: Configuration);
-    protected _runRequestHandler(context: PlaywrightCrawlingContext): Promise<void>;
+    constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
+// @ts-ignore optional peer dependency or compatibility with es2022
+    protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response, Dictionary, Dictionary> & {
+        injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
+        injectJQuery: () => Promise<void>;
+        blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
+        waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<import("@crawlee/browser").CheerioAPI>;
+        infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
+        listDownloads: () => Promise<Download[]>;
+        saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        compileScript: (scriptString: string, ctx?: Dictionary) => import("./utils/playwright-utils.js").CompiledScriptFunction;
+        closeCookieModals: () => Promise<void>;
+        handleCloudflareChallenge: (options?: HandleCloudflareChallengeOptions) => Promise<Response | undefined>;
+    }>;
     protected _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
+    private enhanceContext;
 }
+/**
+ * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
+ * and propagates the post-challenge {@link Response} back into the crawling context via its return value.
+ *
+ * **Example usage**
+ * ```ts
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
+ *
+ * const crawler = new PlaywrightCrawler({
+ *     postNavigationHooks: [handleCloudflareChallengeHook()],
+ * });
+ * ```
+ */
+export declare function handleCloudflareChallengeHook(options?: HandleCloudflareChallengeOptions): PlaywrightHook;
 /**
  * Creates new {@link Router} instance that works based on request labels.
  * This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
@@ -284,6 +305,6 @@ export declare class PlaywrightCrawler extends BrowserCrawler<{
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
-//# sourceMappingURL=playwright-crawler.d.ts.map
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

package/internals/playwright-crawler.js

@@ -1,7 +1,7 @@
-import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';
+import { BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';
 import ow from 'ow';
 import { PlaywrightLauncher } from './playwright-launcher.js';
-import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.js';
+import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless Chromium, Firefox and Webkit browsers with [Playwright](https://github.com/microsoft/playwright).
@@ -13,24 +13,26 @@ import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.j
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
- * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
  * `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
  * and then calls the function provided by user as the {@link PlaywrightCrawlerOptions.requestHandler} option.
  *
- * New pages are only opened when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link PlaywrightCrawlerOptions.autoscaledPoolOptions}
- * parameter of the `PlaywrightCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPoolOptions} are available directly in the `PlaywrightCrawler` constructor.
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * `PlaywrightCrawler` constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * Note that the pool of Playwright instances is internally managed by the [BrowserPool](https://github.com/apify/browser-pool) class.
  *
@@ -66,19 +68,19 @@ import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.j
  * @category Crawlers
  */
 export class PlaywrightCrawler extends BrowserCrawler {
-    options;
-    config;
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
         browserPoolOptions: ow.optional.object,
         launcher: ow.optional.object,
+        ignoreIframes: ow.optional.boolean,
+        ignoreShadowRoots: ow.optional.boolean,
     };
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options = {}) {
         ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
-        const { launchContext = {}, headless, ...browserCrawlerOptions } = options;
+        const { launchContext = {}, headless, contextPipelineBuilder, ...browserCrawlerOptions } = options;
         const browserPoolOptions = {
             ...options.browserPoolOptions,
         };
@@ -95,45 +97,87 @@ export class PlaywrightCrawler extends BrowserCrawler {
             launchContext.launchOptions ??= {};
             launchContext.launchOptions.headless = headless;
         }
-        const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
+        const playwrightLauncher = new PlaywrightLauncher(launchContext, options.configuration);
         browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
-        super({ ...browserCrawlerOptions, launchContext, browserPoolOptions }, config);
-        this.options = options;
-        this.config = config;
+        super({
+            ...browserCrawlerOptions,
+            launchContext,
+            browserPoolOptions,
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+        });
     }
-    async _runRequestHandler(context) {
-        registerUtilsToContext(context, this.options);
-        await super._runRequestHandler(context);
+    buildContextPipeline() {
+        return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });
     }
     async _navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
+    async enhanceContext(context) {
+        const waitForSelector = async (selector, timeoutMs = 5_000) => {
+            const locator = context.page.locator(selector).first();
+            await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+        };
+        const downloads = [];
+        context.page.on('download', (download) => downloads.push(download));
+        return {
+            injectFile: async (filePath, options) => playwrightUtils.injectFile(context.page, filePath, options),
+            injectJQuery: async () => {
+                if (context.request.state === RequestState.BEFORE_NAV) {
+                    context.log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
+                    await playwrightUtils.injectJQuery(context.page);
+                    return;
+                }
+                await playwrightUtils.injectJQuery(context.page, { surviveNavigations: false });
+            },
+            blockRequests: async (options) => playwrightUtils.blockRequests(context.page, options),
+            waitForSelector,
+            parseWithCheerio: async (selector, timeoutMs = 5_000) => {
+                if (selector) {
+                    await waitForSelector(selector, timeoutMs);
+                }
+                return playwrightUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
+            },
+            infiniteScroll: async (options) => playwrightUtils.infiniteScroll(context.page, options),
+            listDownloads: async () => downloads,
+            saveSnapshot: async (options) => playwrightUtils.saveSnapshot(context.page, {
+                ...options,
+                configuration: serviceLocator.getConfiguration(),
+            }),
+            enqueueLinksByClickingElements: async (options) => playwrightUtils.enqueueLinksByClickingElements({
+                ...options,
+                page: context.page,
+                requestManager: this.requestManager,
+            }),
+            compileScript: (scriptString, ctx) => playwrightUtils.compileScript(scriptString, ctx),
+            closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page),
+            handleCloudflareChallenge: async (options) => {
+                return playwrightUtils.handleCloudflareChallenge(context.page, context.request.url, options);
+            },
+        };
+    }
 }
 /**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
- * Defaults to the {@link PlaywrightCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<PlaywrightCrawlingContext>()`.
+ * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
+ * and propagates the post-challenge {@link Response} back into the crawling context via its return value.
  *
+ * **Example usage**
  * ```ts
- * import { PlaywrightCrawler, createPlaywrightRouter } from 'crawlee';
- *
- * const router = createPlaywrightRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
  *
  * const crawler = new PlaywrightCrawler({
- *     requestHandler: router,
+ *     postNavigationHooks: [handleCloudflareChallengeHook()],
  * });
- * await crawler.run();
  * ```
  */
-export function createPlaywrightRouter(routes) {
-    return Router.create(routes);
+export function handleCloudflareChallengeHook(options) {
+    return async (context) => {
+        const response = await context.handleCloudflareChallenge(options);
+        if (response !== undefined) {
+            return { response };
+        }
+        return undefined;
+    };
+}
+export function createPlaywrightRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=playwright-crawler.js.map

package/internals/playwright-launcher.d.ts

@@ -70,7 +70,7 @@ export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOpti
  * @ignore
  */
 export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin> {
-    readonly config: Configuration;
+    readonly configuration: Configuration;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
@@ -84,6 +84,8 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
         useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -94,7 +96,7 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.
      */
-    constructor(launchContext?: PlaywrightLaunchContext, config?: Configuration);
+    constructor(launchContext?: PlaywrightLaunchContext, configuration?: Configuration);
 }
 /**
  * Launches headless browsers using Playwright pre-configured to work within the Apify platform.
@@ -125,9 +127,8 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
  *   Optional settings passed to `browserType.launch()`. In addition to
  *   [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)
  *   the object may contain our own  {@link PlaywrightLaunchContext} that enable additional features.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Playwright's `Browser` instance.
  */
-export declare function launchPlaywright(launchContext?: PlaywrightLaunchContext, config?: Configuration): Promise<Browser>;
-//# sourceMappingURL=playwright-launcher.d.ts.map
+export declare function launchPlaywright(launchContext?: PlaywrightLaunchContext, configuration?: Configuration): Promise<Browser>;

package/internals/playwright-launcher.js

@@ -6,7 +6,7 @@ import ow from 'ow';
  * @ignore
  */
 export class PlaywrightLauncher extends BrowserLauncher {
-    config;
+    configuration;
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
         launcher: ow.optional.object,
@@ -15,7 +15,7 @@ export class PlaywrightLauncher extends BrowserLauncher {
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.
      */
-    constructor(launchContext = {}, config = Configuration.getGlobalConfig()) {
+    constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
         ow(launchContext, 'PlaywrightLauncherOptions', ow.object.exactShape(PlaywrightLauncher.optionsShape));
         const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = launchContext;
         const { launchOptions = {}, ...rest } = launchContext;
@@ -23,11 +23,11 @@ export class PlaywrightLauncher extends BrowserLauncher {
             ...rest,
             launchOptions: {
                 ...launchOptions,
-                executablePath: getDefaultExecutablePath(launchContext, config),
+                executablePath: getDefaultExecutablePath(launchContext, configuration),
             },
             launcher,
-        }, config);
-        this.config = config;
+        }, configuration);
+        this.configuration = configuration;
         this.Plugin = PlaywrightPlugin;
     }
 }
@@ -36,8 +36,8 @@ export class PlaywrightLauncher extends BrowserLauncher {
  * @returns default path to browser.
  * @ignore
  */
-function getDefaultExecutablePath(launchContext, config) {
-    const pathFromPlaywrightImage = config.get('defaultBrowserPath');
+function getDefaultExecutablePath(launchContext, configuration) {
+    const pathFromPlaywrightImage = configuration.defaultBrowserPath;
     const { launchOptions = {} } = launchContext;
     if (launchOptions.executablePath) {
         return launchOptions.executablePath;
@@ -79,12 +79,11 @@ function getDefaultExecutablePath(launchContext, config) {
  *   Optional settings passed to `browserType.launch()`. In addition to
  *   [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)
  *   the object may contain our own  {@link PlaywrightLaunchContext} that enable additional features.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Playwright's `Browser` instance.
  */
-export async function launchPlaywright(launchContext, config = Configuration.getGlobalConfig()) {
-    const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
+export async function launchPlaywright(launchContext, configuration = Configuration.getGlobalConfiguration()) {
+    const playwrightLauncher = new PlaywrightLauncher(launchContext, configuration);
     return playwrightLauncher.launch();
 }
-//# sourceMappingURL=playwright-launcher.js.map