@crawlee/playwright 4.0.0-beta.8 → 4.0.0-beta.81

package/internals/playwright-crawler.js

@@ -1,7 +1,7 @@
-import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';
+import { BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';
 import ow from 'ow';
 import { PlaywrightLauncher } from './playwright-launcher.js';
-import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.js';
+import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless Chromium, Firefox and Webkit browsers with [Playwright](https://github.com/microsoft/playwright).
@@ -13,13 +13,15 @@ import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.j
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
- * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link PlaywrightCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link PlaywrightCrawlerOptions.requestList|`requestList`} and {@link PlaywrightCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
@@ -66,19 +68,19 @@ import { gotoExtended, registerUtilsToContext } from './utils/playwright-utils.j
  * @category Crawlers
  */
 export class PlaywrightCrawler extends BrowserCrawler {
-    options;
-    config;
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
         browserPoolOptions: ow.optional.object,
         launcher: ow.optional.object,
+        ignoreIframes: ow.optional.boolean,
+        ignoreShadowRoots: ow.optional.boolean,
     };
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options = {}) {
         ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
-        const { launchContext = {}, headless, ...browserCrawlerOptions } = options;
+        const { launchContext = {}, headless, contextPipelineBuilder, ...browserCrawlerOptions } = options;
         const browserPoolOptions = {
             ...options.browserPoolOptions,
         };
@@ -95,45 +97,84 @@ export class PlaywrightCrawler extends BrowserCrawler {
             launchContext.launchOptions ??= {};
             launchContext.launchOptions.headless = headless;
         }
-        const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
+        const playwrightLauncher = new PlaywrightLauncher(launchContext, options.configuration);
         browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
-        super({ ...browserCrawlerOptions, launchContext, browserPoolOptions }, config);
-        this.options = options;
-        this.config = config;
+        super({
+            ...browserCrawlerOptions,
+            launchContext,
+            browserPoolOptions,
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+        });
     }
-    async _runRequestHandler(context) {
-        registerUtilsToContext(context, this.options);
-        await super._runRequestHandler(context);
+    buildContextPipeline() {
+        return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });
     }
     async _navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
+    async enhanceContext(context) {
+        const waitForSelector = async (selector, timeoutMs = 5_000) => {
+            const locator = context.page.locator(selector).first();
+            await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+        };
+        const downloads = [];
+        context.page.on('download', (download) => downloads.push(download));
+        return {
+            injectFile: async (filePath, options) => playwrightUtils.injectFile(context.page, filePath, options),
+            injectJQuery: async () => {
+                if (context.request.state === RequestState.BEFORE_NAV) {
+                    context.log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
+                    await playwrightUtils.injectJQuery(context.page);
+                    return;
+                }
+                await playwrightUtils.injectJQuery(context.page, { surviveNavigations: false });
+            },
+            blockRequests: async (options) => playwrightUtils.blockRequests(context.page, options),
+            waitForSelector,
+            parseWithCheerio: async (selector, timeoutMs = 5_000) => {
+                if (selector) {
+                    await waitForSelector(selector, timeoutMs);
+                }
+                return playwrightUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
+            },
+            infiniteScroll: async (options) => playwrightUtils.infiniteScroll(context.page, options),
+            listDownloads: async () => downloads,
+            saveSnapshot: async (options) => playwrightUtils.saveSnapshot(context.page, { ...options, config: serviceLocator.getConfiguration() }),
+            enqueueLinksByClickingElements: async (options) => playwrightUtils.enqueueLinksByClickingElements({
+                ...options,
+                page: context.page,
+                requestManager: this.requestManager,
+            }),
+            compileScript: (scriptString, ctx) => playwrightUtils.compileScript(scriptString, ctx),
+            closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page),
+            handleCloudflareChallenge: async (options) => {
+                return playwrightUtils.handleCloudflareChallenge(context.page, context.request.url, options);
+            },
+        };
+    }
 }
 /**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a `requestHandler` of your {@link PlaywrightCrawler}.
- * Defaults to the {@link PlaywrightCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<PlaywrightCrawlingContext>()`.
+ * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}
+ * and propagates the post-challenge {@link Response} back into the crawling context via its return value.
  *
+ * **Example usage**
  * ```ts
- * import { PlaywrightCrawler, createPlaywrightRouter } from 'crawlee';
- *
- * const router = createPlaywrightRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
+ * import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
  *
  * const crawler = new PlaywrightCrawler({
- *     requestHandler: router,
+ *     postNavigationHooks: [handleCloudflareChallengeHook()],
  * });
- * await crawler.run();
  * ```
  */
-export function createPlaywrightRouter(routes) {
-    return Router.create(routes);
+export function handleCloudflareChallengeHook(options) {
+    return async (context) => {
+        const response = await context.handleCloudflareChallenge(options);
+        if (response !== undefined) {
+            return { response };
+        }
+        return undefined;
+    };
+}
+export function createPlaywrightRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-//# sourceMappingURL=playwright-crawler.js.map

package/internals/playwright-launcher.d.ts

@@ -84,6 +84,8 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
         useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -130,4 +132,3 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
  *   Promise that resolves to Playwright's `Browser` instance.
  */
 export declare function launchPlaywright(launchContext?: PlaywrightLaunchContext, config?: Configuration): Promise<Browser>;
-//# sourceMappingURL=playwright-launcher.d.ts.map

package/internals/playwright-launcher.js

@@ -37,7 +37,7 @@ export class PlaywrightLauncher extends BrowserLauncher {
  * @ignore
  */
 function getDefaultExecutablePath(launchContext, config) {
-    const pathFromPlaywrightImage = config.get('defaultBrowserPath');
+    const pathFromPlaywrightImage = config.defaultBrowserPath;
     const { launchOptions = {} } = launchContext;
     if (launchOptions.executablePath) {
         return launchOptions.executablePath;
@@ -87,4 +87,3 @@ export async function launchPlaywright(launchContext, config = Configuration.get
     const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
     return playwrightLauncher.launch();
 }
-//# sourceMappingURL=playwright-launcher.js.map

package/internals/utils/playwright-utils.d.ts

@@ -17,14 +17,13 @@
  * ```
  * @module playwrightUtils
  */
-import { Configuration, type Request, type Session } from '@crawlee/browser';
+import { Configuration, type Request } from '@crawlee/browser';
 import type { BatchAddRequestsResult } from '@crawlee/types';
 import { type CheerioRoot, type Dictionary } from '@crawlee/utils';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { Page, Response } from 'playwright';
+import type { Download, Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext } from '../playwright-crawler.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
 export interface InjectFileOptions {
     /**
@@ -290,7 +289,7 @@ export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions):
  */
 export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
 export declare function closeCookieModals(page: Page): Promise<void>;
-interface HandleCloudflareChallengeOptions {
+export interface HandleCloudflareChallengeOptions {
     /** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */
     verbose?: boolean;
     /** How long should we wait after the challenge is completed for the final page to load. */
@@ -304,6 +303,13 @@ interface HandleCloudflareChallengeOptions {
     isChallengeCallback?: (page: Page) => Promise<boolean>;
     /** Allows overriding the detection of Cloudflare "blocked page". */
     isBlockedCallback?: (page: Page) => Promise<boolean>;
+    /** Allows overriding how the checkbox click position is calculated. */
+    clickPositionCallback?: (page: Page) => Promise<{
+        x: number;
+        y: number;
+    } | null>;
+    /** Optional delay (in seconds) before the first click attempt on the challenge checkbox. Defaults to 1s. */
+    preChallengeSleepSecs?: number;
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -312,23 +318,24 @@ interface HandleCloudflareChallengeOptions {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-declare function handleCloudflareChallenge(page: Page, url: string, session?: Session, options?: HandleCloudflareChallengeOptions): Promise<void>;
+declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
 /** @internal */
 export interface PlaywrightContextUtils {
     /**
@@ -480,7 +487,7 @@ export interface PlaywrightContextUtils {
      *
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
     /**
      * Compiles a Playwright script into an async function that may be executed at any time
      * by providing it with the following object:
@@ -510,6 +517,15 @@ export interface PlaywrightContextUtils {
     compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
     /**
      * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+     *
+     * Note that this method requires the idcac-playwright package to be installed.
+     * Crawlee does not include it by default due to licensing issues.
+     *
+     * To use this method, please install the package manually by running:
+     *
+     * ```bash
+     * npm install idcac-playwright
+     * ```
      */
     closeCookieModals(): Promise<void>;
     /**
@@ -519,22 +535,44 @@ export interface PlaywrightContextUtils {
      * result in a SessionError which will be automatically retried, so only successful requests will get
      * into the `requestHandler`.
      *
-     * Works best with camoufox.
+     * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned,
+     * which can be returned from the hook to update the crawling context's `response`. For the common case,
+     * prefer the pre-wrapped {@link handleCloudflareChallengeHook} hook.
      *
      * **Example usage**
      * ```ts
      * postNavigationHooks: [
-     *     async ({ handleCloudflareChallenge }) => {
-     *         await handleCloudflareChallenge();
-     *     },
+     *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
      * ],
      * ```
      *
      * @param [options]
      */
-    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<void>;
+    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
+    /**
+     * Returns the list of {@link https://playwright.dev/docs/api/class-download | Download} objects
+     * collected during the current page navigation and request handler.
+     *
+     * Useful for accessing files that the page downloads automatically.
+     * For most use cases, prefer re-enqueueing the URL to {@link FileDownload}.
+     * Use this only when direct access to the Playwright `Download` object is required.
+     *
+     * **Example usage**
+     * ```ts
+     * requestHandler: async ({ listDownloads }) => {
+     *     for (const download of await listDownloads()) {
+     *         try {
+     *             const stream = await download.createReadStream();
+     *             // stream to storage...
+     *         } catch {
+     *             // download failed or was cancelled
+     *         }
+     *     }
+     * },
+     * ```
+     */
+    listDownloads(): Promise<Download[]>;
 }
-export declare function registerUtilsToContext(context: PlaywrightCrawlingContext, crawlerOptions: PlaywrightCrawlerOptions): void;
 export { enqueueLinksByClickingElements };
 /** @internal */
 export declare const playwrightUtils: {
@@ -551,4 +589,3 @@ export declare const playwrightUtils: {
     RenderingTypePredictor: typeof RenderingTypePredictor;
     handleCloudflareChallenge: typeof handleCloudflareChallenge;
 };
-//# sourceMappingURL=playwright-utils.d.ts.map