@crawlee/playwright 4.0.0-beta.13 → 4.0.0-beta.131

package/internals/playwright-launcher.d.ts

@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PlaywrightPlugin } from '@crawlee/browser-pool';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Browser, BrowserType, LaunchOptions } from 'playwright';
+import { z } from 'zod';
 /**
  * Apify extends the launch options of Playwright.
  * You can use any of the Playwright compatible
@@ -70,31 +71,41 @@ export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOpti
  * @ignore
  */
 export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin> {
-    readonly config: Configuration;
+    readonly configuration: Configuration;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launchContextOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    };
+    protected static optionsSchema: z.ZodObject<{
 // @ts-ignore optional peer dependency or compatibility with es2022
-        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-    };
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.
      */
-    constructor(launchContext?: PlaywrightLaunchContext, config?: Configuration);
+    constructor(launchContext?: PlaywrightLaunchContext, configuration?: Configuration);
 }
 /**
  * Launches headless browsers using Playwright pre-configured to work within the Apify platform.
@@ -125,9 +136,8 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
  *   Optional settings passed to `browserType.launch()`. In addition to
  *   [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)
  *   the object may contain our own  {@link PlaywrightLaunchContext} that enable additional features.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Playwright's `Browser` instance.
  */
-export declare function launchPlaywright(launchContext?: PlaywrightLaunchContext, config?: Configuration): Promise<Browser>;
-//# sourceMappingURL=playwright-launcher.d.ts.map
+export declare function launchPlaywright(launchContext?: PlaywrightLaunchContext, configuration?: Configuration): Promise<Browser>;

package/internals/playwright-launcher.js

@@ -1,33 +1,35 @@
-import { BrowserLauncher, Configuration } from '@crawlee/browser';
+import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
 import { PlaywrightPlugin } from '@crawlee/browser-pool';
-import ow from 'ow';
+import { z } from 'zod';
 /**
  * `PlaywrightLauncher` is based on the `BrowserLauncher`. It launches `playwright` browser instance.
  * @ignore
  */
 export class PlaywrightLauncher extends BrowserLauncher {
-    config;
+    configuration;
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
-        launcher: ow.optional.object,
-        launchContextOptions: ow.optional.object,
+        // Passthrough schemas — the launcher module object must keep its prototype through parsing.
+        launcher: schemas.anyObject.optional(),
+        launchContextOptions: schemas.anyObject.optional(),
     };
+    static optionsSchema = z.strictObject(PlaywrightLauncher.optionsShape);
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.
      */
-    constructor(launchContext = {}, config = Configuration.getGlobalConfig()) {
-        ow(launchContext, 'PlaywrightLauncherOptions', ow.object.exactShape(PlaywrightLauncher.optionsShape));
-        const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = launchContext;
-        const { launchOptions = {}, ...rest } = launchContext;
+    constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
+        const parsedContext = parseArgument(launchContext, PlaywrightLauncher.optionsSchema, 'PlaywrightLaunchContext');
+        const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = parsedContext;
+        const { launchOptions = {}, ...rest } = parsedContext;
         super({
             ...rest,
             launchOptions: {
                 ...launchOptions,
-                executablePath: getDefaultExecutablePath(launchContext, config),
+                executablePath: getDefaultExecutablePath(parsedContext, configuration),
             },
             launcher,
-        }, config);
-        this.config = config;
+        }, configuration);
+        this.configuration = configuration;
         this.Plugin = PlaywrightPlugin;
     }
 }
@@ -36,8 +38,8 @@ export class PlaywrightLauncher extends BrowserLauncher {
  * @returns default path to browser.
  * @ignore
  */
-function getDefaultExecutablePath(launchContext, config) {
-    const pathFromPlaywrightImage = config.get('defaultBrowserPath');
+function getDefaultExecutablePath(launchContext, configuration) {
+    const pathFromPlaywrightImage = configuration.defaultBrowserPath;
     const { launchOptions = {} } = launchContext;
     if (launchOptions.executablePath) {
         return launchOptions.executablePath;
@@ -79,12 +81,11 @@ function getDefaultExecutablePath(launchContext, config) {
  *   Optional settings passed to `browserType.launch()`. In addition to
  *   [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)
  *   the object may contain our own  {@link PlaywrightLaunchContext} that enable additional features.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Playwright's `Browser` instance.
  */
-export async function launchPlaywright(launchContext, config = Configuration.getGlobalConfig()) {
-    const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
+export async function launchPlaywright(launchContext, configuration = Configuration.getGlobalConfiguration()) {
+    const playwrightLauncher = new PlaywrightLauncher(launchContext, configuration);
     return playwrightLauncher.launch();
 }
-//# sourceMappingURL=playwright-launcher.js.map

package/internals/utils/playwright-utils.d.ts

@@ -17,11 +17,11 @@
  * ```
  * @module playwrightUtils
  */
-import { Configuration, type Request, type Session } from '@crawlee/browser';
-import type { BatchAddRequestsResult } from '@crawlee/types';
-import { type CheerioRoot, type Dictionary } from '@crawlee/utils';
+import { Configuration, type Request } from '@crawlee/browser';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { Page, Response } from 'playwright';
+import type { Download, Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
@@ -265,9 +265,9 @@ export interface SaveSnapshotOptions {
     keyValueStoreName?: string | null;
     /**
      * Configuration of the crawler that will be used to save the snapshot.
-     * @default Configuration.getGlobalConfig()
+     * @default Configuration.getGlobalConfiguration()
      */
-    config?: Configuration;
+    configuration?: Configuration;
 }
 /**
  * Saves a full screenshot and HTML of the current page into a Key-Value store.
@@ -303,6 +303,13 @@ export interface HandleCloudflareChallengeOptions {
     isChallengeCallback?: (page: Page) => Promise<boolean>;
     /** Allows overriding the detection of Cloudflare "blocked page". */
     isBlockedCallback?: (page: Page) => Promise<boolean>;
+    /** Allows overriding how the checkbox click position is calculated. */
+    clickPositionCallback?: (page: Page) => Promise<{
+        x: number;
+        y: number;
+    } | null>;
+    /** Optional delay (in seconds) before the first click attempt on the challenge checkbox. Defaults to 1s. */
+    preChallengeSleepSecs?: number;
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -311,24 +318,24 @@ export interface HandleCloudflareChallengeOptions {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-declare function handleCloudflareChallenge(page: Page, url: string, session?: Session, options?: HandleCloudflareChallengeOptions): Promise<void>;
-/** @internal */
+declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
 export interface PlaywrightContextUtils {
     /**
      * Injects a JavaScript file into current `page`.
@@ -447,8 +454,7 @@ export interface PlaywrightContextUtils {
      * in `href` elements, but rather navigations are triggered in click handlers.
      * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
      * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -469,9 +475,9 @@ export interface PlaywrightContextUtils {
      * async requestHandler({ enqueueLinksByClickingElements }) {
      *     await enqueueLinksByClickingElements({
      *         selector: 'a.product-detail',
-     *         globs: [
-     *             'https://www.example.com/handbags/**'
-     *             'https://www.example.com/purses/**'
+     *         include: [
+     *             'https://www.example.com/handbags/**',
+     *             'https://www.example.com/purses/**',
      *         ],
      *     });
      * });
@@ -479,7 +485,7 @@ export interface PlaywrightContextUtils {
      *
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
     /**
      * Compiles a Playwright script into an async function that may be executed at any time
      * by providing it with the following object:
@@ -509,6 +515,15 @@ export interface PlaywrightContextUtils {
     compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
     /**
      * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+     *
+     * Note that this method requires the idcac-playwright package to be installed.
+     * Crawlee does not include it by default due to licensing issues.
+     *
+     * To use this method, please install the package manually by running:
+     *
+     * ```bash
+     * npm install idcac-playwright
+     * ```
      */
     closeCookieModals(): Promise<void>;
     /**
@@ -518,20 +533,43 @@ export interface PlaywrightContextUtils {
      * result in a SessionError which will be automatically retried, so only successful requests will get
      * into the `requestHandler`.
      *
-     * Works best with camoufox.
+     * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned,
+     * which can be returned from the hook to update the crawling context's `response`. For the common case,
+     * prefer the pre-wrapped {@link handleCloudflareChallengeHook} hook.
      *
      * **Example usage**
      * ```ts
      * postNavigationHooks: [
-     *     async ({ handleCloudflareChallenge }) => {
-     *         await handleCloudflareChallenge();
-     *     },
+     *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
      * ],
      * ```
      *
      * @param [options]
      */
-    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<void>;
+    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
+    /**
+     * Returns the list of {@link https://playwright.dev/docs/api/class-download | Download} objects
+     * collected during the current page navigation and request handler.
+     *
+     * Useful for accessing files that the page downloads automatically.
+     * For most use cases, prefer re-enqueueing the URL to {@link FileDownload}.
+     * Use this only when direct access to the Playwright `Download` object is required.
+     *
+     * **Example usage**
+     * ```ts
+     * requestHandler: async ({ listDownloads }) => {
+     *     for (const download of await listDownloads()) {
+     *         try {
+     *             const stream = await download.createReadStream();
+     *             // stream to storage...
+     *         } catch {
+     *             // download failed or was cancelled
+     *         }
+     *     }
+     * },
+     * ```
+     */
+    listDownloads(): Promise<Download[]>;
 }
 export { enqueueLinksByClickingElements };
 /** @internal */
@@ -549,4 +587,3 @@ export declare const playwrightUtils: {
     RenderingTypePredictor: typeof RenderingTypePredictor;
     handleCloudflareChallenge: typeof handleCloudflareChallenge;
 };
-//# sourceMappingURL=playwright-utils.d.ts.map