@crawlee/puppeteer 4.0.0-beta.99 → 4.0.0-rc.0

package/index.d.ts

@@ -1,4 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/puppeteer-browser-pool.js';
 export * from './internals/puppeteer-crawler.js';
 export * from './internals/puppeteer-launcher.js';
 export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';

package/index.js

@@ -1,4 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/puppeteer-browser-pool.js';
 export * from './internals/puppeteer-crawler.js';
 export * from './internals/puppeteer-launcher.js';
 export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';

package/internals/enqueue-links/click-elements.d.ts

@@ -1,4 +1,4 @@
-import type { GlobInput, IRequestManager, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from '@crawlee/browser';
+import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { ClickOptions, Page, Target } from 'puppeteer';
@@ -25,64 +25,29 @@ export interface EnqueueLinksByClickingElementsOptions {
      */
     clickOptions?: ClickOptions;
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
+     * An array of URL patterns that URLs must match to be enqueued.
      *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, then the function
-     * enqueues all the intercepted navigation requests produced by the page
-     * after clicking on elements matching the provided CSS selector.
-     */
-    globs?: GlobInput[];
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be enqueued.
-     *
-     * The plain objects must include either the `glob` property or the `regexp` property.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
      * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExpInput)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * If `regexps` is an empty array or `undefined`, then the function
+     * If `include` is an empty array or `undefined`, then the function
      * enqueues all the intercepted navigation requests produced by the page
      * after clicking on elements matching the provided CSS selector.
      */
-    regexps?: RegExpInput[];
+    include?: UrlPatternInput[];
     /**
-     * *NOTE:* In future versions of SDK the options will be removed.
-     * Please use `globs` or `regexps` instead.
-     *
-     * An array of {@link PseudoUrl} strings or plain objects
-     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * An array of URL patterns. Matching URLs will **not** be enqueued.
      *
-     * With a pseudo-URL string, the matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * If `pseudoUrls` is an empty array or `undefined`, then the function
-     * enqueues all the intercepted navigation requests produced by the page
-     * after clicking on elements matching the provided CSS selector.
-     *
-     * @deprecated prefer using `globs` or `regexps` instead
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    pseudoUrls?: PseudoUrlInput[];
+    exclude?: readonly UrlPatternInput[];
     /**
-     * After {@link Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
+     * After request options are filtered by `include`/`exclude` patterns,
      * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
      * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
      * but differ in methods or payloads, or to dynamically update or create `userData`.
@@ -97,8 +62,8 @@ export interface EnqueueLinksByClickingElementsOptions {
      * }
      * ```
      *
-     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite
+     * the global `label` option.
      *
      * The function receives a {@link RequestOptions} object and can return either:
      * - The modified {@link RequestOptions} object
@@ -159,8 +124,7 @@ export interface EnqueueLinksByClickingElementsOptions {
  * in `href` elements, but rather navigations are triggered in click handlers.
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
  *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
  *
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -182,9 +146,9 @@ export interface EnqueueLinksByClickingElementsOptions {
  *   page,
  *   requestManager,
  *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
+ *   include: [
+ *       'https://www.example.com/handbags/*',
+ *       'https://www.example.com/purses/*',
  *   ],
  * });
  * ```

package/internals/enqueue-links/click-elements.js

@@ -1,9 +1,25 @@
 import { URL } from 'node:url';
-import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, Request, serviceLocator, } from '@crawlee/browser';
-import ow from 'ow';
+import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, parseArgument, urlPatternSchema, Request, schemas, serviceLocator, } from '@crawlee/browser';
+import { z } from 'zod';
 import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
 const STARTING_Z_INDEX = 2147400000;
 const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
+const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
+    page: schemas.objectWithKeys(['goto', 'evaluate']),
+    requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
+    selector: z.string(),
+    userData: schemas.anyObject.optional(),
+    clickOptions: schemas.anyObject.optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    transformRequestFunction: schemas.anyFunction.optional(),
+    waitForPageIdleSecs: schemas.anyNumber.default(1),
+    maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
+    label: z.string().optional(),
+    forefront: z.boolean().optional(),
+    skipNavigation: z.boolean().optional(),
+    onSkippedRequest: schemas.anyFunction.optional(),
+});
 /**
  * The function finds elements matching a specific CSS selector in a Puppeteer page,
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -13,8 +29,7 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
  * in `href` elements, but rather navigations are triggered in click handlers.
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
  *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
  *
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -36,9 +51,9 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
  *   page,
  *   requestManager,
  *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
+ *   include: [
+ *       'https://www.example.com/handbags/*',
+ *       'https://www.example.com/purses/*',
  *   ],
  * });
  * ```
@@ -46,51 +61,12 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
  */
 export async function enqueueLinksByClickingElements(options) {
-    ow(options, ow.object.exactShape({
-        page: ow.object.hasKeys('goto', 'evaluate'),
-        requestManager: ow.object.hasKeys('fetchNextRequest', 'addRequestsBatched'),
-        selector: ow.string,
-        userData: ow.optional.object,
-        clickOptions: ow.optional.object,
-        pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
-        globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
-        regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
-        exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
-        transformRequestFunction: ow.optional.function,
-        waitForPageIdleSecs: ow.optional.number,
-        maxWaitForPageIdleSecs: ow.optional.number,
-        label: ow.optional.string,
-        forefront: ow.optional.boolean,
-        skipNavigation: ow.optional.boolean,
-        onSkippedRequest: ow.optional.function,
-    }));
-    const { page, requestManager, selector, clickOptions, 
-    // oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
-    pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
+    const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
+    const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
     const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
     const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
-    const urlExcludePatternObjects = [];
-    const urlPatternObjects = [];
-    if (exclude?.length) {
-        for (const excl of exclude) {
-            if (typeof excl === 'string' || 'glob' in excl) {
-                urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
-            }
-            else if (excl instanceof RegExp || 'regexp' in excl) {
-                urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
-            }
-        }
-    }
-    if (pseudoUrls?.length) {
-        getLog().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
-        urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
-    }
-    if (globs?.length) {
-        urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
-    }
-    if (regexps?.length) {
-        urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
-    }
+    const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
+    const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
     const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
         page,
         selector,
@@ -98,7 +74,7 @@ export async function enqueueLinksByClickingElements(options) {
         maxWaitForPageIdleMillis,
         clickOptions,
     });
-    const requestOptions = createRequestOptions(interceptedRequests, options);
+    const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
     const skippedByFilters = [];
     let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
     if (onSkippedRequest && skippedByFilters.length > 0) {

package/internals/puppeteer-browser-pool.d.ts

@@ -0,0 +1,55 @@
+import type { Configuration } from '@crawlee/browser';
+import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
+// @ts-ignore optional peer dependency or compatibility with es2022
+import type { Page } from 'puppeteer';
+import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
+/** A {@link BrowserPool} of Puppeteer browsers, as built by {@link puppeteerBrowserPool}. */
+export type PuppeteerBrowserPool = BrowserPool<{
+    browserPlugins: [PuppeteerPlugin];
+}, [PuppeteerPlugin]>;
+export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {
+    /** How to launch the browser: proxy, user data dir, whether to use full Chrome, ... */
+    launchContext?: PuppeteerLaunchContext;
+    /**
+     * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
+     * Defaults to `true`, and can also be set via {@link Configuration}.
+     */
+    headless?: boolean | 'new' | 'old';
+    /** Configuration to read the browser defaults from. Defaults to the global configuration. */
+    configuration?: Configuration;
+}
+export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
+}
+/**
+ * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PuppeteerCrawler({
+ *     browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
+ *
+ * A {@link PuppeteerCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;

package/internals/puppeteer-browser-pool.js

@@ -0,0 +1,48 @@
+import { PuppeteerLauncher } from './puppeteer-launcher.js';
+/**
+ * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PuppeteerCrawler({
+ *     browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export function puppeteerBrowserPool(options = {}) {
+    const { launchContext, headless, configuration, ...poolOptions } = options;
+    return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
+}
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
+ *
+ * A {@link PuppeteerCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * @category Browser management
+ */
+export function remotePuppeteerBrowserPool(options) {
+    const { launchContext, headless, configuration, ...remoteOptions } = options;
+    return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
+}
+function puppeteerLauncher(launchContext = {}, headless, configuration) {
+    return new PuppeteerLauncher(headless == null
+        ? launchContext
+        : {
+            ...launchContext,
+            launchOptions: { ...launchContext.launchOptions, headless: headless },
+        }, configuration);
+}

package/internals/puppeteer-crawler.d.ts

@@ -1,26 +1,27 @@
 import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import { BrowserCrawler } from '@crawlee/browser';
-import type { PuppeteerPlugin } from '@crawlee/browser-pool';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
+import { z } from 'zod';
 import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
 import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
 import type { InterceptHandler } from './utils/puppeteer_request_interception.js';
 import type { BlockRequestsOptions, DirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, PuppeteerContextUtils, SaveSnapshotOptions } from './utils/puppeteer_utils.js';
 export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
-export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
+export interface PuppeteerCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
 }
-// @ts-ignore optional peer dependency or compatibility with es2022
-export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {
-}
-export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
-    browserPlugins: [PuppeteerPlugin];
-}, Routes> {
+export type PuppeteerHook<UserData extends Dictionary = any> = BrowserHook<PuppeteerCrawlingContext<UserData>>;
+export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     /**
      * Options used by {@link launchPuppeteer} to start new Puppeteer instances.
      */
     launchContext?: PuppeteerLaunchContext;
+    /**
+     * Whether to run browser in headless mode. Defaults to `true`.
+     * Can be also set via {@link Configuration}.
+     */
+    headless?: boolean | 'new' | 'old';
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
      * or browser properties before navigation. The function receives the `crawlingContext`; the options object
@@ -37,7 +38,7 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
      * ]
      * ```
      */
-    preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];
+    preNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
@@ -54,7 +55,7 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
      * ]
      * ```
      */
-    postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext, ContextExtension>[];
+    postNavigationHooks?: BrowserHook<PuppeteerCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
 }
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -121,103 +122,127 @@ export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, E
  * ```
  * @category Crawlers
  */
-export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>> extends BrowserCrawler<Page, HTTPResponse, {
-    browserPlugins: [PuppeteerPlugin];
-}, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PuppeteerCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, HTTPResponse, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     protected static optionsShape: {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        headless: import("ow").AnyPredicate<string | boolean>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        remoteBrowser: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        blockedStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
+        browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        ignoreIframes: z.ZodDefault<z.ZodBoolean>;
+        ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
     };
+    protected static optionsSchema: z.ZodObject<{
+        headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
+        browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        ignoreIframes: z.ZodDefault<z.ZodBoolean>;
+        ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
     /**
      * All `PuppeteerCrawler` parameters are passed via an options object.
      */
-    constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
+    constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
 // @ts-ignore optional peer dependency or compatibility with es2022
     protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
         injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
@@ -237,7 +262,7 @@ export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, Exte
         closeCookieModals: () => Promise<void>;
     }>;
     private enhanceContext;
-    protected _navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
+    protected navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/puppeteer-crawler.js

@@ -1,7 +1,7 @@
-import { BrowserCrawler, RequestState, Router } from '@crawlee/browser';
-import { serviceLocator } from '@crawlee/core';
-import ow from 'ow';
-import { PuppeteerLauncher } from './puppeteer-launcher.js';
+import { assertBrowserPoolNotConfigured, BrowserCrawler, RequestState, Router } from '@crawlee/browser';
+import { parseArgument, serviceLocator } from '@crawlee/core';
+import { z } from 'zod';
+import { puppeteerBrowserPool, remotePuppeteerBrowserPool } from './puppeteer-browser-pool.js';
 import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -71,37 +71,36 @@ import { gotoExtended, puppeteerUtils } from './utils/puppeteer_utils.js';
 export class PuppeteerCrawler extends BrowserCrawler {
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
-        browserPoolOptions: ow.optional.object,
+        // Deliberately looser than the declared type: Puppeteer's own accepted string values have moved over
+        // time (`'new'`/`'old'`, now `'shell'`), and the value is forwarded to it verbatim.
+        headless: z.union([z.boolean(), z.string()]).optional(),
     };
+    static optionsSchema = z.strictObject(PuppeteerCrawler.optionsShape);
     /**
      * All `PuppeteerCrawler` parameters are passed via an options object.
      */
     constructor(options = {}) {
-        ow(options, 'PuppeteerCrawlerOptions', ow.object.exactShape(PuppeteerCrawler.optionsShape));
-        const { launchContext = {}, headless, proxyConfiguration, contextPipelineBuilder, ...browserCrawlerOptions } = options;
-        const browserPoolOptions = {
-            ...options.browserPoolOptions,
-        };
+        const parsedOptions = parseArgument(options, PuppeteerCrawler.optionsSchema, 'PuppeteerCrawlerOptions');
+        const { launchContext, headless, configuration, proxyConfiguration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
         if (launchContext.proxyUrl) {
             throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.' +
                 'Use PuppeteerCrawlerOptions.proxyConfiguration');
         }
-        // `browserPlugins` is working when it's not overridden by `launchContext`,
-        // which for crawlers it is always overridden. Hence the error to use the other option.
-        if (browserPoolOptions.browserPlugins) {
-            throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
-        }
-        if (headless != null) {
-            launchContext.launchOptions ??= {};
-            launchContext.launchOptions.headless = headless;
+        if (options.browserPool) {
+            // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
+            assertBrowserPoolNotConfigured(new.target.name, {
+                launchContext: options.launchContext,
+                headless: options.headless,
+            });
         }
-        const puppeteerLauncher = new PuppeteerLauncher(launchContext, options.configuration);
-        browserPoolOptions.browserPlugins = [puppeteerLauncher.createBrowserPlugin()];
         super({
             ...browserCrawlerOptions,
             launchContext,
+            configuration,
             proxyConfiguration,
-            browserPoolOptions,
+            browserPoolBuilder: (remoteBrowser) => remoteBrowser
+                ? remotePuppeteerBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
+                : puppeteerBrowserPool({ launchContext, headless, configuration }),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
     }
@@ -146,7 +145,7 @@ export class PuppeteerCrawler extends BrowserCrawler {
             closeCookieModals: async () => puppeteerUtils.closeCookieModals(context.page),
         };
     }
-    async _navigationHandler(crawlingContext, gotoOptions) {
+    async navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
 }

package/internals/puppeteer-launcher.d.ts

@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PuppeteerPlugin } from '@crawlee/browser-pool';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Browser } from 'puppeteer';
+import { z } from 'zod';
 /**
  * Apify extends the launch options of Puppeteer.
  * You can use any of the Puppeteer compatible
@@ -69,24 +70,30 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
     readonly configuration: Configuration;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    };
+    protected static optionsSchema: z.ZodObject<{
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-    };
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
     /**
      * All `PuppeteerLauncher` parameters are passed via an launchContext object.
      */

package/internals/puppeteer-launcher.js

@@ -1,6 +1,6 @@
-import { BrowserLauncher, Configuration } from '@crawlee/browser';
+import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
 import { PuppeteerPlugin } from '@crawlee/browser-pool';
-import ow from 'ow';
+import { z } from 'zod';
 /**
  * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
  * @ignore
@@ -9,14 +9,14 @@ export class PuppeteerLauncher extends BrowserLauncher {
     configuration;
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
-        launcher: ow.optional.object,
+        launcher: schemas.anyObject.optional(),
     };
+    static optionsSchema = z.strictObject(PuppeteerLauncher.optionsShape);
     /**
      * All `PuppeteerLauncher` parameters are passed via an launchContext object.
      */
     constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
-        ow(launchContext, 'PuppeteerLauncher', ow.object.exactShape(PuppeteerLauncher.optionsShape));
-        const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = launchContext;
+        const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = parseArgument(launchContext, PuppeteerLauncher.optionsSchema, 'PuppeteerLaunchContext');
         super({
             ...browserLauncherOptions,
             launcher,

package/internals/utils/puppeteer_request_interception.js

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'node:events';
-import { serviceLocator } from '@crawlee/browser';
-import ow from 'ow';
+import { parseArgument, schemas, serviceLocator } from '@crawlee/browser';
+const pageSchema = schemas.objectWithKeys(['goto', 'evaluate']);
 // We use weak maps here so that the content gets discarded after page gets closed.
 const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
 const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.
@@ -137,8 +137,8 @@ async function handleRequest(request, interceptRequestHandlers) {
  * @param handler Request interception handler.
  */
 export async function addInterceptRequestHandler(page, handler) {
-    ow(page, ow.object.hasKeys('goto', 'evaluate'));
-    ow(handler, ow.function);
+    parseArgument(page, pageSchema);
+    parseArgument(handler, schemas.anyFunction);
     if (!pageInterceptRequestHandlersMap.has(page)) {
         pageInterceptRequestHandlersMap.set(page, []);
     }
@@ -174,8 +174,8 @@ export async function addInterceptRequestHandler(page, handler) {
  * @param handler Request interception handler.
  */
 export async function removeInterceptRequestHandler(page, handler) {
-    ow(page, ow.object.hasKeys('goto', 'evaluate'));
-    ow(handler, ow.function);
+    parseArgument(page, pageSchema);
+    parseArgument(handler, schemas.anyFunction);
     const handlersArray = pageInterceptRequestHandlersMap.get(page).filter((item) => item !== handler);
     pageInterceptRequestHandlersMap.set(page, handlersArray);
     if (handlersArray.length === 0) {

package/internals/utils/puppeteer_utils.d.ts

@@ -20,7 +20,7 @@
 import type { Request } from '@crawlee/browser';
 import { Configuration } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { HTTPResponse, Page, ResponseForRequest } from 'puppeteer';
@@ -387,8 +387,7 @@ export interface PuppeteerContextUtils {
      * in `href` elements, but rather navigations are triggered in click handlers.
      * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
      * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -409,9 +408,9 @@ export interface PuppeteerContextUtils {
      * async requestHandler({ enqueueLinksByClickingElements }) {
      *     await enqueueLinksByClickingElements({
      *         selector: 'a.product-detail',
-     *         globs: [
-     *             'https://www.example.com/handbags/**'
-     *             'https://www.example.com/purses/**'
+     *         include: [
+     *             'https://www.example.com/handbags/**',
+     *             'https://www.example.com/purses/**',
      *         ],
      *     });
      * });

package/internals/utils/puppeteer_utils.js

@@ -20,10 +20,9 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, serviceLocator, validators } from '@crawlee/browser';
+import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, validators } from '@crawlee/browser';
 import { expandShadowRoots, sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
-import ow from 'ow';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js';
@@ -31,6 +30,37 @@ const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
 const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
+const filePathSchema = z.string();
+const injectFileOptionsSchema = z.strictObject({
+    surviveNavigations: z.boolean().optional(),
+});
+const responseUrlRulesSchema = schemas.arrayOf(z.union([z.string(), z.instanceof(RegExp)]), 'strings or RegExps');
+const gotoExtendedRequestSchema = z.looseObject({
+    url: z.url(),
+    method: z.string().optional(),
+    headers: schemas.anyObject.optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+});
+const blockRequestsOptionsSchema = z.strictObject({
+    urlPatterns: schemas.arrayOf(z.string(), 'strings').default(DEFAULT_BLOCK_REQUEST_URL_PATTERNS),
+    extraUrlPatterns: schemas.arrayOf(z.string(), 'strings').default(() => []),
+});
+const infiniteScrollOptionsSchema = z.strictObject({
+    timeoutSecs: schemas.anyNumber.default(0),
+    maxScrollHeight: schemas.anyNumber.default(0),
+    waitForSecs: schemas.anyNumber.default(4),
+    scrollDownAndUp: z.boolean().default(false),
+    buttonSelector: z.string().optional(),
+    stopScrollCallback: schemas.anyFunction.optional(),
+});
+const saveSnapshotOptionsSchema = z.strictObject({
+    key: z.string().min(1).default('SNAPSHOT'),
+    screenshotQuality: schemas.anyNumber.default(50),
+    saveScreenshot: z.boolean().default(true),
+    saveHtml: z.boolean().default(true),
+    keyValueStoreName: z.string().optional(),
+    configuration: schemas.anyObject.optional(),
+});
 const getLog = () => serviceLocator.getChildLog('Puppeteer Utils');
 /**
  * Cache contents of previously injected files to limit file system access.
@@ -48,18 +78,16 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
  * @param [options]
  */
 export async function injectFile(page, filePath, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(filePath, ow.string);
-    ow(options, ow.object.exactShape({
-        surviveNavigations: ow.optional.boolean,
-    }));
+    parseArgument(page, validators.browserPage);
+    parseArgument(filePath, filePathSchema);
+    const { surviveNavigations } = parseArgument(options, injectFileOptionsSchema);
     let contents = injectedFilesCache.get(filePath);
     if (!contents) {
         contents = await readFile(filePath, 'utf8');
         injectedFilesCache.add(filePath, contents);
     }
     const evalP = page.evaluate(contents);
-    if (options.surviveNavigations) {
+    if (surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
             .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
@@ -93,7 +121,7 @@ export async function injectFile(page, filePath, options = {}) {
  * @param [options.surviveNavigations] Opt-out option to disable the JQuery reinjection after navigation.
  */
 export async function injectJQuery(page, options) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     return injectFile(page, jqueryPath, { surviveNavigations: options?.surviveNavigations ?? true });
 }
 /**
@@ -109,7 +137,7 @@ export async function injectJQuery(page, options) {
  * @param ignoreShadowRoots
  */
 export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     if (page.frames().length > 1 && !ignoreIframes) {
         const frames = await page.$$('iframe');
         await Promise.all(frames.map(async (frame) => {
@@ -142,7 +170,8 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
         ? null
         : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
     const pageContent = html || (await page.content());
-    return cheerio.load(pageContent);
+    const { load } = await import('cheerio');
+    return load(pageContent);
 }
 /**
  * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
@@ -187,12 +216,8 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
  * @param [options]
  */
 export async function blockRequests(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        urlPatterns: ow.optional.array.ofType(ow.string),
-        extraUrlPatterns: ow.optional.array.ofType(ow.string),
-    }));
-    const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [] } = options;
+    parseArgument(page, validators.browserPage);
+    const { urlPatterns, extraUrlPatterns } = parseArgument(options, blockRequestsOptionsSchema);
     const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
     // We use CDP commands instead of request interception as the latter disables caching, which is not ideal
     await sendCDPCommand(page, 'Network.setBlockedURLs', { urls: patternsToBlock });
@@ -249,9 +274,9 @@ export const blockResources = async (page, resourceTypes = ['stylesheet', 'font'
  * @deprecated
  */
 export async function cacheResponses(page, cache, responseUrlRules) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(cache, ow.object);
-    ow(responseUrlRules, ow.array.ofType(ow.any(ow.string, ow.regExp)));
+    parseArgument(page, validators.browserPage);
+    parseArgument(cache, schemas.anyObject);
+    parseArgument(responseUrlRules, responseUrlRulesSchema);
     serviceLocator
         .getLogger()
         .deprecated('utils.puppeteer.cacheResponses() has a high impact on performance ' +
@@ -344,14 +369,9 @@ export function compileScript(scriptString, context = Object.create(null)) {
  * @param [gotoOptions] Custom options for `page.goto()`.
  */
 export async function gotoExtended(page, request, gotoOptions = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(request, ow.object.partialShape({
-        url: ow.string.url,
-        method: ow.optional.string,
-        headers: ow.optional.object,
-        payload: ow.optional.any(ow.string, ow.uint8Array),
-    }));
-    ow(gotoOptions, ow.object);
+    parseArgument(page, validators.browserPage);
+    parseArgument(request, gotoExtendedRequestSchema);
+    parseArgument(gotoOptions, schemas.anyObject);
     gotoOptions = { ...gotoOptions };
     if (gotoOptions.waitUntil === 'networkidle') {
         gotoOptions.waitUntil = 'networkidle0';
@@ -407,16 +427,8 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
  * @param [options]
  */
 export async function infiniteScroll(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        timeoutSecs: ow.optional.number,
-        maxScrollHeight: ow.optional.number,
-        waitForSecs: ow.optional.number,
-        scrollDownAndUp: ow.optional.boolean,
-        buttonSelector: ow.optional.string,
-        stopScrollCallback: ow.optional.function,
-    }));
-    const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback, } = options;
+    parseArgument(page, validators.browserPage);
+    const { timeoutSecs, maxScrollHeight, waitForSecs, scrollDownAndUp, buttonSelector, stopScrollCallback } = parseArgument(options, infiniteScrollOptionsSchema);
     let finished;
     const startTime = Date.now();
     const CHECK_INTERVAL_MILLIS = 1000;
@@ -508,16 +520,8 @@ export async function infiniteScroll(page, options = {}) {
  * @param [options]
  */
 export async function saveSnapshot(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        key: ow.optional.string.nonEmpty,
-        screenshotQuality: ow.optional.number,
-        saveScreenshot: ow.optional.boolean,
-        saveHtml: ow.optional.boolean,
-        keyValueStoreName: ow.optional.string,
-        configuration: ow.optional.object,
-    }));
-    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, configuration, } = options;
+    parseArgument(page, validators.browserPage);
+    const { key, screenshotQuality, saveScreenshot, saveHtml, keyValueStoreName, configuration } = parseArgument(options, saveSnapshotOptionsSchema);
     try {
         const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
             configuration: configuration ?? Configuration.getGlobalConfiguration(),
@@ -564,7 +568,7 @@ ${error.message}
     return idcacPlaywright;
 }
 export async function closeCookieModals(page) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     const idcac = await getIdcacPlaywright();
     if (idcac?.getInjectableScript()) {
         await page.evaluate(idcac.getInjectableScript());

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/puppeteer",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,17 +48,16 @@
     },
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
-        "@crawlee/browser": "4.0.0-beta.99",
-        "@crawlee/browser-pool": "4.0.0-beta.99",
-        "@crawlee/core": "4.0.0-beta.99",
-        "@crawlee/types": "4.0.0-beta.99",
-        "@crawlee/utils": "4.0.0-beta.99",
+        "@crawlee/browser": "4.0.0-rc.0",
+        "@crawlee/browser-pool": "4.0.0-rc.0",
+        "@crawlee/core": "4.0.0-rc.0",
+        "@crawlee/types": "4.0.0-rc.0",
+        "@crawlee/utils": "4.0.0-rc.0",
         "cheerio": "^1.0.0",
         "devtools-protocol": "*",
-        "idcac-playwright": "^0.2.0",
         "jquery": "^3.7.1",
-        "ow": "^2.0.0",
-        "tslib": "^2.8.1"
+        "tslib": "^2.8.1",
+        "zod": "^4.4.3"
     },
     "peerDependencies": {
         "idcac-playwright": "^0.2.0",
@@ -79,5 +78,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
 }