@crawlee/puppeteer 4.0.0-beta.2 → 4.0.0-beta.200

package/internals/puppeteer-launcher.d.ts

@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PuppeteerPlugin } from '@crawlee/browser-pool';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Browser } from 'puppeteer';
+import { z } from 'zod';
 /**
  * Apify extends the launch options of Puppeteer.
  * You can use any of the Puppeteer compatible
@@ -66,30 +67,42 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
  * @ignore
  */
 export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
-    readonly config: Configuration;
+    readonly configuration: Configuration;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+    };
+    /** @internal */
+    protected static optionsSchema: z.ZodObject<{
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-    };
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+    }, z.core.$strict>;
     /**
      * All `PuppeteerLauncher` parameters are passed via an launchContext object.
      */
-    constructor(launchContext?: PuppeteerLaunchContext, config?: Configuration);
-    protected _getDefaultHeadlessOption(): boolean;
+    constructor(launchContext?: PuppeteerLaunchContext, configuration?: Configuration);
+    protected getDefaultHeadlessOption(): boolean;
 }
 /**
  * Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
@@ -119,9 +132,8 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
  * @param [launchContext]
  *   All `PuppeteerLauncher` parameters are passed via an launchContext object.
  *   If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Puppeteer's `Browser` instance.
  */
-export declare function launchPuppeteer(launchContext?: PuppeteerLaunchContext, config?: Configuration): Promise<Browser>;
-//# sourceMappingURL=puppeteer-launcher.d.ts.map
+export declare function launchPuppeteer(launchContext?: PuppeteerLaunchContext, configuration?: Configuration): Promise<Browser>;

package/internals/puppeteer-launcher.js

@@ -1,31 +1,36 @@
 import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PuppeteerPlugin } from '@crawlee/browser-pool';
-import ow from 'ow';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 /**
  * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
  * @ignore
  */
 export class PuppeteerLauncher extends BrowserLauncher {
-    config;
+    configuration;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
-        launcher: ow.optional.object,
+        launcher: schemas.anyObject.optional(),
     };
+    /** @internal */
+    static optionsSchema = z.strictObject(PuppeteerLauncher.optionsShape);
     /**
      * All `PuppeteerLauncher` parameters are passed via an launchContext object.
      */
-    constructor(launchContext = {}, config = Configuration.getGlobalConfig()) {
-        ow(launchContext, 'PuppeteerLauncher', ow.object.exactShape(PuppeteerLauncher.optionsShape));
-        const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = launchContext;
+    constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
+        const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = parseArgument(launchContext, PuppeteerLauncher.optionsSchema, 'PuppeteerLaunchContext');
         super({
             ...browserLauncherOptions,
             launcher,
-        }, config);
-        this.config = config;
+        }, configuration);
+        this.configuration = configuration;
         this.Plugin = PuppeteerPlugin;
     }
-    _getDefaultHeadlessOption() {
-        const headless = super._getDefaultHeadlessOption();
+    getDefaultHeadlessOption() {
+        const headless = super.getDefaultHeadlessOption();
         return headless ? 'new' : headless;
     }
 }
@@ -57,12 +62,11 @@ export class PuppeteerLauncher extends BrowserLauncher {
  * @param [launchContext]
  *   All `PuppeteerLauncher` parameters are passed via an launchContext object.
  *   If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Puppeteer's `Browser` instance.
  */
-export async function launchPuppeteer(launchContext, config = Configuration.getGlobalConfig()) {
-    const puppeteerLauncher = new PuppeteerLauncher(launchContext, config);
+export async function launchPuppeteer(launchContext, configuration = Configuration.getGlobalConfiguration()) {
+    const puppeteerLauncher = new PuppeteerLauncher(launchContext, configuration);
     return puppeteerLauncher.launch();
 }
-//# sourceMappingURL=puppeteer-launcher.js.map

package/internals/utils/puppeteer_request_interception.d.ts

@@ -58,4 +58,3 @@ export declare function addInterceptRequestHandler(page: Page, handler: Intercep
  * @param handler Request interception handler.
  */
 export declare function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
-//# sourceMappingURL=puppeteer_request_interception.d.ts.map

package/internals/utils/puppeteer_request_interception.js

@@ -1,6 +1,7 @@
 import { EventEmitter } from 'node:events';
-import ow from 'ow';
-import log from '@apify/log';
+import { serviceLocator } from '@crawlee/browser';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+const pageSchema = schemas.objectWithKeys(['goto', 'evaluate']);
 // We use weak maps here so that the content gets discarded after page gets closed.
 const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
 const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.
@@ -137,8 +138,8 @@ async function handleRequest(request, interceptRequestHandlers) {
  * @param handler Request interception handler.
  */
 export async function addInterceptRequestHandler(page, handler) {
-    ow(page, ow.object.hasKeys('goto', 'evaluate'));
-    ow(handler, ow.function);
+    parseArgument(page, pageSchema);
+    parseArgument(handler, schemas.anyFunction);
     if (!pageInterceptRequestHandlersMap.has(page)) {
         pageInterceptRequestHandlersMap.set(page, []);
     }
@@ -174,8 +175,8 @@ export async function addInterceptRequestHandler(page, handler) {
  * @param handler Request interception handler.
  */
 export async function removeInterceptRequestHandler(page, handler) {
-    ow(page, ow.object.hasKeys('goto', 'evaluate'));
-    ow(handler, ow.function);
+    parseArgument(page, pageSchema);
+    parseArgument(handler, schemas.anyFunction);
     const handlersArray = pageInterceptRequestHandlersMap.get(page).filter((item) => item !== handler);
     pageInterceptRequestHandlersMap.set(page, handlersArray);
     if (handlersArray.length === 0) {
@@ -194,7 +195,7 @@ export async function removeInterceptRequestHandler(page, handler) {
                         interceptedRequestsInProgress.removeListener('delete', onDelete);
                     }
                     catch (error) {
-                        log.debug('Error while disabling request interception', { error });
+                        serviceLocator.getLogger().debug('Error while disabling request interception', { error });
                     }
                 }
             };
@@ -207,4 +208,3 @@ async function disableRequestInterception(page) {
     const requestHandler = pageInterceptRequestMasterHandlerMap.get(page);
     page.off('request', requestHandler);
 }
-//# sourceMappingURL=puppeteer_request_interception.js.map

package/internals/utils/puppeteer_utils.d.ts

@@ -5,7 +5,7 @@
  * **Example usage:**
  *
  * ```javascript
- * import { launchPuppeteer, utils } from 'crawlee';
+ * import { launchPuppeteer, puppeteerUtils } from 'crawlee';
  *
  * // Open https://www.example.com in Puppeteer
  * const browser = await launchPuppeteer();
@@ -13,20 +13,18 @@
  * await page.goto('https://www.example.com');
  *
  * // Inject jQuery into a page
- * await utils.puppeteer.injectJQuery(page);
+ * await puppeteerUtils.injectJQuery(page);
  * ```
  * @module puppeteerUtils
  */
 import type { Request } from '@crawlee/browser';
 import { Configuration } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
-import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
+import type { CheerioAPI } from 'cheerio';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { HTTPResponse, Page, ResponseForRequest } from 'puppeteer';
+import type { HTTPResponse, Page } from 'puppeteer';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import type { PuppeteerCrawlerOptions, PuppeteerCrawlingContext } from '../puppeteer-crawler.js';
 import type { InterceptHandler } from './puppeteer_request_interception.js';
 import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js';
 export interface DirectNavigationOptions {
@@ -102,7 +100,7 @@ export declare function injectFile(page: Page, filePath: string, options?: Injec
  *
  * **Example usage:**
  * ```javascript
- * await utils.puppeteer.injectJQuery(page);
+ * await puppeteerUtils.injectJQuery(page);
  * const title = await page.evaluate(() => {
  *   return $('head title').text();
  * });
@@ -123,14 +121,14 @@ export declare function injectJQuery(page: Page, options?: {
  *
  * **Example usage:**
  * ```javascript
- * const $ = await utils.puppeteer.parseWithCheerio(page);
+ * const $ = await puppeteerUtils.parseWithCheerio(page);
  * const title = $('title').text();
  * ```
  *
  * @param page Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
  * @param ignoreShadowRoots
  */
-export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
+export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
 /**
  * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
  * This is useful to speed up crawling of websites, since it reduces the amount
@@ -157,13 +155,13 @@ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean
  *
  * **Example usage**
  * ```javascript
- * import { launchPuppeteer, utils } from 'crawlee';
+ * import { launchPuppeteer, puppeteerUtils } from 'crawlee';
  *
  * const browser = await launchPuppeteer();
  * const page = await browser.newPage();
  *
  * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
- * await utils.puppeteer.blockRequests(page, {
+ * await puppeteerUtils.blockRequests(page, {
  *     extraUrlPatterns: ['adsbygoogle.js'],
  * });
  *
@@ -174,33 +172,6 @@ export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean
  * @param [options]
  */
 export declare function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
-/**
- * @internal
- */
-export declare function sendCDPCommand<T extends keyof ProtocolMapping.Commands>(page: Page, command: T, ...args: ProtocolMapping.Commands[T]['paramsType']): Promise<ProtocolMapping.Commands[T]['returnType']>;
-/**
- * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
- * Until this resolves, please use `utils.puppeteer.blockRequests()`.
- * @deprecated
- */
-export declare const blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
-/**
- * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
- * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
- *
- * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
- * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
- *   This issue should be resolved or atleast mitigated in future iterations of this feature.
- * @param page
- *   Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page) object.
- * @param cache
- *   Object in which responses are stored
- * @param responseUrlRules
- *   List of rules that are used to check if the response should be cached.
- *   String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
- * @deprecated
- */
-export declare function cacheResponses(page: Page, cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
 /**
  * Compiles a Puppeteer script into an async function that may be executed at any time
  * by providing it with the following object:
@@ -306,9 +277,9 @@ export interface SaveSnapshotOptions {
     keyValueStoreName?: string | null;
     /**
      * Configuration of the crawler that will be used to save the snapshot.
-     * @default Configuration.getGlobalConfig()
+     * @default Configuration.getGlobalConfiguration()
      */
-    config?: Configuration;
+    configuration?: Configuration;
 }
 /**
  * Saves a full screenshot and HTML of the current page into a Key-Value store.
@@ -316,8 +287,6 @@ export interface SaveSnapshotOptions {
  * @param [options]
  */
 export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
-export declare function closeCookieModals(page: Page): Promise<void>;
-/** @internal */
 export interface PuppeteerContextUtils {
     /**
      * Injects a JavaScript file into current `page`.
@@ -379,7 +348,7 @@ export interface PuppeteerContextUtils {
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
     /**
      * The function finds elements matching a specific CSS selector in a Puppeteer page,
      * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -389,8 +358,7 @@ export interface PuppeteerContextUtils {
      * in `href` elements, but rather navigations are triggered in click handlers.
      * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
      * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -411,9 +379,9 @@ export interface PuppeteerContextUtils {
      * async requestHandler({ enqueueLinksByClickingElements }) {
      *     await enqueueLinksByClickingElements({
      *         selector: 'a.product-detail',
-     *         globs: [
-     *             'https://www.example.com/handbags/**'
-     *             'https://www.example.com/purses/**'
+     *         include: [
+     *             'https://www.example.com/handbags/**',
+     *             'https://www.example.com/purses/**',
      *         ],
      *     });
      * });
@@ -421,7 +389,7 @@ export interface PuppeteerContextUtils {
      *
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
     /**
      * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
      * This is useful to speed up crawling of websites, since it reduces the amount
@@ -459,27 +427,6 @@ export interface PuppeteerContextUtils {
      * ```
      */
     blockRequests(options?: BlockRequestsOptions): Promise<void>;
-    /**
-     * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
-     * Until this resolves, please use `utils.puppeteer.blockRequests()`.
-     * @deprecated
-     */
-    blockResources(resourceTypes?: string[]): Promise<void>;
-    /**
-     * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
-     * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
-     *
-     * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
-     * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
-     *   This issue should be resolved or atleast mitigated in future iterations of this feature.
-     * @param cache
-     *   Object in which responses are stored
-     * @param responseUrlRules
-     *   List of rules that are used to check if the response should be cached.
-     *   String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
-     * @deprecated
-     */
-    cacheResponses(cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
     /**
      * Compiles a Puppeteer script into an async function that may be executed at any time
      * by providing it with the following object:
@@ -572,29 +519,5 @@ export interface PuppeteerContextUtils {
      * Saves a full screenshot and HTML of the current page into a Key-Value store.
      */
     saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
-    /**
-     * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
-     */
-    closeCookieModals(): Promise<void>;
 }
-/** @internal */
-export declare function registerUtilsToContext(context: PuppeteerCrawlingContext, crawlerOptions: PuppeteerCrawlerOptions): void;
 export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
-/** @internal */
-export declare const puppeteerUtils: {
-    injectFile: typeof injectFile;
-    injectJQuery: typeof injectJQuery;
-    enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
-    blockRequests: typeof blockRequests;
-    blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
-    cacheResponses: typeof cacheResponses;
-    compileScript: typeof compileScript;
-    gotoExtended: typeof gotoExtended;
-    addInterceptRequestHandler: typeof addInterceptRequestHandler;
-    removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
-    infiniteScroll: typeof infiniteScroll;
-    saveSnapshot: typeof saveSnapshot;
-    parseWithCheerio: typeof parseWithCheerio;
-    closeCookieModals: typeof closeCookieModals;
-};
-//# sourceMappingURL=puppeteer_utils.d.ts.map