@crawlee/puppeteer 4.0.0-beta.13 → 4.0.0-beta.131

package/internals/puppeteer-launcher.d.ts

@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PuppeteerPlugin } from '@crawlee/browser-pool';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Browser } from 'puppeteer';
+import { z } from 'zod';
 /**
  * Apify extends the launch options of Puppeteer.
  * You can use any of the Puppeteer compatible
@@ -66,30 +67,38 @@ export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPl
  * @ignore
  */
 export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
-    readonly config: Configuration;
+    readonly configuration: Configuration;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    };
+    protected static optionsSchema: z.ZodObject<{
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-    };
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
     /**
      * All `PuppeteerLauncher` parameters are passed via an launchContext object.
      */
-    constructor(launchContext?: PuppeteerLaunchContext, config?: Configuration);
-    protected _getDefaultHeadlessOption(): boolean;
+    constructor(launchContext?: PuppeteerLaunchContext, configuration?: Configuration);
+    protected getDefaultHeadlessOption(): boolean;
 }
 /**
  * Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
@@ -119,9 +128,8 @@ export declare class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin,
  * @param [launchContext]
  *   All `PuppeteerLauncher` parameters are passed via an launchContext object.
  *   If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Puppeteer's `Browser` instance.
  */
-export declare function launchPuppeteer(launchContext?: PuppeteerLaunchContext, config?: Configuration): Promise<Browser>;
-//# sourceMappingURL=puppeteer-launcher.d.ts.map
+export declare function launchPuppeteer(launchContext?: PuppeteerLaunchContext, configuration?: Configuration): Promise<Browser>;

package/internals/puppeteer-launcher.js

@@ -1,31 +1,31 @@
-import { BrowserLauncher, Configuration } from '@crawlee/browser';
+import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
 import { PuppeteerPlugin } from '@crawlee/browser-pool';
-import ow from 'ow';
+import { z } from 'zod';
 /**
  * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
  * @ignore
  */
 export class PuppeteerLauncher extends BrowserLauncher {
-    config;
+    configuration;
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
-        launcher: ow.optional.object,
+        launcher: schemas.anyObject.optional(),
     };
+    static optionsSchema = z.strictObject(PuppeteerLauncher.optionsShape);
     /**
      * All `PuppeteerLauncher` parameters are passed via an launchContext object.
      */
-    constructor(launchContext = {}, config = Configuration.getGlobalConfig()) {
-        ow(launchContext, 'PuppeteerLauncher', ow.object.exactShape(PuppeteerLauncher.optionsShape));
-        const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = launchContext;
+    constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
+        const { launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-puppeteer-chrome'), ...browserLauncherOptions } = parseArgument(launchContext, PuppeteerLauncher.optionsSchema, 'PuppeteerLaunchContext');
         super({
             ...browserLauncherOptions,
             launcher,
-        }, config);
-        this.config = config;
+        }, configuration);
+        this.configuration = configuration;
         this.Plugin = PuppeteerPlugin;
     }
-    _getDefaultHeadlessOption() {
-        const headless = super._getDefaultHeadlessOption();
+    getDefaultHeadlessOption() {
+        const headless = super.getDefaultHeadlessOption();
         return headless ? 'new' : headless;
     }
 }
@@ -57,12 +57,11 @@ export class PuppeteerLauncher extends BrowserLauncher {
  * @param [launchContext]
  *   All `PuppeteerLauncher` parameters are passed via an launchContext object.
  *   If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
- * @param [config]
+ * @param [configuration]
  * @returns
  *   Promise that resolves to Puppeteer's `Browser` instance.
  */
-export async function launchPuppeteer(launchContext, config = Configuration.getGlobalConfig()) {
-    const puppeteerLauncher = new PuppeteerLauncher(launchContext, config);
+export async function launchPuppeteer(launchContext, configuration = Configuration.getGlobalConfiguration()) {
+    const puppeteerLauncher = new PuppeteerLauncher(launchContext, configuration);
     return puppeteerLauncher.launch();
 }
-//# sourceMappingURL=puppeteer-launcher.js.map

package/internals/utils/puppeteer_request_interception.d.ts

@@ -58,4 +58,3 @@ export declare function addInterceptRequestHandler(page: Page, handler: Intercep
  * @param handler Request interception handler.
  */
 export declare function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
-//# sourceMappingURL=puppeteer_request_interception.d.ts.map

package/internals/utils/puppeteer_request_interception.js

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'node:events';
-import ow from 'ow';
-import log from '@apify/log';
+import { parseArgument, schemas, serviceLocator } from '@crawlee/browser';
+const pageSchema = schemas.objectWithKeys(['goto', 'evaluate']);
 // We use weak maps here so that the content gets discarded after page gets closed.
 const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
 const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.
@@ -137,8 +137,8 @@ async function handleRequest(request, interceptRequestHandlers) {
  * @param handler Request interception handler.
  */
 export async function addInterceptRequestHandler(page, handler) {
-    ow(page, ow.object.hasKeys('goto', 'evaluate'));
-    ow(handler, ow.function);
+    parseArgument(page, pageSchema);
+    parseArgument(handler, schemas.anyFunction);
     if (!pageInterceptRequestHandlersMap.has(page)) {
         pageInterceptRequestHandlersMap.set(page, []);
     }
@@ -174,8 +174,8 @@ export async function addInterceptRequestHandler(page, handler) {
  * @param handler Request interception handler.
  */
 export async function removeInterceptRequestHandler(page, handler) {
-    ow(page, ow.object.hasKeys('goto', 'evaluate'));
-    ow(handler, ow.function);
+    parseArgument(page, pageSchema);
+    parseArgument(handler, schemas.anyFunction);
     const handlersArray = pageInterceptRequestHandlersMap.get(page).filter((item) => item !== handler);
     pageInterceptRequestHandlersMap.set(page, handlersArray);
     if (handlersArray.length === 0) {
@@ -194,7 +194,7 @@ export async function removeInterceptRequestHandler(page, handler) {
                         interceptedRequestsInProgress.removeListener('delete', onDelete);
                     }
                     catch (error) {
-                        log.debug('Error while disabling request interception', { error });
+                        serviceLocator.getLogger().debug('Error while disabling request interception', { error });
                     }
                 }
             };
@@ -207,4 +207,3 @@ async function disableRequestInterception(page) {
     const requestHandler = pageInterceptRequestMasterHandlerMap.get(page);
     page.off('request', requestHandler);
 }
-//# sourceMappingURL=puppeteer_request_interception.js.map

package/internals/utils/puppeteer_utils.d.ts

@@ -20,7 +20,7 @@
 import type { Request } from '@crawlee/browser';
 import { Configuration } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { HTTPResponse, Page, ResponseForRequest } from 'puppeteer';
@@ -305,9 +305,9 @@ export interface SaveSnapshotOptions {
     keyValueStoreName?: string | null;
     /**
      * Configuration of the crawler that will be used to save the snapshot.
-     * @default Configuration.getGlobalConfig()
+     * @default Configuration.getGlobalConfiguration()
      */
-    config?: Configuration;
+    configuration?: Configuration;
 }
 /**
  * Saves a full screenshot and HTML of the current page into a Key-Value store.
@@ -316,7 +316,6 @@ export interface SaveSnapshotOptions {
  */
 export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
 export declare function closeCookieModals(page: Page): Promise<void>;
-/** @internal */
 export interface PuppeteerContextUtils {
     /**
      * Injects a JavaScript file into current `page`.
@@ -388,8 +387,7 @@ export interface PuppeteerContextUtils {
      * in `href` elements, but rather navigations are triggered in click handlers.
      * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
      * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -410,9 +408,9 @@ export interface PuppeteerContextUtils {
      * async requestHandler({ enqueueLinksByClickingElements }) {
      *     await enqueueLinksByClickingElements({
      *         selector: 'a.product-detail',
-     *         globs: [
-     *             'https://www.example.com/handbags/**'
-     *             'https://www.example.com/purses/**'
+     *         include: [
+     *             'https://www.example.com/handbags/**',
+     *             'https://www.example.com/purses/**',
      *         ],
      *     });
      * });
@@ -420,7 +418,7 @@ export interface PuppeteerContextUtils {
      *
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
     /**
      * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
      * This is useful to speed up crawling of websites, since it reduces the amount
@@ -552,6 +550,15 @@ export interface PuppeteerContextUtils {
     saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
     /**
      * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+     *
+     * Note that this method requires the idcac-playwright package to be installed.
+     * Crawlee does not include it by default due to licensing issues.
+     *
+     * To use this method, please install the package manually by running:
+     *
+     * ```bash
+     * npm install idcac-playwright
+     * ```
      */
     closeCookieModals(): Promise<void>;
 }
@@ -571,4 +578,3 @@ export declare const puppeteerUtils: {
     parseWithCheerio: typeof parseWithCheerio;
     closeCookieModals: typeof closeCookieModals;
 };
-//# sourceMappingURL=puppeteer_utils.d.ts.map

package/internals/utils/puppeteer_utils.js

@@ -20,20 +20,48 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, validators } from '@crawlee/browser';
+import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, validators } from '@crawlee/browser';
 import { expandShadowRoots, sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
-import { getInjectableScript } from 'idcac-playwright';
-import ow from 'ow';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
-import log_ from '@apify/log';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception.js';
 const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
 const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
-const log = log_.child({ prefix: 'Puppeteer Utils' });
+const filePathSchema = z.string();
+const injectFileOptionsSchema = z.strictObject({
+    surviveNavigations: z.boolean().optional(),
+});
+const responseUrlRulesSchema = schemas.arrayOf(z.union([z.string(), z.instanceof(RegExp)]), 'strings or RegExps');
+const gotoExtendedRequestSchema = z.looseObject({
+    url: z.url(),
+    method: z.string().optional(),
+    headers: schemas.anyObject.optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+});
+const blockRequestsOptionsSchema = z.strictObject({
+    urlPatterns: schemas.arrayOf(z.string(), 'strings').default(DEFAULT_BLOCK_REQUEST_URL_PATTERNS),
+    extraUrlPatterns: schemas.arrayOf(z.string(), 'strings').default(() => []),
+});
+const infiniteScrollOptionsSchema = z.strictObject({
+    timeoutSecs: schemas.anyNumber.default(0),
+    maxScrollHeight: schemas.anyNumber.default(0),
+    waitForSecs: schemas.anyNumber.default(4),
+    scrollDownAndUp: z.boolean().default(false),
+    buttonSelector: z.string().optional(),
+    stopScrollCallback: schemas.anyFunction.optional(),
+});
+const saveSnapshotOptionsSchema = z.strictObject({
+    key: z.string().min(1).default('SNAPSHOT'),
+    screenshotQuality: schemas.anyNumber.default(50),
+    saveScreenshot: z.boolean().default(true),
+    saveHtml: z.boolean().default(true),
+    keyValueStoreName: z.string().optional(),
+    configuration: schemas.anyObject.optional(),
+});
+const getLog = () => serviceLocator.getChildLog('Puppeteer Utils');
 /**
  * Cache contents of previously injected files to limit file system access.
  */
@@ -50,21 +78,19 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
  * @param [options]
  */
 export async function injectFile(page, filePath, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(filePath, ow.string);
-    ow(options, ow.object.exactShape({
-        surviveNavigations: ow.optional.boolean,
-    }));
+    parseArgument(page, validators.browserPage);
+    parseArgument(filePath, filePathSchema);
+    const { surviveNavigations } = parseArgument(options, injectFileOptionsSchema);
     let contents = injectedFilesCache.get(filePath);
     if (!contents) {
         contents = await readFile(filePath, 'utf8');
         injectedFilesCache.add(filePath, contents);
     }
     const evalP = page.evaluate(contents);
-    if (options.surviveNavigations) {
+    if (surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
-            .catch((error) => log.warning('An error occurred during the script injection!', { error })));
+            .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
     }
     return evalP;
 }
@@ -95,7 +121,7 @@ export async function injectFile(page, filePath, options = {}) {
  * @param [options.surviveNavigations] Opt-out option to disable the JQuery reinjection after navigation.
  */
 export async function injectJQuery(page, options) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     return injectFile(page, jqueryPath, { surviveNavigations: options?.surviveNavigations ?? true });
 }
 /**
@@ -111,14 +137,22 @@ export async function injectJQuery(page, options) {
  * @param ignoreShadowRoots
  */
 export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     if (page.frames().length > 1 && !ignoreIframes) {
         const frames = await page.$$('iframe');
         await Promise.all(frames.map(async (frame) => {
             try {
                 const iframe = await frame.contentFrame();
                 if (iframe) {
-                    const contents = await iframe.content();
+                    const getIframeHTML = async () => {
+                        try {
+                            return iframe.$eval('body', (el) => el.innerHTML);
+                        }
+                        catch {
+                            return iframe.content();
+                        }
+                    };
+                    const contents = await getIframeHTML();
                     await frame.evaluate((f, c) => {
                         const replacementNode = document.createElement('div');
                         replacementNode.innerHTML = c;
@@ -128,7 +162,7 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
                 }
             }
             catch (error) {
-                log.warning(`Failed to extract iframe content: ${error}`);
+                getLog().warning(`Failed to extract iframe content: ${error}`);
             }
         }));
     }
@@ -136,7 +170,8 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
         ? null
         : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
     const pageContent = html || (await page.content());
-    return cheerio.load(pageContent);
+    const { load } = await import('cheerio');
+    return load(pageContent);
 }
 /**
  * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
@@ -181,12 +216,8 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
  * @param [options]
  */
 export async function blockRequests(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        urlPatterns: ow.optional.array.ofType(ow.string),
-        extraUrlPatterns: ow.optional.array.ofType(ow.string),
-    }));
-    const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [] } = options;
+    parseArgument(page, validators.browserPage);
+    const { urlPatterns, extraUrlPatterns } = parseArgument(options, blockRequestsOptionsSchema);
     const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
     // We use CDP commands instead of request interception as the latter disables caching, which is not ideal
     await sendCDPCommand(page, 'Network.setBlockedURLs', { urls: patternsToBlock });
@@ -214,7 +245,9 @@ export async function sendCDPCommand(page, command, ...args) {
  * @deprecated
  */
 export const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'image', 'media']) => {
-    log.deprecated('utils.puppeteer.blockResources() has a high impact on performance in recent versions of Puppeteer. ' +
+    serviceLocator
+        .getLogger()
+        .deprecated('utils.puppeteer.blockResources() has a high impact on performance in recent versions of Puppeteer. ' +
         'Until this resolves, please use utils.puppeteer.blockRequests()');
     await addInterceptRequestHandler(page, async (request) => {
         const type = request.resourceType();
@@ -241,10 +274,12 @@ export const blockResources = async (page, resourceTypes = ['stylesheet', 'font'
  * @deprecated
  */
 export async function cacheResponses(page, cache, responseUrlRules) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(cache, ow.object);
-    ow(responseUrlRules, ow.array.ofType(ow.any(ow.string, ow.regExp)));
-    log.deprecated('utils.puppeteer.cacheResponses() has a high impact on performance ' +
+    parseArgument(page, validators.browserPage);
+    parseArgument(cache, schemas.anyObject);
+    parseArgument(responseUrlRules, responseUrlRulesSchema);
+    serviceLocator
+        .getLogger()
+        .deprecated('utils.puppeteer.cacheResponses() has a high impact on performance ' +
         "in recent versions of Puppeteer so it's use is discouraged until this issue resolves.");
     await addInterceptRequestHandler(page, async (request) => {
         const url = request.url();
@@ -314,7 +349,7 @@ export function compileScript(scriptString, context = Object.create(null)) {
         func = vm.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
     }
     catch (err) {
-        log.exception(err, 'Cannot compile script!');
+        getLog().exception(err, 'Cannot compile script!');
         throw err;
     }
     if (typeof func !== 'function')
@@ -334,23 +369,20 @@ export function compileScript(scriptString, context = Object.create(null)) {
  * @param [gotoOptions] Custom options for `page.goto()`.
  */
 export async function gotoExtended(page, request, gotoOptions = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(request, ow.object.partialShape({
-        url: ow.string.url,
-        method: ow.optional.string,
-        headers: ow.optional.object,
-        payload: ow.optional.any(ow.string, ow.uint8Array),
-    }));
-    ow(gotoOptions, ow.object);
+    parseArgument(page, validators.browserPage);
+    parseArgument(request, gotoExtendedRequestSchema);
+    parseArgument(gotoOptions, schemas.anyObject);
     gotoOptions = { ...gotoOptions };
     if (gotoOptions.waitUntil === 'networkidle') {
         gotoOptions.waitUntil = 'networkidle0';
     }
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
-        log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
+        serviceLocator
+            .getLogger()
+            .deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
             'in recent versions of Puppeteer. Use only when necessary.');
         let wasCalled = false;
         const interceptRequestHandler = async (interceptedRequest) => {
@@ -373,6 +405,19 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
         };
         await addInterceptRequestHandler(page, interceptRequestHandler);
     }
+    else if (!isEmpty(headers)) {
+        const extraHeaders = { ...headers };
+        // Chrome bundled with Puppeteer 25+ ignores a `User-Agent` passed via `setExtraHTTPHeaders()`, it has to be set explicitly.
+        for (const name of Object.keys(extraHeaders)) {
+            if (name.toLowerCase() === 'user-agent') {
+                await page.setUserAgent(extraHeaders[name]);
+                delete extraHeaders[name];
+            }
+        }
+        if (!isEmpty(extraHeaders)) {
+            await page.setExtraHTTPHeaders(extraHeaders);
+        }
+    }
     return page.goto(url, gotoOptions);
 }
 /**
@@ -382,16 +427,8 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
  * @param [options]
  */
 export async function infiniteScroll(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        timeoutSecs: ow.optional.number,
-        maxScrollHeight: ow.optional.number,
-        waitForSecs: ow.optional.number,
-        scrollDownAndUp: ow.optional.boolean,
-        buttonSelector: ow.optional.string,
-        stopScrollCallback: ow.optional.function,
-    }));
-    const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback, } = options;
+    parseArgument(page, validators.browserPage);
+    const { timeoutSecs, maxScrollHeight, waitForSecs, scrollDownAndUp, buttonSelector, stopScrollCallback } = parseArgument(options, infiniteScrollOptionsSchema);
     let finished;
     const startTime = Date.now();
     const CHECK_INTERVAL_MILLIS = 1000;
@@ -483,19 +520,11 @@ export async function infiniteScroll(page, options = {}) {
  * @param [options]
  */
 export async function saveSnapshot(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        key: ow.optional.string.nonEmpty,
-        screenshotQuality: ow.optional.number,
-        saveScreenshot: ow.optional.boolean,
-        saveHtml: ow.optional.boolean,
-        keyValueStoreName: ow.optional.string,
-        config: ow.optional.object,
-    }));
-    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, config, } = options;
+    parseArgument(page, validators.browserPage);
+    const { key, screenshotQuality, saveScreenshot, saveHtml, keyValueStoreName, configuration } = parseArgument(options, saveSnapshotOptionsSchema);
     try {
-        const store = await KeyValueStore.open(keyValueStoreName, {
-            config: config ?? Configuration.getGlobalConfig(),
+        const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
+            configuration: configuration ?? Configuration.getGlobalConfiguration(),
         });
         if (saveScreenshot) {
             const screenshotName = `${key}.jpg`;
@@ -516,8 +545,34 @@ export async function saveSnapshot(page, options = {}) {
         throw new Error(`saveSnapshot with key ${key} failed.\nCause:${err.message}`);
     }
 }
+let idcacPlaywright = null;
+async function getIdcacPlaywright() {
+    if (idcacPlaywright)
+        return idcacPlaywright;
+    try {
+        idcacPlaywright = await import('idcac-playwright');
+    }
+    catch (error) {
+        getLog().warning(`Failed to import 'idcac-playwright'.
+
+We recently made idcac-playwright an optional dependency due to licensing issues.
+To use this feature, please install it manually by running
+
+npm install idcac-playwright
+
+Original error message follows:
+
+${error.message}
+`);
+    }
+    return idcacPlaywright;
+}
 export async function closeCookieModals(page) {
-    await page.evaluate(getInjectableScript());
+    parseArgument(page, validators.browserPage);
+    const idcac = await getIdcacPlaywright();
+    if (idcac?.getInjectableScript()) {
+        await page.evaluate(idcac.getInjectableScript());
+    }
 }
 export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler };
 /** @internal */
@@ -535,4 +590,3 @@ export const puppeteerUtils = {
     parseWithCheerio,
     closeCookieModals,
 };
-//# sourceMappingURL=puppeteer_utils.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/puppeteer",
-    "version": "4.0.0-beta.13",
+    "version": "4.0.0-beta.131",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -38,7 +38,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -48,22 +48,25 @@
     },
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
-        "@apify/log": "^2.5.18",
-        "@crawlee/browser": "4.0.0-beta.13",
-        "@crawlee/browser-pool": "4.0.0-beta.13",
-        "@crawlee/types": "4.0.0-beta.13",
-        "@crawlee/utils": "4.0.0-beta.13",
+        "@crawlee/browser": "4.0.0-beta.131",
+        "@crawlee/browser-pool": "4.0.0-beta.131",
+        "@crawlee/core": "4.0.0-beta.131",
+        "@crawlee/types": "4.0.0-beta.131",
+        "@crawlee/utils": "4.0.0-beta.131",
         "cheerio": "^1.0.0",
         "devtools-protocol": "*",
-        "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
-        "ow": "^2.0.0",
-        "tslib": "^2.8.1"
+        "tslib": "^2.8.1",
+        "zod": "^4.4.3"
     },
     "peerDependencies": {
+        "idcac-playwright": "^0.2.0",
         "puppeteer": "*"
     },
     "peerDependenciesMeta": {
+        "idcac-playwright": {
+            "optional": true
+        },
         "puppeteer": {
             "optional": true
         }
@@ -75,5 +78,5 @@
             }
         }
     },
-    "gitHead": "24df7c8dba6b6c9623c02ccc655d8ee748f003e3"
+    "gitHead": "de741c0b63d0da061e4ffcd6ce6fdc0c92401399"
 }

package/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAElD,OAAO,KAAK,4BAA4B,MAAM,qDAAqD,CAAC;AACpG,YAAY,EAAE,gBAAgB,EAAE,MAAM,qDAAqD,CAAC;AAE5F,OAAO,KAAK,cAAc,MAAM,sCAAsC,CAAC;AACvE,YAAY,EACR,oBAAoB,EACpB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,IAAI,gCAAgC,EAC3D,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,GACtB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,KAAK,sBAAsB,MAAM,6CAA6C,CAAC;AACtF,YAAY,EAAE,qCAAqC,EAAE,MAAM,6CAA6C,CAAC"}

package/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,kCAAkC,CAAC;AACjD,cAAc,mCAAmC,CAAC;AAElD,OAAO,KAAK,4BAA4B,MAAM,qDAAqD,CAAC;AAGpG,OAAO,KAAK,cAAc,MAAM,sCAAsC,CAAC;AAWvE,OAAO,KAAK,sBAAsB,MAAM,6CAA6C,CAAC"}