@crawlee/puppeteer 4.0.0-beta.2 → 4.0.0-beta.200

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/index.d.ts

@@ -1,10 +1,8 @@
 export * from '@crawlee/browser';
+export * from './internals/puppeteer-browser-pool.js';
 export * from './internals/puppeteer-crawler.js';
 export * from './internals/puppeteer-launcher.js';
-export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
 export type { InterceptHandler } from './internals/utils/puppeteer_request_interception.js';
 export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
-export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils.js';
-export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
+export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer_utils.js';
 export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,7 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/puppeteer-browser-pool.js';
 export * from './internals/puppeteer-crawler.js';
 export * from './internals/puppeteer-launcher.js';
-export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
 export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
-export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
-//# sourceMappingURL=index.js.map

package/internals/enqueue-links/click-elements.d.ts

@@ -1,16 +1,16 @@
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestProvider, RequestTransform } from '@crawlee/browser';
+import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { ClickOptions, Page, Target } from 'puppeteer';
+import type { ClickOptions, Page } from 'puppeteer';
 export interface EnqueueLinksByClickingElementsOptions {
     /**
      * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
      */
     page: Page;
     /**
-     * A request queue to which the URLs will be enqueued.
+     * * A request manager to which the URLs will be enqueued.
      */
-    requestQueue: RequestProvider;
+    requestManager: IRequestManager;
     /**
      * A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
      * value. This is to prevent suboptimal use of this function by using it too broadly.
@@ -25,72 +25,50 @@ export interface EnqueueLinksByClickingElementsOptions {
      */
     clickOptions?: ClickOptions;
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
+     * An array of URL patterns that URLs must match to be enqueued.
      *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * If `globs` is an empty array or `undefined`, then the function
+     * If `include` is an empty array or `undefined`, then the function
      * enqueues all the intercepted navigation requests produced by the page
      * after clicking on elements matching the provided CSS selector.
      */
-    globs?: GlobInput[];
+    include?: UrlPatternInput[];
     /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
+     * An array of URL patterns. Matching URLs will **not** be enqueued.
      *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * If `regexps` is an empty array or `undefined`, then the function
-     * enqueues all the intercepted navigation requests produced by the page
-     * after clicking on elements matching the provided CSS selector.
-     */
-    regexps?: RegExpInput[];
-    /**
-     * *NOTE:* In future versions of SDK the options will be removed.
-     * Please use `globs` or `regexps` instead.
-     *
-     * An array of {@link PseudoUrl} strings or plain objects
-     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * With a pseudo-URL string, the matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `pseudoUrls` is an empty array or `undefined`, then the function
-     * enqueues all the intercepted navigation requests produced by the page
-     * after clicking on elements matching the provided CSS selector.
-     *
-     * @deprecated prefer using `globs` or `regexps` instead
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    pseudoUrls?: PseudoUrlInput[];
+    exclude?: readonly UrlPatternInput[];
     /**
-     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
-     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
-     * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
-     * or to dynamically update or create `userData`.
-     *
-     * For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
-     * a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
-     * (POST requests).
+     * After request options are filtered by `include`/`exclude` patterns,
+     * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
+     * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
+     * but differ in methods or payloads, or to dynamically update or create `userData`.
      *
      * **Example:**
      * ```javascript
      * {
      *     transformRequestFunction: (request) => {
      *         request.userData.foo = 'bar';
-     *         request.useExtendedUniqueKey = true;
      *         return request;
      *     }
      * }
      * ```
+     *
+     * Note that `transformRequestFunction` has the highest priority and can overwrite
+     * the global `label` option.
+     *
+     * The function receives a {@link RequestOptions} object and can return either:
+     * - The modified {@link RequestOptions} object
+     * - `'unchanged'` to keep the original options as-is
+     * - A falsy value or `'skip'` to exclude the request from the queue
      */
     transformRequestFunction?: RequestTransform;
     /**
@@ -130,6 +108,12 @@ export interface EnqueueLinksByClickingElementsOptions {
      * @default false
      */
     skipNavigation?: boolean;
+    /**
+     * When a request is skipped for some reason, you can use this callback to act on it.
+     * This is fired for requests skipped because they don't match enqueueLinks filters
+     * or because they were removed by `transformRequestFunction`.
+     */
+    onSkippedRequest?: SkippedRequestCallback;
 }
 /**
  * The function finds elements matching a specific CSS selector in a Puppeteer page,
@@ -140,8 +124,7 @@ export interface EnqueueLinksByClickingElementsOptions {
  * in `href` elements, but rather navigations are triggered in click handlers.
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
  *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
  *
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -159,13 +142,13 @@ export interface EnqueueLinksByClickingElementsOptions {
  * **Example usage**
  *
  * ```javascript
- * await utils.puppeteer.enqueueLinksByClickingElements({
+ * await puppeteerUtils.enqueueLinksByClickingElements({
  *   page,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
+ *   include: [
+ *       'https://www.example.com/handbags/*',
+ *       'https://www.example.com/purses/*',
  *   ],
  * });
  * ```
@@ -186,22 +169,22 @@ interface ClickElementsAndInterceptNavigationRequestsOptions extends WaitForPage
  * Clicks all elements of given page matching given selector.
  * Catches and intercepts all initiated navigation requests and opened pages.
  * Returns a list of all target URLs.
+ *
+ * Not part of the public API — exported only so tests can import this module directly.
  * @ignore
  */
 export declare function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
-/**
- * We're only interested in pages created by the page we're currently clicking in.
- * There will generally be a lot of other targets being created in the browser.
- */
-export declare function isTargetRelevant(page: Page, target: Target): boolean;
 /**
  * Click all elements matching the given selector. To be able to do this using
  * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
  * so we first move them to the top of the page's stacking context and then click.
  * We do all in series to prevent elements from hiding one another. Therefore,
  * for large element sets, this will take considerable amount of time.
+ *
+ * Not part of the public API — exported only so tests can import this module directly.
  * @ignore
  */
-export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
+export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
+    clickCount?: number;
+}): Promise<void>;
 export {};
-//# sourceMappingURL=click-elements.d.ts.map

package/internals/enqueue-links/click-elements.js

@@ -1,10 +1,26 @@
 import { URL } from 'node:url';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, } from '@crawlee/browser';
-import ow from 'ow';
-import log_ from '@apify/log';
+import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
 const STARTING_Z_INDEX = 2147400000;
-const log = log_.child({ prefix: 'Puppeteer Click Elements' });
+const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
+const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
+    page: schemas.objectWithKeys(['goto', 'evaluate']),
+    requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
+    selector: z.string(),
+    userData: schemas.anyObject.optional(),
+    clickOptions: schemas.anyObject.optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    transformRequestFunction: schemas.anyFunction.optional(),
+    waitForPageIdleSecs: schemas.anyNumber.default(1),
+    maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
+    label: z.string().optional(),
+    forefront: z.boolean().optional(),
+    skipNavigation: z.boolean().optional(),
+    onSkippedRequest: schemas.anyFunction.optional(),
+});
 /**
  * The function finds elements matching a specific CSS selector in a Puppeteer page,
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -14,8 +30,7 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
  * in `href` elements, but rather navigations are triggered in click handlers.
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
  *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
  *
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -33,13 +48,13 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
  * **Example usage**
  *
  * ```javascript
- * await utils.puppeteer.enqueueLinksByClickingElements({
+ * await puppeteerUtils.enqueueLinksByClickingElements({
  *   page,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
+ *   include: [
+ *       'https://www.example.com/handbags/*',
+ *       'https://www.example.com/purses/*',
  *   ],
  * });
  * ```
@@ -47,36 +62,13 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
  */
 export async function enqueueLinksByClickingElements(options) {
-    ow(options, ow.object.exactShape({
-        page: ow.object.hasKeys('goto', 'evaluate'),
-        requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
-        selector: ow.string,
-        userData: ow.optional.object,
-        clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
-        pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
-        globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
-        regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
-        transformRequestFunction: ow.optional.function,
-        waitForPageIdleSecs: ow.optional.number,
-        maxWaitForPageIdleSecs: ow.optional.number,
-        label: ow.optional.string,
-        forefront: ow.optional.boolean,
-        skipNavigation: ow.optional.boolean,
-    }));
-    const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, } = options;
+    const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
+    const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
     const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
     const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
-    const urlPatternObjects = [];
-    if (pseudoUrls?.length) {
-        log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
-        urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
-    }
-    if (globs?.length) {
-        urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
-    }
-    if (regexps?.length) {
-        urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
-    }
+    const hasOnSkippedRequest = onSkippedRequest !== undefined;
+    const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
+    const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
     const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
         page,
         selector,
@@ -84,18 +76,29 @@ export async function enqueueLinksByClickingElements(options) {
         maxWaitForPageIdleMillis,
         clickOptions,
     });
-    let requestOptions = createRequestOptions(interceptedRequests, options);
+    const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
+    const skippedByFilters = [];
+    let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
+    if (onSkippedRequest && skippedByFilters.length > 0) {
+        await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
+    }
     if (transformRequestFunction) {
-        requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
+        const skippedByTransform = [];
+        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
+        if (onSkippedRequest && skippedByTransform.length > 0) {
+            await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
+        }
     }
-    const requests = createRequests(requestOptions, urlPatternObjects);
-    const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
+    const requests = filteredOptions.map((opts) => new Request(opts));
+    const { addedRequests } = await requestManager.addRequestsBatched(requests, { forefront });
     return { processedRequests: addedRequests, unprocessedRequests: [] };
 }
 /**
  * Clicks all elements of given page matching given selector.
  * Catches and intercepts all initiated navigation requests and opened pages.
  * Returns a list of all target URLs.
+ *
+ * Not part of the public API — exported only so tests can import this module directly.
  * @ignore
  */
 export async function clickElementsAndInterceptNavigationRequests(options) {
@@ -130,6 +133,7 @@ function createInterceptRequestHandler(page, requests) {
             url,
             headers: req.headers(),
             method: req.method(),
+            // oxlint-disable-next-line typescript/no-deprecated -- fetchPostData() is async and adds a CDP roundtrip per request; keep the sync page-cached read
             payload: req.postData(),
         }));
         if (req.redirectChain().length) {
@@ -163,7 +167,9 @@ function createTargetCreatedHandler(page, requests) {
             await createdPage.close();
         }
         catch (err) {
-            log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
+            getLog().debug('enqueueLinksByClickingElements: Could not close spawned page.', {
+                error: err.stack,
+            });
         }
     };
 }
@@ -171,7 +177,8 @@ function createTargetCreatedHandler(page, requests) {
  * We're only interested in pages created by the page we're currently clicking in.
  * There will generally be a lot of other targets being created in the browser.
  */
-export function isTargetRelevant(page, target) {
+function isTargetRelevant(page, target) {
+    // oxlint-disable-next-line typescript/no-deprecated -- the non-deprecated replacement (opener.page()) is async and would force every call site to await, including EventEmitter callbacks
     return target.type() === 'page' && page.target() === target.opener();
 }
 /**
@@ -215,11 +222,17 @@ async function preventHistoryNavigation(page) {
  * so we first move them to the top of the page's stacking context and then click.
  * We do all in series to prevent elements from hiding one another. Therefore,
  * for large element sets, this will take considerable amount of time.
+ *
+ * Not part of the public API — exported only so tests can import this module directly.
  * @ignore
  */
 export async function clickElements(page, selector, clickOptions) {
+    // Puppeteer 25 removed the deprecated `clickCount` option in favor of `count`, older versions ignore `count`, so we pass both.
+    if (clickOptions?.clickCount !== undefined && clickOptions.count === undefined) {
+        clickOptions = { ...clickOptions, count: clickOptions.clickCount };
+    }
     const elementHandles = await page.$$(selector);
-    log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
+    getLog().debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
     let clickedElementsCount = 0;
     let zIndex = STARTING_Z_INDEX;
     let shouldLogWarning = true;
@@ -232,15 +245,15 @@ export async function clickElements(page, selector, clickOptions) {
         catch (err) {
             const e = err;
             if (shouldLogWarning && e.stack.includes('is detached from document')) {
-                log.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
+                getLog().warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
                     'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. ' +
                     'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
                 shouldLogWarning = false;
             }
-            log.debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
+            getLog().debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
         }
     }
-    log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
+    getLog().debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
 }
 /* istanbul ignore next */
 /**
@@ -277,7 +290,6 @@ function updateElementCssToEnableMouseClick(el, zIndex) {
 async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis, }) {
     return new Promise((resolve) => {
         let timeout;
-        let maxTimeout;
         const context = page.browserContext();
         function newTabTracker(target) {
             if (isTargetRelevant(page, target))
@@ -291,7 +303,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
             }, waitForPageIdleMillis);
         }
         function maxTimeoutHandler() {
-            log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
+            getLog().debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
                 'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
             finish();
         }
@@ -301,7 +313,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
             context.off('targetcreated', newTabTracker);
             resolve();
         }
-        maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
+        const maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
         activityHandler(); // We call this once manually in case there would be no requests at all.
         page.on('request', activityHandler);
         page.on('framenavigated', activityHandler);
@@ -325,8 +337,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
             requests.add(JSON.stringify({ url }));
         }
         catch (err) {
-            log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
+            getLog().debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
         }
     });
 }
-//# sourceMappingURL=click-elements.js.map

package/internals/puppeteer-browser-pool.d.ts

@@ -0,0 +1,55 @@
+import type { Configuration } from '@crawlee/browser';
+import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
+// @ts-ignore optional peer dependency or compatibility with es2022
+import type { Page } from 'puppeteer';
+import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
+/** A {@link BrowserPool} of Puppeteer browsers, as built by {@link puppeteerBrowserPool}. */
+export type PuppeteerBrowserPool = BrowserPool<{
+    browserPlugins: [PuppeteerPlugin];
+}, [PuppeteerPlugin]>;
+export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {
+    /** How to launch the browser: proxy, user data dir, whether to use full Chrome, ... */
+    launchContext?: PuppeteerLaunchContext;
+    /**
+     * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
+     * Defaults to `true`, and can also be set via {@link Configuration}.
+     */
+    headless?: boolean | 'new' | 'old';
+    /** Configuration to read the browser defaults from. Defaults to the global configuration. */
+    configuration?: Configuration;
+}
+export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
+}
+/**
+ * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PuppeteerCrawler({
+ *     browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
+ *
+ * A {@link PuppeteerCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;

package/internals/puppeteer-browser-pool.js

@@ -0,0 +1,48 @@
+import { PuppeteerLauncher } from './puppeteer-launcher.js';
+/**
+ * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PuppeteerCrawler({
+ *     browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export function puppeteerBrowserPool(options = {}) {
+    const { launchContext, headless, configuration, ...poolOptions } = options;
+    return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
+}
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
+ *
+ * A {@link PuppeteerCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * @category Browser management
+ */
+export function remotePuppeteerBrowserPool(options) {
+    const { launchContext, headless, configuration, ...remoteOptions } = options;
+    return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
+}
+function puppeteerLauncher(launchContext = {}, headless, configuration) {
+    return new PuppeteerLauncher(headless == null
+        ? launchContext
+        : {
+            ...launchContext,
+            launchOptions: { ...launchContext.launchOptions, headless: headless },
+        }, configuration);
+}