@crawlee/puppeteer 4.0.0-beta.28 → 4.0.0-beta.280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +4 -6
  3. package/index.js +2 -4
  4. package/internals/enqueue-links/click-elements.d.ts +30 -66
  5. package/internals/enqueue-links/click-elements.js +59 -70
  6. package/internals/puppeteer-browser-pool.d.ts +44 -0
  7. package/internals/puppeteer-browser-pool.js +34 -0
  8. package/internals/puppeteer-crawler.d.ts +163 -124
  9. package/internals/puppeteer-crawler.js +68 -72
  10. package/internals/puppeteer-launcher.d.ts +31 -19
  11. package/internals/puppeteer-launcher.js +20 -15
  12. package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts} +0 -1
  13. package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js} +12 -9
  14. package/internals/utils/{puppeteer_utils.d.ts → puppeteer-utils.d.ts} +29 -79
  15. package/internals/utils/{puppeteer_utils.js → puppeteer-utils.js} +83 -182
  16. package/package.json +15 -19
  17. package/index.d.ts.map +0 -1
  18. package/index.js.map +0 -1
  19. package/internals/enqueue-links/click-elements.d.ts.map +0 -1
  20. package/internals/enqueue-links/click-elements.js.map +0 -1
  21. package/internals/puppeteer-crawler.d.ts.map +0 -1
  22. package/internals/puppeteer-crawler.js.map +0 -1
  23. package/internals/puppeteer-launcher.d.ts.map +0 -1
  24. package/internals/puppeteer-launcher.js.map +0 -1
  25. package/internals/utils/puppeteer_request_interception.d.ts.map +0 -1
  26. package/internals/utils/puppeteer_request_interception.js.map +0 -1
  27. package/internals/utils/puppeteer_utils.d.ts.map +0 -1
  28. package/internals/utils/puppeteer_utils.js.map +0 -1
package/README.md CHANGED
@@ -1,23 +1,23 @@
1
1
  <h1 align="center">
2
2
  <a href="https://crawlee.dev">
3
3
  <picture>
4
- <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
5
- <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
5
+ <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
6
6
  </picture>
7
7
  </a>
8
- <br>
8
+ <br />
9
9
  <small>A web scraping and browser automation library</small>
10
10
  </h1>
11
11
 
12
- <p align=center>
13
- <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
12
+ <p align="center">
13
+ <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
14
14
  </p>
15
15
 
16
- <p align=center>
17
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
18
- <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
19
- <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
20
- <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
16
+ <p align="center">
17
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
18
+ <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
19
+ <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
20
+ <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
21
21
  </p>
22
22
 
23
23
  Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
89
89
  We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
90
90
 
91
91
  ```bash
92
- npm install crawlee@3.12.3-beta.13
92
+ npm install crawlee@next
93
93
  ```
94
94
 
95
95
  If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
98
98
  {
99
99
  "overrides": {
100
100
  "apify": {
101
- "@crawlee/core": "3.12.3-beta.13",
102
- "@crawlee/types": "3.12.3-beta.13",
103
- "@crawlee/utils": "3.12.3-beta.13"
101
+ "@crawlee/core": "$crawlee",
102
+ "@crawlee/types": "$crawlee",
103
+ "@crawlee/utils": "$crawlee"
104
104
  }
105
105
  }
106
106
  }
package/index.d.ts CHANGED
@@ -1,10 +1,8 @@
1
1
  export * from '@crawlee/browser';
2
+ export * from './internals/puppeteer-browser-pool.js';
2
3
  export * from './internals/puppeteer-crawler.js';
3
4
  export * from './internals/puppeteer-launcher.js';
4
- export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
5
- export type { InterceptHandler } from './internals/utils/puppeteer_request_interception.js';
6
- export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
7
- export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils.js';
8
- export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
5
+ export type { InterceptHandler } from './internals/utils/puppeteer-request-interception.js';
6
+ export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
7
+ export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer-utils.js';
9
8
  export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
10
- //# sourceMappingURL=index.d.ts.map
package/index.js CHANGED
@@ -1,7 +1,5 @@
1
1
  export * from '@crawlee/browser';
2
+ export * from './internals/puppeteer-browser-pool.js';
2
3
  export * from './internals/puppeteer-crawler.js';
3
4
  export * from './internals/puppeteer-launcher.js';
4
- export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
5
- export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
6
- export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
7
- //# sourceMappingURL=index.js.map
5
+ export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
@@ -1,16 +1,16 @@
1
- import type { GlobInput, PseudoUrlInput, RegExpInput, RequestProvider, RequestTransform, SkippedRequestCallback } from '@crawlee/browser';
1
+ import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
2
2
  import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
3
3
  // @ts-ignore optional peer dependency or compatibility with es2022
4
- import type { ClickOptions, Page, Target } from 'puppeteer';
4
+ import type { ClickOptions, Page } from 'puppeteer';
5
5
  export interface EnqueueLinksByClickingElementsOptions {
6
6
  /**
7
7
  * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
8
8
  */
9
9
  page: Page;
10
10
  /**
11
- * A request queue to which the URLs will be enqueued.
11
+ * * A request manager to which the URLs will be enqueued.
12
12
  */
13
- requestQueue: RequestProvider;
13
+ requestManager: IRequestManager;
14
14
  /**
15
15
  * A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
16
16
  * value. This is to prevent suboptimal use of this function by using it too broadly.
@@ -25,64 +25,29 @@ export interface EnqueueLinksByClickingElementsOptions {
25
25
  */
26
26
  clickOptions?: ClickOptions;
27
27
  /**
28
- * An array of glob pattern strings or plain objects
29
- * containing glob pattern strings matching the URLs to be enqueued.
28
+ * An array of URL patterns that URLs must match to be enqueued.
30
29
  *
31
- * The plain objects must include at least the `glob` property, which holds the glob pattern string.
32
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
33
- *
34
- * The matching is always case-insensitive.
35
- * If you need case-sensitive matching, use `regexps` property directly.
36
- *
37
- * If `globs` is an empty array or `undefined`, then the function
38
- * enqueues all the intercepted navigation requests produced by the page
39
- * after clicking on elements matching the provided CSS selector.
40
- */
41
- globs?: GlobInput[];
42
- /**
43
- * An array of glob pattern strings, regexp patterns or plain objects
44
- * containing patterns matching URLs that will **never** be enqueued.
45
- *
46
- * The plain objects must include either the `glob` property or the `regexp` property.
30
+ * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
47
31
  *
48
32
  * Glob matching is always case-insensitive.
49
- * If you need case-sensitive matching, provide a regexp.
50
- */
51
- exclude?: readonly (GlobInput | RegExpInput)[];
52
- /**
53
- * An array of regular expressions or plain objects
54
- * containing regular expressions matching the URLs to be enqueued.
55
- *
56
- * The plain objects must include at least the `regexp` property, which holds the regular expression.
57
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
33
+ * If you need case-sensitive matching, use a `RegExp`.
58
34
  *
59
- * If `regexps` is an empty array or `undefined`, then the function
35
+ * If `include` is an empty array or `undefined`, then the function
60
36
  * enqueues all the intercepted navigation requests produced by the page
61
37
  * after clicking on elements matching the provided CSS selector.
62
38
  */
63
- regexps?: RegExpInput[];
39
+ include?: UrlPatternInput[];
64
40
  /**
65
- * *NOTE:* In future versions of SDK the options will be removed.
66
- * Please use `globs` or `regexps` instead.
41
+ * An array of URL patterns. Matching URLs will **not** be enqueued.
67
42
  *
68
- * An array of {@link PseudoUrl} strings or plain objects
69
- * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
43
+ * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
70
44
  *
71
- * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
72
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
73
- *
74
- * With a pseudo-URL string, the matching is always case-insensitive.
75
- * If you need case-sensitive matching, use `regexps` property directly.
76
- *
77
- * If `pseudoUrls` is an empty array or `undefined`, then the function
78
- * enqueues all the intercepted navigation requests produced by the page
79
- * after clicking on elements matching the provided CSS selector.
80
- *
81
- * @deprecated prefer using `globs` or `regexps` instead
45
+ * Glob matching is always case-insensitive.
46
+ * If you need case-sensitive matching, use a `RegExp`.
82
47
  */
83
- pseudoUrls?: PseudoUrlInput[];
48
+ exclude?: readonly UrlPatternInput[];
84
49
  /**
85
- * After {@link Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
50
+ * After request options are filtered by `include`/`exclude` patterns,
86
51
  * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
87
52
  * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
88
53
  * but differ in methods or payloads, or to dynamically update or create `userData`.
@@ -97,8 +62,8 @@ export interface EnqueueLinksByClickingElementsOptions {
97
62
  * }
98
63
  * ```
99
64
  *
100
- * Note that `transformRequestFunction` has the highest priority and can overwrite request options
101
- * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
65
+ * Note that `transformRequestFunction` has the highest priority and can overwrite
66
+ * the global `label` option.
102
67
  *
103
68
  * The function receives a {@link RequestOptions} object and can return either:
104
69
  * - The modified {@link RequestOptions} object
@@ -159,8 +124,7 @@ export interface EnqueueLinksByClickingElementsOptions {
159
124
  * in `href` elements, but rather navigations are triggered in click handlers.
160
125
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
161
126
  *
162
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
163
- * and override settings of the enqueued {@link Request} objects.
127
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
164
128
  *
165
129
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
166
130
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -178,13 +142,13 @@ export interface EnqueueLinksByClickingElementsOptions {
178
142
  * **Example usage**
179
143
  *
180
144
  * ```javascript
181
- * await utils.puppeteer.enqueueLinksByClickingElements({
145
+ * await puppeteerUtils.enqueueLinksByClickingElements({
182
146
  * page,
183
- * requestQueue,
147
+ * requestManager,
184
148
  * selector: 'a.product-detail',
185
- * pseudoUrls: [
186
- * 'https://www.example.com/handbags/[.*]'
187
- * 'https://www.example.com/purses/[.*]'
149
+ * include: [
150
+ * 'https://www.example.com/handbags/*',
151
+ * 'https://www.example.com/purses/*',
188
152
  * ],
189
153
  * });
190
154
  * ```
@@ -205,22 +169,22 @@ interface ClickElementsAndInterceptNavigationRequestsOptions extends WaitForPage
205
169
  * Clicks all elements of given page matching given selector.
206
170
  * Catches and intercepts all initiated navigation requests and opened pages.
207
171
  * Returns a list of all target URLs.
172
+ *
173
+ * Not part of the public API — exported only so tests can import this module directly.
208
174
  * @ignore
209
175
  */
210
176
  export declare function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
211
- /**
212
- * We're only interested in pages created by the page we're currently clicking in.
213
- * There will generally be a lot of other targets being created in the browser.
214
- */
215
- export declare function isTargetRelevant(page: Page, target: Target): boolean;
216
177
  /**
217
178
  * Click all elements matching the given selector. To be able to do this using
218
179
  * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
219
180
  * so we first move them to the top of the page's stacking context and then click.
220
181
  * We do all in series to prevent elements from hiding one another. Therefore,
221
182
  * for large element sets, this will take considerable amount of time.
183
+ *
184
+ * Not part of the public API — exported only so tests can import this module directly.
222
185
  * @ignore
223
186
  */
224
- export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
187
+ export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
188
+ clickCount?: number;
189
+ }): Promise<void>;
225
190
  export {};
226
- //# sourceMappingURL=click-elements.d.ts.map
@@ -1,10 +1,26 @@
1
1
  import { URL } from 'node:url';
2
- import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, Request, } from '@crawlee/browser';
3
- import ow from 'ow';
4
- import log_ from '@apify/log';
5
- import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
2
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
3
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
5
+ import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer-request-interception.js';
6
6
  const STARTING_Z_INDEX = 2147400000;
7
- const log = log_.child({ prefix: 'Puppeteer Click Elements' });
7
+ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
8
+ const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
9
+ page: schemas.objectWithKeys(['goto', 'evaluate']),
10
+ requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
11
+ selector: z.string(),
12
+ userData: schemas.anyObject.optional(),
13
+ clickOptions: schemas.anyObject.optional(),
14
+ include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
15
+ exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
16
+ transformRequestFunction: schemas.anyFunction.optional(),
17
+ waitForPageIdleSecs: schemas.anyNumber.default(1),
18
+ maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
19
+ label: z.string().optional(),
20
+ forefront: z.boolean().optional(),
21
+ skipNavigation: z.boolean().optional(),
22
+ onSkippedRequest: schemas.anyFunction.optional(),
23
+ });
8
24
  /**
9
25
  * The function finds elements matching a specific CSS selector in a Puppeteer page,
10
26
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -14,8 +30,7 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
14
30
  * in `href` elements, but rather navigations are triggered in click handlers.
15
31
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
16
32
  *
17
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
18
- * and override settings of the enqueued {@link Request} objects.
33
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
19
34
  *
20
35
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
21
36
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -33,13 +48,13 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
33
48
  * **Example usage**
34
49
  *
35
50
  * ```javascript
36
- * await utils.puppeteer.enqueueLinksByClickingElements({
51
+ * await puppeteerUtils.enqueueLinksByClickingElements({
37
52
  * page,
38
- * requestQueue,
53
+ * requestManager,
39
54
  * selector: 'a.product-detail',
40
- * pseudoUrls: [
41
- * 'https://www.example.com/handbags/[.*]'
42
- * 'https://www.example.com/purses/[.*]'
55
+ * include: [
56
+ * 'https://www.example.com/handbags/*',
57
+ * 'https://www.example.com/purses/*',
43
58
  * ],
44
59
  * });
45
60
  * ```
@@ -47,49 +62,13 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
47
62
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
48
63
  */
49
64
  export async function enqueueLinksByClickingElements(options) {
50
- ow(options, ow.object.exactShape({
51
- page: ow.object.hasKeys('goto', 'evaluate'),
52
- requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
53
- selector: ow.string,
54
- userData: ow.optional.object,
55
- clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
56
- pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
57
- globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
58
- regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
59
- exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
60
- transformRequestFunction: ow.optional.function,
61
- waitForPageIdleSecs: ow.optional.number,
62
- maxWaitForPageIdleSecs: ow.optional.number,
63
- label: ow.optional.string,
64
- forefront: ow.optional.boolean,
65
- skipNavigation: ow.optional.boolean,
66
- onSkippedRequest: ow.optional.function,
67
- }));
68
- const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
65
+ const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
66
+ const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
69
67
  const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
70
68
  const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
71
- const urlExcludePatternObjects = [];
72
- const urlPatternObjects = [];
73
- if (exclude?.length) {
74
- for (const excl of exclude) {
75
- if (typeof excl === 'string' || 'glob' in excl) {
76
- urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
77
- }
78
- else if (excl instanceof RegExp || 'regexp' in excl) {
79
- urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
80
- }
81
- }
82
- }
83
- if (pseudoUrls?.length) {
84
- log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
85
- urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
86
- }
87
- if (globs?.length) {
88
- urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
89
- }
90
- if (regexps?.length) {
91
- urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
92
- }
69
+ const hasOnSkippedRequest = onSkippedRequest !== undefined;
70
+ const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
71
+ const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
93
72
  const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
94
73
  page,
95
74
  selector,
@@ -97,27 +76,29 @@ export async function enqueueLinksByClickingElements(options) {
97
76
  maxWaitForPageIdleMillis,
98
77
  clickOptions,
99
78
  });
100
- const requestOptions = createRequestOptions(interceptedRequests, options);
79
+ const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
101
80
  const skippedByFilters = [];
102
- let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
81
+ let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
103
82
  if (onSkippedRequest && skippedByFilters.length > 0) {
104
- await Promise.all(skippedByFilters.map((url) => onSkippedRequest({ url, reason: 'filters' })));
83
+ await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
105
84
  }
106
85
  if (transformRequestFunction) {
107
86
  const skippedByTransform = [];
108
- filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
87
+ filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
109
88
  if (onSkippedRequest && skippedByTransform.length > 0) {
110
- await Promise.all(skippedByTransform.map((r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
89
+ await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
111
90
  }
112
91
  }
113
92
  const requests = filteredOptions.map((opts) => new Request(opts));
114
- const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
93
+ const { addedRequests } = await requestManager.addRequestsBatched(requests, { forefront });
115
94
  return { processedRequests: addedRequests, unprocessedRequests: [] };
116
95
  }
117
96
  /**
118
97
  * Clicks all elements of given page matching given selector.
119
98
  * Catches and intercepts all initiated navigation requests and opened pages.
120
99
  * Returns a list of all target URLs.
100
+ *
101
+ * Not part of the public API — exported only so tests can import this module directly.
121
102
  * @ignore
122
103
  */
123
104
  export async function clickElementsAndInterceptNavigationRequests(options) {
@@ -152,6 +133,7 @@ function createInterceptRequestHandler(page, requests) {
152
133
  url,
153
134
  headers: req.headers(),
154
135
  method: req.method(),
136
+ // oxlint-disable-next-line typescript/no-deprecated -- fetchPostData() is async and adds a CDP roundtrip per request; keep the sync page-cached read
155
137
  payload: req.postData(),
156
138
  }));
157
139
  if (req.redirectChain().length) {
@@ -185,7 +167,9 @@ function createTargetCreatedHandler(page, requests) {
185
167
  await createdPage.close();
186
168
  }
187
169
  catch (err) {
188
- log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
170
+ getLog().debug('enqueueLinksByClickingElements: Could not close spawned page.', {
171
+ error: err.stack,
172
+ });
189
173
  }
190
174
  };
191
175
  }
@@ -193,7 +177,8 @@ function createTargetCreatedHandler(page, requests) {
193
177
  * We're only interested in pages created by the page we're currently clicking in.
194
178
  * There will generally be a lot of other targets being created in the browser.
195
179
  */
196
- export function isTargetRelevant(page, target) {
180
+ function isTargetRelevant(page, target) {
181
+ // oxlint-disable-next-line typescript/no-deprecated -- the non-deprecated replacement (opener.page()) is async and would force every call site to await, including EventEmitter callbacks
197
182
  return target.type() === 'page' && page.target() === target.opener();
198
183
  }
199
184
  /**
@@ -237,11 +222,17 @@ async function preventHistoryNavigation(page) {
237
222
  * so we first move them to the top of the page's stacking context and then click.
238
223
  * We do all in series to prevent elements from hiding one another. Therefore,
239
224
  * for large element sets, this will take considerable amount of time.
225
+ *
226
+ * Not part of the public API — exported only so tests can import this module directly.
240
227
  * @ignore
241
228
  */
242
229
  export async function clickElements(page, selector, clickOptions) {
230
+ // Puppeteer 25 removed the deprecated `clickCount` option in favor of `count`, older versions ignore `count`, so we pass both.
231
+ if (clickOptions?.clickCount !== undefined && clickOptions.count === undefined) {
232
+ clickOptions = { ...clickOptions, count: clickOptions.clickCount };
233
+ }
243
234
  const elementHandles = await page.$$(selector);
244
- log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
235
+ getLog().debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
245
236
  let clickedElementsCount = 0;
246
237
  let zIndex = STARTING_Z_INDEX;
247
238
  let shouldLogWarning = true;
@@ -254,15 +245,15 @@ export async function clickElements(page, selector, clickOptions) {
254
245
  catch (err) {
255
246
  const e = err;
256
247
  if (shouldLogWarning && e.stack.includes('is detached from document')) {
257
- log.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
248
+ getLog().warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
258
249
  'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. ' +
259
250
  'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
260
251
  shouldLogWarning = false;
261
252
  }
262
- log.debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
253
+ getLog().debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
263
254
  }
264
255
  }
265
- log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
256
+ getLog().debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
266
257
  }
267
258
  /* istanbul ignore next */
268
259
  /**
@@ -299,7 +290,6 @@ function updateElementCssToEnableMouseClick(el, zIndex) {
299
290
  async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis, }) {
300
291
  return new Promise((resolve) => {
301
292
  let timeout;
302
- let maxTimeout;
303
293
  const context = page.browserContext();
304
294
  function newTabTracker(target) {
305
295
  if (isTargetRelevant(page, target))
@@ -313,7 +303,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
313
303
  }, waitForPageIdleMillis);
314
304
  }
315
305
  function maxTimeoutHandler() {
316
- log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
306
+ getLog().debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
317
307
  'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
318
308
  finish();
319
309
  }
@@ -323,7 +313,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
323
313
  context.off('targetcreated', newTabTracker);
324
314
  resolve();
325
315
  }
326
- maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
316
+ const maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
327
317
  activityHandler(); // We call this once manually in case there would be no requests at all.
328
318
  page.on('request', activityHandler);
329
319
  page.on('framenavigated', activityHandler);
@@ -347,8 +337,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
347
337
  requests.add(JSON.stringify({ url }));
348
338
  }
349
339
  catch (err) {
350
- log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
340
+ getLog().debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
351
341
  }
352
342
  });
353
343
  }
354
- //# sourceMappingURL=click-elements.js.map
@@ -0,0 +1,44 @@
1
+ import type { Configuration, CrawlerRemoteBrowserOptions } from '@crawlee/browser';
2
+ import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';
3
+ // @ts-ignore optional peer dependency or compatibility with es2022
4
+ import type { Page } from 'puppeteer';
5
+ import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
6
+ /** A {@link BrowserPool} of Puppeteer browsers, as built by {@link puppeteerBrowserPool}. */
7
+ export type PuppeteerBrowserPool = BrowserPool<{
8
+ browserPlugins: [PuppeteerPlugin];
9
+ }, [PuppeteerPlugin]>;
10
+ export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {
11
+ /** How to launch the browser: proxy, user data dir, whether to use full Chrome, ... */
12
+ launchContext?: PuppeteerLaunchContext;
13
+ /**
14
+ * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
15
+ * Defaults to `true`, and can also be set via {@link Configuration}.
16
+ */
17
+ headless?: boolean | 'new' | 'old';
18
+ /** Configuration to read the browser defaults from. Defaults to the global configuration. */
19
+ configuration?: Configuration;
20
+ /** Connect to a remote browser service instead of launching locally; see {@link BrowserCrawlerOptions.remoteBrowser}. */
21
+ remoteBrowser?: CrawlerRemoteBrowserOptions;
22
+ }
23
+ /**
24
+ * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
25
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
26
+ *
27
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
28
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
29
+ * given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
30
+ *
31
+ * **Example usage:**
32
+ *
33
+ * ```javascript
34
+ * const crawler = new PuppeteerCrawler({
35
+ * browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
36
+ * requestHandler: async ({ page }) => { ... },
37
+ * });
38
+ * ```
39
+ *
40
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
41
+ *
42
+ * @category Browser management
43
+ */
44
+ export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
@@ -0,0 +1,34 @@
1
+ import { PuppeteerLauncher } from './puppeteer-launcher.js';
2
+ /**
3
+ * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
4
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
5
+ *
6
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
7
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
8
+ * given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
9
+ *
10
+ * **Example usage:**
11
+ *
12
+ * ```javascript
13
+ * const crawler = new PuppeteerCrawler({
14
+ * browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
15
+ * requestHandler: async ({ page }) => { ... },
16
+ * });
17
+ * ```
18
+ *
19
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
20
+ *
21
+ * @category Browser management
22
+ */
23
+ export function puppeteerBrowserPool(options = {}) {
24
+ const { launchContext, headless, configuration, ...poolOptions } = options;
25
+ return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
26
+ }
27
+ function puppeteerLauncher(launchContext = {}, headless, configuration) {
28
+ return new PuppeteerLauncher(headless == null
29
+ ? launchContext
30
+ : {
31
+ ...launchContext,
32
+ launchOptions: { ...launchContext.launchOptions, headless: headless },
33
+ }, configuration);
34
+ }