@crawlee/puppeteer 4.0.0-beta.27 → 4.0.0-beta.270
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +4 -6
- package/index.js +2 -4
- package/internals/enqueue-links/click-elements.d.ts +30 -66
- package/internals/enqueue-links/click-elements.js +59 -70
- package/internals/puppeteer-browser-pool.d.ts +44 -0
- package/internals/puppeteer-browser-pool.js +34 -0
- package/internals/puppeteer-crawler.d.ts +164 -120
- package/internals/puppeteer-crawler.js +72 -77
- package/internals/puppeteer-launcher.d.ts +31 -19
- package/internals/puppeteer-launcher.js +20 -15
- package/internals/utils/{puppeteer_request_interception.d.ts → puppeteer-request-interception.d.ts} +0 -1
- package/internals/utils/{puppeteer_request_interception.js → puppeteer-request-interception.js} +8 -8
- package/internals/utils/{puppeteer_utils.d.ts → puppeteer-utils.d.ts} +29 -79
- package/internals/utils/{puppeteer_utils.js → puppeteer-utils.js} +83 -182
- package/package.json +15 -19
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/enqueue-links/click-elements.d.ts.map +0 -1
- package/internals/enqueue-links/click-elements.js.map +0 -1
- package/internals/puppeteer-crawler.d.ts.map +0 -1
- package/internals/puppeteer-crawler.js.map +0 -1
- package/internals/puppeteer-launcher.d.ts.map +0 -1
- package/internals/puppeteer-launcher.js.map +0 -1
- package/internals/utils/puppeteer_request_interception.d.ts.map +0 -1
- package/internals/utils/puppeteer_request_interception.js.map +0 -1
- package/internals/utils/puppeteer_utils.d.ts.map +0 -1
- package/internals/utils/puppeteer_utils.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
34
34
|
|
|
35
35
|
We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
|
|
36
36
|
|
|
37
|
-
> Crawlee requires **Node.js
|
|
37
|
+
> Crawlee requires **Node.js 22.13 or higher**.
|
|
38
38
|
|
|
39
39
|
### With Crawlee CLI
|
|
40
40
|
|
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
package/index.d.ts
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
|
+
export * from './internals/puppeteer-browser-pool.js';
|
|
2
3
|
export * from './internals/puppeteer-crawler.js';
|
|
3
4
|
export * from './internals/puppeteer-launcher.js';
|
|
4
|
-
export
|
|
5
|
-
export
|
|
6
|
-
export
|
|
7
|
-
export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils.js';
|
|
8
|
-
export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
|
|
5
|
+
export type { InterceptHandler } from './internals/utils/puppeteer-request-interception.js';
|
|
6
|
+
export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
|
|
7
|
+
export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer-utils.js';
|
|
9
8
|
export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
|
|
10
|
-
//# sourceMappingURL=index.d.ts.map
|
package/index.js
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
|
+
export * from './internals/puppeteer-browser-pool.js';
|
|
2
3
|
export * from './internals/puppeteer-crawler.js';
|
|
3
4
|
export * from './internals/puppeteer-launcher.js';
|
|
4
|
-
export * as
|
|
5
|
-
export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
|
|
6
|
-
export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
|
|
7
|
-
//# sourceMappingURL=index.js.map
|
|
5
|
+
export * as puppeteerUtils from './internals/utils/puppeteer-utils.js';
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
|
|
2
2
|
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
3
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
-
import type { ClickOptions, Page
|
|
4
|
+
import type { ClickOptions, Page } from 'puppeteer';
|
|
5
5
|
export interface EnqueueLinksByClickingElementsOptions {
|
|
6
6
|
/**
|
|
7
7
|
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
|
|
8
8
|
*/
|
|
9
9
|
page: Page;
|
|
10
10
|
/**
|
|
11
|
-
* A request
|
|
11
|
+
* * A request manager to which the URLs will be enqueued.
|
|
12
12
|
*/
|
|
13
|
-
|
|
13
|
+
requestManager: IRequestManager;
|
|
14
14
|
/**
|
|
15
15
|
* A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
|
|
16
16
|
* value. This is to prevent suboptimal use of this function by using it too broadly.
|
|
@@ -25,64 +25,29 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
25
25
|
*/
|
|
26
26
|
clickOptions?: ClickOptions;
|
|
27
27
|
/**
|
|
28
|
-
* An array of
|
|
29
|
-
* containing glob pattern strings matching the URLs to be enqueued.
|
|
28
|
+
* An array of URL patterns that URLs must match to be enqueued.
|
|
30
29
|
*
|
|
31
|
-
*
|
|
32
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
|
|
33
|
-
*
|
|
34
|
-
* The matching is always case-insensitive.
|
|
35
|
-
* If you need case-sensitive matching, use `regexps` property directly.
|
|
36
|
-
*
|
|
37
|
-
* If `globs` is an empty array or `undefined`, then the function
|
|
38
|
-
* enqueues all the intercepted navigation requests produced by the page
|
|
39
|
-
* after clicking on elements matching the provided CSS selector.
|
|
40
|
-
*/
|
|
41
|
-
globs?: GlobInput[];
|
|
42
|
-
/**
|
|
43
|
-
* An array of glob pattern strings, regexp patterns or plain objects
|
|
44
|
-
* containing patterns matching URLs that will **never** be enqueued.
|
|
45
|
-
*
|
|
46
|
-
* The plain objects must include either the `glob` property or the `regexp` property.
|
|
30
|
+
* Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
|
|
47
31
|
*
|
|
48
32
|
* Glob matching is always case-insensitive.
|
|
49
|
-
* If you need case-sensitive matching,
|
|
50
|
-
*/
|
|
51
|
-
exclude?: readonly (GlobInput | RegExpInput)[];
|
|
52
|
-
/**
|
|
53
|
-
* An array of regular expressions or plain objects
|
|
54
|
-
* containing regular expressions matching the URLs to be enqueued.
|
|
55
|
-
*
|
|
56
|
-
* The plain objects must include at least the `regexp` property, which holds the regular expression.
|
|
57
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
|
|
33
|
+
* If you need case-sensitive matching, use a `RegExp`.
|
|
58
34
|
*
|
|
59
|
-
* If `
|
|
35
|
+
* If `include` is an empty array or `undefined`, then the function
|
|
60
36
|
* enqueues all the intercepted navigation requests produced by the page
|
|
61
37
|
* after clicking on elements matching the provided CSS selector.
|
|
62
38
|
*/
|
|
63
|
-
|
|
39
|
+
include?: UrlPatternInput[];
|
|
64
40
|
/**
|
|
65
|
-
*
|
|
66
|
-
* Please use `globs` or `regexps` instead.
|
|
41
|
+
* An array of URL patterns. Matching URLs will **not** be enqueued.
|
|
67
42
|
*
|
|
68
|
-
*
|
|
69
|
-
* containing {@link PseudoUrl} strings matching the URLs to be enqueued.
|
|
43
|
+
* Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
|
|
70
44
|
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
* With a pseudo-URL string, the matching is always case-insensitive.
|
|
75
|
-
* If you need case-sensitive matching, use `regexps` property directly.
|
|
76
|
-
*
|
|
77
|
-
* If `pseudoUrls` is an empty array or `undefined`, then the function
|
|
78
|
-
* enqueues all the intercepted navigation requests produced by the page
|
|
79
|
-
* after clicking on elements matching the provided CSS selector.
|
|
80
|
-
*
|
|
81
|
-
* @deprecated prefer using `globs` or `regexps` instead
|
|
45
|
+
* Glob matching is always case-insensitive.
|
|
46
|
+
* If you need case-sensitive matching, use a `RegExp`.
|
|
82
47
|
*/
|
|
83
|
-
|
|
48
|
+
exclude?: readonly UrlPatternInput[];
|
|
84
49
|
/**
|
|
85
|
-
* After
|
|
50
|
+
* After request options are filtered by `include`/`exclude` patterns,
|
|
86
51
|
* this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
|
|
87
52
|
* `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
|
|
88
53
|
* but differ in methods or payloads, or to dynamically update or create `userData`.
|
|
@@ -97,8 +62,8 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
97
62
|
* }
|
|
98
63
|
* ```
|
|
99
64
|
*
|
|
100
|
-
* Note that `transformRequestFunction` has the highest priority and can overwrite
|
|
101
|
-
*
|
|
65
|
+
* Note that `transformRequestFunction` has the highest priority and can overwrite
|
|
66
|
+
* the global `label` option.
|
|
102
67
|
*
|
|
103
68
|
* The function receives a {@link RequestOptions} object and can return either:
|
|
104
69
|
* - The modified {@link RequestOptions} object
|
|
@@ -159,8 +124,7 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
159
124
|
* in `href` elements, but rather navigations are triggered in click handlers.
|
|
160
125
|
* If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
|
|
161
126
|
*
|
|
162
|
-
* Optionally, the function allows you to filter the target links' URLs using an array of
|
|
163
|
-
* and override settings of the enqueued {@link Request} objects.
|
|
127
|
+
* Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
|
|
164
128
|
*
|
|
165
129
|
* **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
|
|
166
130
|
* such as changing the Z-index of elements being clicked and their visibility. Therefore,
|
|
@@ -178,13 +142,13 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
178
142
|
* **Example usage**
|
|
179
143
|
*
|
|
180
144
|
* ```javascript
|
|
181
|
-
* await
|
|
145
|
+
* await puppeteerUtils.enqueueLinksByClickingElements({
|
|
182
146
|
* page,
|
|
183
|
-
*
|
|
147
|
+
* requestManager,
|
|
184
148
|
* selector: 'a.product-detail',
|
|
185
|
-
*
|
|
186
|
-
* 'https://www.example.com/handbags
|
|
187
|
-
* 'https://www.example.com/purses
|
|
149
|
+
* include: [
|
|
150
|
+
* 'https://www.example.com/handbags/*',
|
|
151
|
+
* 'https://www.example.com/purses/*',
|
|
188
152
|
* ],
|
|
189
153
|
* });
|
|
190
154
|
* ```
|
|
@@ -205,22 +169,22 @@ interface ClickElementsAndInterceptNavigationRequestsOptions extends WaitForPage
|
|
|
205
169
|
* Clicks all elements of given page matching given selector.
|
|
206
170
|
* Catches and intercepts all initiated navigation requests and opened pages.
|
|
207
171
|
* Returns a list of all target URLs.
|
|
172
|
+
*
|
|
173
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
208
174
|
* @ignore
|
|
209
175
|
*/
|
|
210
176
|
export declare function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
|
|
211
|
-
/**
|
|
212
|
-
* We're only interested in pages created by the page we're currently clicking in.
|
|
213
|
-
* There will generally be a lot of other targets being created in the browser.
|
|
214
|
-
*/
|
|
215
|
-
export declare function isTargetRelevant(page: Page, target: Target): boolean;
|
|
216
177
|
/**
|
|
217
178
|
* Click all elements matching the given selector. To be able to do this using
|
|
218
179
|
* Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
|
|
219
180
|
* so we first move them to the top of the page's stacking context and then click.
|
|
220
181
|
* We do all in series to prevent elements from hiding one another. Therefore,
|
|
221
182
|
* for large element sets, this will take considerable amount of time.
|
|
183
|
+
*
|
|
184
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
222
185
|
* @ignore
|
|
223
186
|
*/
|
|
224
|
-
export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions
|
|
187
|
+
export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
|
|
188
|
+
clickCount?: number;
|
|
189
|
+
}): Promise<void>;
|
|
225
190
|
export {};
|
|
226
|
-
//# sourceMappingURL=click-elements.d.ts.map
|
|
@@ -1,10 +1,26 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
|
-
import { applyRequestTransform,
|
|
3
|
-
import
|
|
4
|
-
import
|
|
5
|
-
import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/
|
|
2
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer-request-interception.js';
|
|
6
6
|
const STARTING_Z_INDEX = 2147400000;
|
|
7
|
-
const
|
|
7
|
+
const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
|
|
8
|
+
const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
|
|
9
|
+
page: schemas.objectWithKeys(['goto', 'evaluate']),
|
|
10
|
+
requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
|
|
11
|
+
selector: z.string(),
|
|
12
|
+
userData: schemas.anyObject.optional(),
|
|
13
|
+
clickOptions: schemas.anyObject.optional(),
|
|
14
|
+
include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
15
|
+
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
16
|
+
transformRequestFunction: schemas.anyFunction.optional(),
|
|
17
|
+
waitForPageIdleSecs: schemas.anyNumber.default(1),
|
|
18
|
+
maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
|
|
19
|
+
label: z.string().optional(),
|
|
20
|
+
forefront: z.boolean().optional(),
|
|
21
|
+
skipNavigation: z.boolean().optional(),
|
|
22
|
+
onSkippedRequest: schemas.anyFunction.optional(),
|
|
23
|
+
});
|
|
8
24
|
/**
|
|
9
25
|
* The function finds elements matching a specific CSS selector in a Puppeteer page,
|
|
10
26
|
* clicks all those elements using a mouse move and a left mouse button click and intercepts
|
|
@@ -14,8 +30,7 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
|
|
|
14
30
|
* in `href` elements, but rather navigations are triggered in click handlers.
|
|
15
31
|
* If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
|
|
16
32
|
*
|
|
17
|
-
* Optionally, the function allows you to filter the target links' URLs using an array of
|
|
18
|
-
* and override settings of the enqueued {@link Request} objects.
|
|
33
|
+
* Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
|
|
19
34
|
*
|
|
20
35
|
* **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
|
|
21
36
|
* such as changing the Z-index of elements being clicked and their visibility. Therefore,
|
|
@@ -33,13 +48,13 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
|
|
|
33
48
|
* **Example usage**
|
|
34
49
|
*
|
|
35
50
|
* ```javascript
|
|
36
|
-
* await
|
|
51
|
+
* await puppeteerUtils.enqueueLinksByClickingElements({
|
|
37
52
|
* page,
|
|
38
|
-
*
|
|
53
|
+
* requestManager,
|
|
39
54
|
* selector: 'a.product-detail',
|
|
40
|
-
*
|
|
41
|
-
* 'https://www.example.com/handbags
|
|
42
|
-
* 'https://www.example.com/purses
|
|
55
|
+
* include: [
|
|
56
|
+
* 'https://www.example.com/handbags/*',
|
|
57
|
+
* 'https://www.example.com/purses/*',
|
|
43
58
|
* ],
|
|
44
59
|
* });
|
|
45
60
|
* ```
|
|
@@ -47,49 +62,13 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
|
|
|
47
62
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
48
63
|
*/
|
|
49
64
|
export async function enqueueLinksByClickingElements(options) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
|
|
53
|
-
selector: ow.string,
|
|
54
|
-
userData: ow.optional.object,
|
|
55
|
-
clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
|
|
56
|
-
pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
|
|
57
|
-
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
|
|
58
|
-
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
|
|
59
|
-
exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
|
|
60
|
-
transformRequestFunction: ow.optional.function,
|
|
61
|
-
waitForPageIdleSecs: ow.optional.number,
|
|
62
|
-
maxWaitForPageIdleSecs: ow.optional.number,
|
|
63
|
-
label: ow.optional.string,
|
|
64
|
-
forefront: ow.optional.boolean,
|
|
65
|
-
skipNavigation: ow.optional.boolean,
|
|
66
|
-
onSkippedRequest: ow.optional.function,
|
|
67
|
-
}));
|
|
68
|
-
const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
|
|
65
|
+
const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
|
|
66
|
+
const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
|
|
69
67
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
70
68
|
const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
|
|
71
|
-
const
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
for (const excl of exclude) {
|
|
75
|
-
if (typeof excl === 'string' || 'glob' in excl) {
|
|
76
|
-
urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
|
|
77
|
-
}
|
|
78
|
-
else if (excl instanceof RegExp || 'regexp' in excl) {
|
|
79
|
-
urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
if (pseudoUrls?.length) {
|
|
84
|
-
log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
|
|
85
|
-
urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
|
|
86
|
-
}
|
|
87
|
-
if (globs?.length) {
|
|
88
|
-
urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
|
|
89
|
-
}
|
|
90
|
-
if (regexps?.length) {
|
|
91
|
-
urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
|
|
92
|
-
}
|
|
69
|
+
const hasOnSkippedRequest = onSkippedRequest !== undefined;
|
|
70
|
+
const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
|
|
71
|
+
const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
|
|
93
72
|
const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
|
|
94
73
|
page,
|
|
95
74
|
selector,
|
|
@@ -97,27 +76,29 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
97
76
|
maxWaitForPageIdleMillis,
|
|
98
77
|
clickOptions,
|
|
99
78
|
});
|
|
100
|
-
const requestOptions = createRequestOptions(interceptedRequests,
|
|
79
|
+
const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
|
|
101
80
|
const skippedByFilters = [];
|
|
102
|
-
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (
|
|
81
|
+
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
|
|
103
82
|
if (onSkippedRequest && skippedByFilters.length > 0) {
|
|
104
|
-
await Promise.all(skippedByFilters.map((
|
|
83
|
+
await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
|
|
105
84
|
}
|
|
106
85
|
if (transformRequestFunction) {
|
|
107
86
|
const skippedByTransform = [];
|
|
108
|
-
filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
|
|
87
|
+
filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
|
|
109
88
|
if (onSkippedRequest && skippedByTransform.length > 0) {
|
|
110
|
-
await Promise.all(skippedByTransform.map((r) => onSkippedRequest(
|
|
89
|
+
await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
|
|
111
90
|
}
|
|
112
91
|
}
|
|
113
92
|
const requests = filteredOptions.map((opts) => new Request(opts));
|
|
114
|
-
const { addedRequests } = await
|
|
93
|
+
const { addedRequests } = await requestManager.addRequestsBatched(requests, { forefront });
|
|
115
94
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
116
95
|
}
|
|
117
96
|
/**
|
|
118
97
|
* Clicks all elements of given page matching given selector.
|
|
119
98
|
* Catches and intercepts all initiated navigation requests and opened pages.
|
|
120
99
|
* Returns a list of all target URLs.
|
|
100
|
+
*
|
|
101
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
121
102
|
* @ignore
|
|
122
103
|
*/
|
|
123
104
|
export async function clickElementsAndInterceptNavigationRequests(options) {
|
|
@@ -152,6 +133,7 @@ function createInterceptRequestHandler(page, requests) {
|
|
|
152
133
|
url,
|
|
153
134
|
headers: req.headers(),
|
|
154
135
|
method: req.method(),
|
|
136
|
+
// oxlint-disable-next-line typescript/no-deprecated -- fetchPostData() is async and adds a CDP roundtrip per request; keep the sync page-cached read
|
|
155
137
|
payload: req.postData(),
|
|
156
138
|
}));
|
|
157
139
|
if (req.redirectChain().length) {
|
|
@@ -185,7 +167,9 @@ function createTargetCreatedHandler(page, requests) {
|
|
|
185
167
|
await createdPage.close();
|
|
186
168
|
}
|
|
187
169
|
catch (err) {
|
|
188
|
-
|
|
170
|
+
getLog().debug('enqueueLinksByClickingElements: Could not close spawned page.', {
|
|
171
|
+
error: err.stack,
|
|
172
|
+
});
|
|
189
173
|
}
|
|
190
174
|
};
|
|
191
175
|
}
|
|
@@ -193,7 +177,8 @@ function createTargetCreatedHandler(page, requests) {
|
|
|
193
177
|
* We're only interested in pages created by the page we're currently clicking in.
|
|
194
178
|
* There will generally be a lot of other targets being created in the browser.
|
|
195
179
|
*/
|
|
196
|
-
|
|
180
|
+
function isTargetRelevant(page, target) {
|
|
181
|
+
// oxlint-disable-next-line typescript/no-deprecated -- the non-deprecated replacement (opener.page()) is async and would force every call site to await, including EventEmitter callbacks
|
|
197
182
|
return target.type() === 'page' && page.target() === target.opener();
|
|
198
183
|
}
|
|
199
184
|
/**
|
|
@@ -237,11 +222,17 @@ async function preventHistoryNavigation(page) {
|
|
|
237
222
|
* so we first move them to the top of the page's stacking context and then click.
|
|
238
223
|
* We do all in series to prevent elements from hiding one another. Therefore,
|
|
239
224
|
* for large element sets, this will take considerable amount of time.
|
|
225
|
+
*
|
|
226
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
240
227
|
* @ignore
|
|
241
228
|
*/
|
|
242
229
|
export async function clickElements(page, selector, clickOptions) {
|
|
230
|
+
// Puppeteer 25 removed the deprecated `clickCount` option in favor of `count`, older versions ignore `count`, so we pass both.
|
|
231
|
+
if (clickOptions?.clickCount !== undefined && clickOptions.count === undefined) {
|
|
232
|
+
clickOptions = { ...clickOptions, count: clickOptions.clickCount };
|
|
233
|
+
}
|
|
243
234
|
const elementHandles = await page.$$(selector);
|
|
244
|
-
|
|
235
|
+
getLog().debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
|
|
245
236
|
let clickedElementsCount = 0;
|
|
246
237
|
let zIndex = STARTING_Z_INDEX;
|
|
247
238
|
let shouldLogWarning = true;
|
|
@@ -254,15 +245,15 @@ export async function clickElements(page, selector, clickOptions) {
|
|
|
254
245
|
catch (err) {
|
|
255
246
|
const e = err;
|
|
256
247
|
if (shouldLogWarning && e.stack.includes('is detached from document')) {
|
|
257
|
-
|
|
248
|
+
getLog().warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
|
|
258
249
|
'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. ' +
|
|
259
250
|
'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
|
|
260
251
|
shouldLogWarning = false;
|
|
261
252
|
}
|
|
262
|
-
|
|
253
|
+
getLog().debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
|
|
263
254
|
}
|
|
264
255
|
}
|
|
265
|
-
|
|
256
|
+
getLog().debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
|
|
266
257
|
}
|
|
267
258
|
/* istanbul ignore next */
|
|
268
259
|
/**
|
|
@@ -299,7 +290,6 @@ function updateElementCssToEnableMouseClick(el, zIndex) {
|
|
|
299
290
|
async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis, }) {
|
|
300
291
|
return new Promise((resolve) => {
|
|
301
292
|
let timeout;
|
|
302
|
-
let maxTimeout;
|
|
303
293
|
const context = page.browserContext();
|
|
304
294
|
function newTabTracker(target) {
|
|
305
295
|
if (isTargetRelevant(page, target))
|
|
@@ -313,7 +303,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
|
|
|
313
303
|
}, waitForPageIdleMillis);
|
|
314
304
|
}
|
|
315
305
|
function maxTimeoutHandler() {
|
|
316
|
-
|
|
306
|
+
getLog().debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
|
|
317
307
|
'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
|
|
318
308
|
finish();
|
|
319
309
|
}
|
|
@@ -323,7 +313,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
|
|
|
323
313
|
context.off('targetcreated', newTabTracker);
|
|
324
314
|
resolve();
|
|
325
315
|
}
|
|
326
|
-
maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
|
|
316
|
+
const maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
|
|
327
317
|
activityHandler(); // We call this once manually in case there would be no requests at all.
|
|
328
318
|
page.on('request', activityHandler);
|
|
329
319
|
page.on('framenavigated', activityHandler);
|
|
@@ -347,8 +337,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
|
|
|
347
337
|
requests.add(JSON.stringify({ url }));
|
|
348
338
|
}
|
|
349
339
|
catch (err) {
|
|
350
|
-
|
|
340
|
+
getLog().debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
|
|
351
341
|
}
|
|
352
342
|
});
|
|
353
343
|
}
|
|
354
|
-
//# sourceMappingURL=click-elements.js.map
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { Configuration, CrawlerRemoteBrowserOptions } from '@crawlee/browser';
|
|
2
|
+
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
3
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
+
import type { Page } from 'puppeteer';
|
|
5
|
+
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
6
|
+
/** A {@link BrowserPool} of Puppeteer browsers, as built by {@link puppeteerBrowserPool}. */
|
|
7
|
+
export type PuppeteerBrowserPool = BrowserPool<{
|
|
8
|
+
browserPlugins: [PuppeteerPlugin];
|
|
9
|
+
}, [PuppeteerPlugin]>;
|
|
10
|
+
export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {
|
|
11
|
+
/** How to launch the browser: proxy, user data dir, whether to use full Chrome, ... */
|
|
12
|
+
launchContext?: PuppeteerLaunchContext;
|
|
13
|
+
/**
|
|
14
|
+
* Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
|
|
15
|
+
* Defaults to `true`, and can also be set via {@link Configuration}.
|
|
16
|
+
*/
|
|
17
|
+
headless?: boolean | 'new' | 'old';
|
|
18
|
+
/** Configuration to read the browser defaults from. Defaults to the global configuration. */
|
|
19
|
+
configuration?: Configuration;
|
|
20
|
+
/** Connect to a remote browser service instead of launching locally; see {@link BrowserCrawlerOptions.remoteBrowser}. */
|
|
21
|
+
remoteBrowser?: CrawlerRemoteBrowserOptions;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
|
|
25
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
26
|
+
*
|
|
27
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
28
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
29
|
+
* given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
|
|
30
|
+
*
|
|
31
|
+
* **Example usage:**
|
|
32
|
+
*
|
|
33
|
+
* ```javascript
|
|
34
|
+
* const crawler = new PuppeteerCrawler({
|
|
35
|
+
* browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
|
|
36
|
+
* requestHandler: async ({ page }) => { ... },
|
|
37
|
+
* });
|
|
38
|
+
* ```
|
|
39
|
+
*
|
|
40
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
41
|
+
*
|
|
42
|
+
* @category Browser management
|
|
43
|
+
*/
|
|
44
|
+
export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { PuppeteerLauncher } from './puppeteer-launcher.js';
|
|
2
|
+
/**
|
|
3
|
+
* Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
|
|
4
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
5
|
+
*
|
|
6
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
7
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
8
|
+
* given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
|
|
9
|
+
*
|
|
10
|
+
* **Example usage:**
|
|
11
|
+
*
|
|
12
|
+
* ```javascript
|
|
13
|
+
* const crawler = new PuppeteerCrawler({
|
|
14
|
+
* browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
|
|
15
|
+
* requestHandler: async ({ page }) => { ... },
|
|
16
|
+
* });
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
20
|
+
*
|
|
21
|
+
* @category Browser management
|
|
22
|
+
*/
|
|
23
|
+
export function puppeteerBrowserPool(options = {}) {
|
|
24
|
+
const { launchContext, headless, configuration, ...poolOptions } = options;
|
|
25
|
+
return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
|
|
26
|
+
}
|
|
27
|
+
function puppeteerLauncher(launchContext = {}, headless, configuration) {
|
|
28
|
+
return new PuppeteerLauncher(headless == null
|
|
29
|
+
? launchContext
|
|
30
|
+
: {
|
|
31
|
+
...launchContext,
|
|
32
|
+
launchOptions: { ...launchContext.launchOptions, headless: headless },
|
|
33
|
+
}, configuration);
|
|
34
|
+
}
|