@crawlee/puppeteer 4.0.0-beta.99 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +2 -3
- package/index.js +1 -2
- package/internals/enqueue-links/click-elements.d.ts +24 -61
- package/internals/enqueue-links/click-elements.js +39 -57
- package/internals/puppeteer-browser-pool.d.ts +55 -0
- package/internals/puppeteer-browser-pool.js +48 -0
- package/internals/puppeteer-crawler.d.ts +137 -126
- package/internals/puppeteer-crawler.js +31 -28
- package/internals/puppeteer-launcher.d.ts +26 -15
- package/internals/puppeteer-launcher.js +9 -4
- package/internals/utils/puppeteer_request_interception.js +6 -5
- package/internals/utils/puppeteer_utils.d.ts +14 -72
- package/internals/utils/puppeteer_utils.js +58 -174
- package/package.json +11 -16
package/README.md
CHANGED
|
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
34
34
|
|
|
35
35
|
We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
|
|
36
36
|
|
|
37
|
-
> Crawlee requires **Node.js
|
|
37
|
+
> Crawlee requires **Node.js 22.13 or higher**.
|
|
38
38
|
|
|
39
39
|
### With Crawlee CLI
|
|
40
40
|
|
package/index.d.ts
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
|
+
export * from './internals/puppeteer-browser-pool.js';
|
|
2
3
|
export * from './internals/puppeteer-crawler.js';
|
|
3
4
|
export * from './internals/puppeteer-launcher.js';
|
|
4
|
-
export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
|
|
5
5
|
export type { InterceptHandler } from './internals/utils/puppeteer_request_interception.js';
|
|
6
6
|
export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
|
|
7
|
-
export type {
|
|
8
|
-
export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
|
|
7
|
+
export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer_utils.js';
|
|
9
8
|
export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
|
package/index.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
export * from '@crawlee/browser';
|
|
2
|
+
export * from './internals/puppeteer-browser-pool.js';
|
|
2
3
|
export * from './internals/puppeteer-crawler.js';
|
|
3
4
|
export * from './internals/puppeteer-launcher.js';
|
|
4
|
-
export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
|
|
5
5
|
export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
|
|
6
|
-
export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
|
|
2
2
|
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
3
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
-
import type { ClickOptions, Page
|
|
4
|
+
import type { ClickOptions, Page } from 'puppeteer';
|
|
5
5
|
export interface EnqueueLinksByClickingElementsOptions {
|
|
6
6
|
/**
|
|
7
7
|
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
|
|
@@ -25,64 +25,29 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
25
25
|
*/
|
|
26
26
|
clickOptions?: ClickOptions;
|
|
27
27
|
/**
|
|
28
|
-
* An array of
|
|
29
|
-
* containing glob pattern strings matching the URLs to be enqueued.
|
|
28
|
+
* An array of URL patterns that URLs must match to be enqueued.
|
|
30
29
|
*
|
|
31
|
-
*
|
|
32
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
|
|
33
|
-
*
|
|
34
|
-
* The matching is always case-insensitive.
|
|
35
|
-
* If you need case-sensitive matching, use `regexps` property directly.
|
|
36
|
-
*
|
|
37
|
-
* If `globs` is an empty array or `undefined`, then the function
|
|
38
|
-
* enqueues all the intercepted navigation requests produced by the page
|
|
39
|
-
* after clicking on elements matching the provided CSS selector.
|
|
40
|
-
*/
|
|
41
|
-
globs?: GlobInput[];
|
|
42
|
-
/**
|
|
43
|
-
* An array of glob pattern strings, regexp patterns or plain objects
|
|
44
|
-
* containing patterns matching URLs that will **never** be enqueued.
|
|
45
|
-
*
|
|
46
|
-
* The plain objects must include either the `glob` property or the `regexp` property.
|
|
30
|
+
* Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
|
|
47
31
|
*
|
|
48
32
|
* Glob matching is always case-insensitive.
|
|
49
|
-
* If you need case-sensitive matching,
|
|
50
|
-
*/
|
|
51
|
-
exclude?: readonly (GlobInput | RegExpInput)[];
|
|
52
|
-
/**
|
|
53
|
-
* An array of regular expressions or plain objects
|
|
54
|
-
* containing regular expressions matching the URLs to be enqueued.
|
|
55
|
-
*
|
|
56
|
-
* The plain objects must include at least the `regexp` property, which holds the regular expression.
|
|
57
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
|
|
33
|
+
* If you need case-sensitive matching, use a `RegExp`.
|
|
58
34
|
*
|
|
59
|
-
* If `
|
|
35
|
+
* If `include` is an empty array or `undefined`, then the function
|
|
60
36
|
* enqueues all the intercepted navigation requests produced by the page
|
|
61
37
|
* after clicking on elements matching the provided CSS selector.
|
|
62
38
|
*/
|
|
63
|
-
|
|
39
|
+
include?: UrlPatternInput[];
|
|
64
40
|
/**
|
|
65
|
-
*
|
|
66
|
-
* Please use `globs` or `regexps` instead.
|
|
41
|
+
* An array of URL patterns. Matching URLs will **not** be enqueued.
|
|
67
42
|
*
|
|
68
|
-
*
|
|
69
|
-
* containing {@link PseudoUrl} strings matching the URLs to be enqueued.
|
|
43
|
+
* Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
|
|
70
44
|
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
* With a pseudo-URL string, the matching is always case-insensitive.
|
|
75
|
-
* If you need case-sensitive matching, use `regexps` property directly.
|
|
76
|
-
*
|
|
77
|
-
* If `pseudoUrls` is an empty array or `undefined`, then the function
|
|
78
|
-
* enqueues all the intercepted navigation requests produced by the page
|
|
79
|
-
* after clicking on elements matching the provided CSS selector.
|
|
80
|
-
*
|
|
81
|
-
* @deprecated prefer using `globs` or `regexps` instead
|
|
45
|
+
* Glob matching is always case-insensitive.
|
|
46
|
+
* If you need case-sensitive matching, use a `RegExp`.
|
|
82
47
|
*/
|
|
83
|
-
|
|
48
|
+
exclude?: readonly UrlPatternInput[];
|
|
84
49
|
/**
|
|
85
|
-
* After
|
|
50
|
+
* After request options are filtered by `include`/`exclude` patterns,
|
|
86
51
|
* this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
|
|
87
52
|
* `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
|
|
88
53
|
* but differ in methods or payloads, or to dynamically update or create `userData`.
|
|
@@ -97,8 +62,8 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
97
62
|
* }
|
|
98
63
|
* ```
|
|
99
64
|
*
|
|
100
|
-
* Note that `transformRequestFunction` has the highest priority and can overwrite
|
|
101
|
-
*
|
|
65
|
+
* Note that `transformRequestFunction` has the highest priority and can overwrite
|
|
66
|
+
* the global `label` option.
|
|
102
67
|
*
|
|
103
68
|
* The function receives a {@link RequestOptions} object and can return either:
|
|
104
69
|
* - The modified {@link RequestOptions} object
|
|
@@ -159,8 +124,7 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
159
124
|
* in `href` elements, but rather navigations are triggered in click handlers.
|
|
160
125
|
* If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
|
|
161
126
|
*
|
|
162
|
-
* Optionally, the function allows you to filter the target links' URLs using an array of
|
|
163
|
-
* and override settings of the enqueued {@link Request} objects.
|
|
127
|
+
* Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
|
|
164
128
|
*
|
|
165
129
|
* **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
|
|
166
130
|
* such as changing the Z-index of elements being clicked and their visibility. Therefore,
|
|
@@ -178,13 +142,13 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
178
142
|
* **Example usage**
|
|
179
143
|
*
|
|
180
144
|
* ```javascript
|
|
181
|
-
* await
|
|
145
|
+
* await puppeteerUtils.enqueueLinksByClickingElements({
|
|
182
146
|
* page,
|
|
183
147
|
* requestManager,
|
|
184
148
|
* selector: 'a.product-detail',
|
|
185
|
-
*
|
|
186
|
-
* 'https://www.example.com/handbags
|
|
187
|
-
* 'https://www.example.com/purses
|
|
149
|
+
* include: [
|
|
150
|
+
* 'https://www.example.com/handbags/*',
|
|
151
|
+
* 'https://www.example.com/purses/*',
|
|
188
152
|
* ],
|
|
189
153
|
* });
|
|
190
154
|
* ```
|
|
@@ -205,20 +169,19 @@ interface ClickElementsAndInterceptNavigationRequestsOptions extends WaitForPage
|
|
|
205
169
|
* Clicks all elements of given page matching given selector.
|
|
206
170
|
* Catches and intercepts all initiated navigation requests and opened pages.
|
|
207
171
|
* Returns a list of all target URLs.
|
|
172
|
+
*
|
|
173
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
208
174
|
* @ignore
|
|
209
175
|
*/
|
|
210
176
|
export declare function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
|
|
211
|
-
/**
|
|
212
|
-
* We're only interested in pages created by the page we're currently clicking in.
|
|
213
|
-
* There will generally be a lot of other targets being created in the browser.
|
|
214
|
-
*/
|
|
215
|
-
export declare function isTargetRelevant(page: Page, target: Target): boolean;
|
|
216
177
|
/**
|
|
217
178
|
* Click all elements matching the given selector. To be able to do this using
|
|
218
179
|
* Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
|
|
219
180
|
* so we first move them to the top of the page's stacking context and then click.
|
|
220
181
|
* We do all in series to prevent elements from hiding one another. Therefore,
|
|
221
182
|
* for large element sets, this will take considerable amount of time.
|
|
183
|
+
*
|
|
184
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
222
185
|
* @ignore
|
|
223
186
|
*/
|
|
224
187
|
export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
|
|
@@ -1,9 +1,26 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
|
-
import { applyRequestTransform,
|
|
3
|
-
import
|
|
2
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
4
5
|
import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
|
|
5
6
|
const STARTING_Z_INDEX = 2147400000;
|
|
6
7
|
const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
|
|
8
|
+
const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
|
|
9
|
+
page: schemas.objectWithKeys(['goto', 'evaluate']),
|
|
10
|
+
requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
|
|
11
|
+
selector: z.string(),
|
|
12
|
+
userData: schemas.anyObject.optional(),
|
|
13
|
+
clickOptions: schemas.anyObject.optional(),
|
|
14
|
+
include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
15
|
+
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
16
|
+
transformRequestFunction: schemas.anyFunction.optional(),
|
|
17
|
+
waitForPageIdleSecs: schemas.anyNumber.default(1),
|
|
18
|
+
maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
|
|
19
|
+
label: z.string().optional(),
|
|
20
|
+
forefront: z.boolean().optional(),
|
|
21
|
+
skipNavigation: z.boolean().optional(),
|
|
22
|
+
onSkippedRequest: schemas.anyFunction.optional(),
|
|
23
|
+
});
|
|
7
24
|
/**
|
|
8
25
|
* The function finds elements matching a specific CSS selector in a Puppeteer page,
|
|
9
26
|
* clicks all those elements using a mouse move and a left mouse button click and intercepts
|
|
@@ -13,8 +30,7 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
|
|
|
13
30
|
* in `href` elements, but rather navigations are triggered in click handlers.
|
|
14
31
|
* If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
|
|
15
32
|
*
|
|
16
|
-
* Optionally, the function allows you to filter the target links' URLs using an array of
|
|
17
|
-
* and override settings of the enqueued {@link Request} objects.
|
|
33
|
+
* Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
|
|
18
34
|
*
|
|
19
35
|
* **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
|
|
20
36
|
* such as changing the Z-index of elements being clicked and their visibility. Therefore,
|
|
@@ -32,13 +48,13 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
|
|
|
32
48
|
* **Example usage**
|
|
33
49
|
*
|
|
34
50
|
* ```javascript
|
|
35
|
-
* await
|
|
51
|
+
* await puppeteerUtils.enqueueLinksByClickingElements({
|
|
36
52
|
* page,
|
|
37
53
|
* requestManager,
|
|
38
54
|
* selector: 'a.product-detail',
|
|
39
|
-
*
|
|
40
|
-
* 'https://www.example.com/handbags
|
|
41
|
-
* 'https://www.example.com/purses
|
|
55
|
+
* include: [
|
|
56
|
+
* 'https://www.example.com/handbags/*',
|
|
57
|
+
* 'https://www.example.com/purses/*',
|
|
42
58
|
* ],
|
|
43
59
|
* });
|
|
44
60
|
* ```
|
|
@@ -46,51 +62,13 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
|
|
|
46
62
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
47
63
|
*/
|
|
48
64
|
export async function enqueueLinksByClickingElements(options) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
requestManager: ow.object.hasKeys('fetchNextRequest', 'addRequestsBatched'),
|
|
52
|
-
selector: ow.string,
|
|
53
|
-
userData: ow.optional.object,
|
|
54
|
-
clickOptions: ow.optional.object,
|
|
55
|
-
pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
|
|
56
|
-
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
|
|
57
|
-
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
|
|
58
|
-
exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
|
|
59
|
-
transformRequestFunction: ow.optional.function,
|
|
60
|
-
waitForPageIdleSecs: ow.optional.number,
|
|
61
|
-
maxWaitForPageIdleSecs: ow.optional.number,
|
|
62
|
-
label: ow.optional.string,
|
|
63
|
-
forefront: ow.optional.boolean,
|
|
64
|
-
skipNavigation: ow.optional.boolean,
|
|
65
|
-
onSkippedRequest: ow.optional.function,
|
|
66
|
-
}));
|
|
67
|
-
const { page, requestManager, selector, clickOptions,
|
|
68
|
-
// oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
|
|
69
|
-
pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
|
|
65
|
+
const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
|
|
66
|
+
const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
|
|
70
67
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
71
68
|
const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
|
|
72
|
-
const
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
for (const excl of exclude) {
|
|
76
|
-
if (typeof excl === 'string' || 'glob' in excl) {
|
|
77
|
-
urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
|
|
78
|
-
}
|
|
79
|
-
else if (excl instanceof RegExp || 'regexp' in excl) {
|
|
80
|
-
urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
if (pseudoUrls?.length) {
|
|
85
|
-
getLog().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
|
|
86
|
-
urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
|
|
87
|
-
}
|
|
88
|
-
if (globs?.length) {
|
|
89
|
-
urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
|
|
90
|
-
}
|
|
91
|
-
if (regexps?.length) {
|
|
92
|
-
urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
|
|
93
|
-
}
|
|
69
|
+
const hasOnSkippedRequest = onSkippedRequest !== undefined;
|
|
70
|
+
const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
|
|
71
|
+
const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
|
|
94
72
|
const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
|
|
95
73
|
page,
|
|
96
74
|
selector,
|
|
@@ -98,17 +76,17 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
98
76
|
maxWaitForPageIdleMillis,
|
|
99
77
|
clickOptions,
|
|
100
78
|
});
|
|
101
|
-
const requestOptions = createRequestOptions(interceptedRequests,
|
|
79
|
+
const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
|
|
102
80
|
const skippedByFilters = [];
|
|
103
|
-
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (
|
|
81
|
+
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
|
|
104
82
|
if (onSkippedRequest && skippedByFilters.length > 0) {
|
|
105
|
-
await Promise.all(skippedByFilters.map(async (
|
|
83
|
+
await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
|
|
106
84
|
}
|
|
107
85
|
if (transformRequestFunction) {
|
|
108
86
|
const skippedByTransform = [];
|
|
109
|
-
filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
|
|
87
|
+
filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
|
|
110
88
|
if (onSkippedRequest && skippedByTransform.length > 0) {
|
|
111
|
-
await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(
|
|
89
|
+
await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
|
|
112
90
|
}
|
|
113
91
|
}
|
|
114
92
|
const requests = filteredOptions.map((opts) => new Request(opts));
|
|
@@ -119,6 +97,8 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
119
97
|
* Clicks all elements of given page matching given selector.
|
|
120
98
|
* Catches and intercepts all initiated navigation requests and opened pages.
|
|
121
99
|
* Returns a list of all target URLs.
|
|
100
|
+
*
|
|
101
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
122
102
|
* @ignore
|
|
123
103
|
*/
|
|
124
104
|
export async function clickElementsAndInterceptNavigationRequests(options) {
|
|
@@ -197,7 +177,7 @@ function createTargetCreatedHandler(page, requests) {
|
|
|
197
177
|
* We're only interested in pages created by the page we're currently clicking in.
|
|
198
178
|
* There will generally be a lot of other targets being created in the browser.
|
|
199
179
|
*/
|
|
200
|
-
|
|
180
|
+
function isTargetRelevant(page, target) {
|
|
201
181
|
// oxlint-disable-next-line typescript/no-deprecated -- the non-deprecated replacement (opener.page()) is async and would force every call site to await, including EventEmitter callbacks
|
|
202
182
|
return target.type() === 'page' && page.target() === target.opener();
|
|
203
183
|
}
|
|
@@ -242,6 +222,8 @@ async function preventHistoryNavigation(page) {
|
|
|
242
222
|
* so we first move them to the top of the page's stacking context and then click.
|
|
243
223
|
* We do all in series to prevent elements from hiding one another. Therefore,
|
|
244
224
|
* for large element sets, this will take considerable amount of time.
|
|
225
|
+
*
|
|
226
|
+
* Not part of the public API — exported only so tests can import this module directly.
|
|
245
227
|
* @ignore
|
|
246
228
|
*/
|
|
247
229
|
export async function clickElements(page, selector, clickOptions) {
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { Configuration } from '@crawlee/browser';
|
|
2
|
+
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
|
|
3
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
+
import type { Page } from 'puppeteer';
|
|
5
|
+
import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
|
|
6
|
+
/** A {@link BrowserPool} of Puppeteer browsers, as built by {@link puppeteerBrowserPool}. */
|
|
7
|
+
export type PuppeteerBrowserPool = BrowserPool<{
|
|
8
|
+
browserPlugins: [PuppeteerPlugin];
|
|
9
|
+
}, [PuppeteerPlugin]>;
|
|
10
|
+
export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {
|
|
11
|
+
/** How to launch the browser: proxy, user data dir, whether to use full Chrome, ... */
|
|
12
|
+
launchContext?: PuppeteerLaunchContext;
|
|
13
|
+
/**
|
|
14
|
+
* Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
|
|
15
|
+
* Defaults to `true`, and can also be set via {@link Configuration}.
|
|
16
|
+
*/
|
|
17
|
+
headless?: boolean | 'new' | 'old';
|
|
18
|
+
/** Configuration to read the browser defaults from. Defaults to the global configuration. */
|
|
19
|
+
configuration?: Configuration;
|
|
20
|
+
}
|
|
21
|
+
export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
|
|
25
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
26
|
+
*
|
|
27
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
28
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
29
|
+
* given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
|
|
30
|
+
*
|
|
31
|
+
* **Example usage:**
|
|
32
|
+
*
|
|
33
|
+
* ```javascript
|
|
34
|
+
* const crawler = new PuppeteerCrawler({
|
|
35
|
+
* browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
|
|
36
|
+
* requestHandler: async ({ page }) => { ... },
|
|
37
|
+
* });
|
|
38
|
+
* ```
|
|
39
|
+
*
|
|
40
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
41
|
+
*
|
|
42
|
+
* @category Browser management
|
|
43
|
+
*/
|
|
44
|
+
export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
|
|
45
|
+
/**
|
|
46
|
+
* The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
|
|
47
|
+
* service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
|
|
48
|
+
*
|
|
49
|
+
* A {@link PuppeteerCrawler} accepts the same connection details directly via
|
|
50
|
+
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
51
|
+
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
52
|
+
*
|
|
53
|
+
* @category Browser management
|
|
54
|
+
*/
|
|
55
|
+
export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { PuppeteerLauncher } from './puppeteer-launcher.js';
|
|
2
|
+
/**
|
|
3
|
+
* Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
|
|
4
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
5
|
+
*
|
|
6
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
7
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
8
|
+
* given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
|
|
9
|
+
*
|
|
10
|
+
* **Example usage:**
|
|
11
|
+
*
|
|
12
|
+
* ```javascript
|
|
13
|
+
* const crawler = new PuppeteerCrawler({
|
|
14
|
+
* browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
|
|
15
|
+
* requestHandler: async ({ page }) => { ... },
|
|
16
|
+
* });
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
20
|
+
*
|
|
21
|
+
* @category Browser management
|
|
22
|
+
*/
|
|
23
|
+
export function puppeteerBrowserPool(options = {}) {
|
|
24
|
+
const { launchContext, headless, configuration, ...poolOptions } = options;
|
|
25
|
+
return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
|
|
29
|
+
* service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
|
|
30
|
+
*
|
|
31
|
+
* A {@link PuppeteerCrawler} accepts the same connection details directly via
|
|
32
|
+
* {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
|
|
33
|
+
* tune the wrapping pool, or to share one remote pool between crawlers.
|
|
34
|
+
*
|
|
35
|
+
* @category Browser management
|
|
36
|
+
*/
|
|
37
|
+
export function remotePuppeteerBrowserPool(options) {
|
|
38
|
+
const { launchContext, headless, configuration, ...remoteOptions } = options;
|
|
39
|
+
return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
|
|
40
|
+
}
|
|
41
|
+
function puppeteerLauncher(launchContext = {}, headless, configuration) {
|
|
42
|
+
return new PuppeteerLauncher(headless == null
|
|
43
|
+
? launchContext
|
|
44
|
+
: {
|
|
45
|
+
...launchContext,
|
|
46
|
+
launchOptions: { ...launchContext.launchOptions, headless: headless },
|
|
47
|
+
}, configuration);
|
|
48
|
+
}
|