@crawlee/playwright 4.0.0-beta.27 → 4.0.0-beta.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +4 -3
- package/index.js +2 -2
- package/internals/adaptive-playwright-crawler.d.ts +145 -74
- package/internals/adaptive-playwright-crawler.js +492 -296
- package/internals/enqueue-links/click-elements.d.ts +21 -58
- package/internals/enqueue-links/click-elements.js +58 -70
- package/internals/playwright-browser-pool.d.ts +55 -0
- package/internals/playwright-browser-pool.js +34 -0
- package/internals/playwright-crawler.d.ts +179 -149
- package/internals/playwright-crawler.js +93 -79
- package/internals/playwright-launcher.d.ts +30 -20
- package/internals/playwright-launcher.js +24 -18
- package/internals/utils/playwright-utils.d.ts +64 -56
- package/internals/utils/playwright-utils.js +117 -138
- package/internals/utils/rendering-type-prediction.d.ts +34 -14
- package/internals/utils/rendering-type-prediction.js +87 -35
- package/package.json +19 -22
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/adaptive-playwright-crawler.d.ts.map +0 -1
- package/internals/adaptive-playwright-crawler.js.map +0 -1
- package/internals/enqueue-links/click-elements.d.ts.map +0 -1
- package/internals/enqueue-links/click-elements.js.map +0 -1
- package/internals/playwright-crawler.d.ts.map +0 -1
- package/internals/playwright-crawler.js.map +0 -1
- package/internals/playwright-launcher.d.ts.map +0 -1
- package/internals/playwright-launcher.js.map +0 -1
- package/internals/utils/playwright-utils.d.ts.map +0 -1
- package/internals/utils/playwright-utils.js.map +0 -1
- package/internals/utils/rendering-type-prediction.d.ts.map +0 -1
- package/internals/utils/rendering-type-prediction.js.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
|
|
2
2
|
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
3
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
4
|
import type { Page } from 'playwright';
|
|
@@ -9,9 +9,9 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
9
9
|
*/
|
|
10
10
|
page: Page;
|
|
11
11
|
/**
|
|
12
|
-
* A request
|
|
12
|
+
* * A request manager to which the URLs will be enqueued.
|
|
13
13
|
*/
|
|
14
|
-
|
|
14
|
+
requestManager: IRequestManager;
|
|
15
15
|
/**
|
|
16
16
|
* A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
|
|
17
17
|
* value. This is to prevent suboptimal use of this function by using it too broadly.
|
|
@@ -26,64 +26,29 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
26
26
|
*/
|
|
27
27
|
clickOptions?: ClickOptions;
|
|
28
28
|
/**
|
|
29
|
-
* An array of
|
|
30
|
-
* containing glob pattern strings matching the URLs to be enqueued.
|
|
29
|
+
* An array of URL patterns that URLs must match to be enqueued.
|
|
31
30
|
*
|
|
32
|
-
*
|
|
33
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
|
|
34
|
-
*
|
|
35
|
-
* The matching is always case-insensitive.
|
|
36
|
-
* If you need case-sensitive matching, use `regexps` property directly.
|
|
37
|
-
*
|
|
38
|
-
* If `globs` is an empty array or `undefined`, then the function
|
|
39
|
-
* enqueues all the intercepted navigation requests produced by the page
|
|
40
|
-
* after clicking on elements matching the provided CSS selector.
|
|
41
|
-
*/
|
|
42
|
-
globs?: GlobInput[];
|
|
43
|
-
/**
|
|
44
|
-
* An array of glob pattern strings, regexp patterns or plain objects
|
|
45
|
-
* containing patterns matching URLs that will **never** be enqueued.
|
|
46
|
-
*
|
|
47
|
-
* The plain objects must include either the `glob` property or the `regexp` property.
|
|
31
|
+
* Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
|
|
48
32
|
*
|
|
49
33
|
* Glob matching is always case-insensitive.
|
|
50
|
-
* If you need case-sensitive matching,
|
|
51
|
-
*/
|
|
52
|
-
exclude?: readonly (GlobInput | RegExpInput)[];
|
|
53
|
-
/**
|
|
54
|
-
* An array of regular expressions or plain objects
|
|
55
|
-
* containing regular expressions matching the URLs to be enqueued.
|
|
56
|
-
*
|
|
57
|
-
* The plain objects must include at least the `regexp` property, which holds the regular expression.
|
|
58
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
|
|
34
|
+
* If you need case-sensitive matching, use a `RegExp`.
|
|
59
35
|
*
|
|
60
|
-
* If `
|
|
36
|
+
* If `include` is an empty array or `undefined`, then the function
|
|
61
37
|
* enqueues all the intercepted navigation requests produced by the page
|
|
62
38
|
* after clicking on elements matching the provided CSS selector.
|
|
63
39
|
*/
|
|
64
|
-
|
|
40
|
+
include?: UrlPatternInput[];
|
|
65
41
|
/**
|
|
66
|
-
*
|
|
67
|
-
* Please use `globs` or `regexps` instead.
|
|
68
|
-
*
|
|
69
|
-
* An array of {@link PseudoUrl} strings or plain objects
|
|
70
|
-
* containing {@link PseudoUrl} strings matching the URLs to be enqueued.
|
|
71
|
-
*
|
|
72
|
-
* The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
|
|
73
|
-
* All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
|
|
42
|
+
* An array of URL patterns. Matching URLs will **not** be enqueued.
|
|
74
43
|
*
|
|
75
|
-
*
|
|
76
|
-
* If you need case-sensitive matching, use `regexps` property directly.
|
|
44
|
+
* Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
|
|
77
45
|
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
* after clicking on elements matching the provided CSS selector.
|
|
81
|
-
*
|
|
82
|
-
* @deprecated prefer using `globs` or `regexps` instead
|
|
46
|
+
* Glob matching is always case-insensitive.
|
|
47
|
+
* If you need case-sensitive matching, use a `RegExp`.
|
|
83
48
|
*/
|
|
84
|
-
|
|
49
|
+
exclude?: readonly UrlPatternInput[];
|
|
85
50
|
/**
|
|
86
|
-
* After
|
|
51
|
+
* After request options are filtered by `include`/`exclude` patterns,
|
|
87
52
|
* this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
|
|
88
53
|
* `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
|
|
89
54
|
* but differ in methods or payloads, or to dynamically update or create `userData`.
|
|
@@ -98,8 +63,8 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
98
63
|
* }
|
|
99
64
|
* ```
|
|
100
65
|
*
|
|
101
|
-
* Note that `transformRequestFunction` has the highest priority and can overwrite
|
|
102
|
-
*
|
|
66
|
+
* Note that `transformRequestFunction` has the highest priority and can overwrite
|
|
67
|
+
* the global `label` option.
|
|
103
68
|
*
|
|
104
69
|
* The function receives a {@link RequestOptions} object and can return either:
|
|
105
70
|
* - The modified {@link RequestOptions} object
|
|
@@ -160,8 +125,7 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
160
125
|
* in `href` elements, but rather navigations are triggered in click handlers.
|
|
161
126
|
* If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
|
|
162
127
|
*
|
|
163
|
-
* Optionally, the function allows you to filter the target links' URLs using an array of
|
|
164
|
-
* and override settings of the enqueued {@link Request} objects.
|
|
128
|
+
* Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
|
|
165
129
|
*
|
|
166
130
|
* **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
|
|
167
131
|
* such as changing the Z-index of elements being clicked and their visibility. Therefore,
|
|
@@ -181,11 +145,11 @@ export interface EnqueueLinksByClickingElementsOptions {
|
|
|
181
145
|
* ```javascript
|
|
182
146
|
* await playwrightUtils.enqueueLinksByClickingElements({
|
|
183
147
|
* page,
|
|
184
|
-
*
|
|
148
|
+
* requestManager,
|
|
185
149
|
* selector: 'a.product-detail',
|
|
186
|
-
*
|
|
187
|
-
* 'https://www.example.com/handbags
|
|
188
|
-
* 'https://www.example.com/purses
|
|
150
|
+
* include: [
|
|
151
|
+
* 'https://www.example.com/handbags/*',
|
|
152
|
+
* 'https://www.example.com/purses/*',
|
|
189
153
|
* ],
|
|
190
154
|
* });
|
|
191
155
|
* ```
|
|
@@ -219,4 +183,3 @@ export declare function clickElementsAndInterceptNavigationRequests(options: Cli
|
|
|
219
183
|
*/
|
|
220
184
|
export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
|
|
221
185
|
export {};
|
|
222
|
-
//# sourceMappingURL=click-elements.d.ts.map
|
|
@@ -1,9 +1,25 @@
|
|
|
1
1
|
import { URL } from 'node:url';
|
|
2
|
-
import { applyRequestTransform,
|
|
3
|
-
import
|
|
4
|
-
import
|
|
2
|
+
import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request as CrawleeRequest, serviceLocator, } from '@crawlee/browser';
|
|
3
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
5
5
|
const STARTING_Z_INDEX = 2147400000;
|
|
6
|
-
const
|
|
6
|
+
const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
|
|
7
|
+
const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
|
|
8
|
+
page: schemas.objectWithKeys(['goto', 'evaluate']),
|
|
9
|
+
requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
|
|
10
|
+
selector: z.string(),
|
|
11
|
+
userData: schemas.anyObject.optional(),
|
|
12
|
+
clickOptions: schemas.anyObject.optional(),
|
|
13
|
+
include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
14
|
+
exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
|
|
15
|
+
transformRequestFunction: schemas.anyFunction.optional(),
|
|
16
|
+
waitForPageIdleSecs: schemas.anyNumber.default(1),
|
|
17
|
+
maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
|
|
18
|
+
label: z.string().optional(),
|
|
19
|
+
forefront: z.boolean().optional(),
|
|
20
|
+
skipNavigation: z.boolean().optional(),
|
|
21
|
+
onSkippedRequest: schemas.anyFunction.optional(),
|
|
22
|
+
});
|
|
7
23
|
/**
|
|
8
24
|
* The function finds elements matching a specific CSS selector in a Playwright page,
|
|
9
25
|
* clicks all those elements using a mouse move and a left mouse button click and intercepts
|
|
@@ -13,8 +29,7 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
|
|
|
13
29
|
* in `href` elements, but rather navigations are triggered in click handlers.
|
|
14
30
|
* If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
|
|
15
31
|
*
|
|
16
|
-
* Optionally, the function allows you to filter the target links' URLs using an array of
|
|
17
|
-
* and override settings of the enqueued {@link Request} objects.
|
|
32
|
+
* Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
|
|
18
33
|
*
|
|
19
34
|
* **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
|
|
20
35
|
* such as changing the Z-index of elements being clicked and their visibility. Therefore,
|
|
@@ -34,11 +49,11 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
|
|
|
34
49
|
* ```javascript
|
|
35
50
|
* await playwrightUtils.enqueueLinksByClickingElements({
|
|
36
51
|
* page,
|
|
37
|
-
*
|
|
52
|
+
* requestManager,
|
|
38
53
|
* selector: 'a.product-detail',
|
|
39
|
-
*
|
|
40
|
-
* 'https://www.example.com/handbags
|
|
41
|
-
* 'https://www.example.com/purses
|
|
54
|
+
* include: [
|
|
55
|
+
* 'https://www.example.com/handbags/*',
|
|
56
|
+
* 'https://www.example.com/purses/*',
|
|
42
57
|
* ],
|
|
43
58
|
* });
|
|
44
59
|
* ```
|
|
@@ -46,49 +61,13 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
|
|
|
46
61
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
47
62
|
*/
|
|
48
63
|
export async function enqueueLinksByClickingElements(options) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
|
|
52
|
-
selector: ow.string,
|
|
53
|
-
userData: ow.optional.object,
|
|
54
|
-
clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
|
|
55
|
-
pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
|
|
56
|
-
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
|
|
57
|
-
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
|
|
58
|
-
exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
|
|
59
|
-
transformRequestFunction: ow.optional.function,
|
|
60
|
-
waitForPageIdleSecs: ow.optional.number,
|
|
61
|
-
maxWaitForPageIdleSecs: ow.optional.number,
|
|
62
|
-
label: ow.optional.string,
|
|
63
|
-
forefront: ow.optional.boolean,
|
|
64
|
-
skipNavigation: ow.optional.boolean,
|
|
65
|
-
onSkippedRequest: ow.optional.function,
|
|
66
|
-
}));
|
|
67
|
-
const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
|
|
64
|
+
const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
|
|
65
|
+
const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
|
|
68
66
|
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
|
|
69
67
|
const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
|
|
70
|
-
const
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
for (const excl of exclude) {
|
|
74
|
-
if (typeof excl === 'string' || 'glob' in excl) {
|
|
75
|
-
urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
|
|
76
|
-
}
|
|
77
|
-
else if (excl instanceof RegExp || 'regexp' in excl) {
|
|
78
|
-
urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
if (pseudoUrls?.length) {
|
|
83
|
-
log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
|
|
84
|
-
urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
|
|
85
|
-
}
|
|
86
|
-
if (globs?.length) {
|
|
87
|
-
urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
|
|
88
|
-
}
|
|
89
|
-
if (regexps?.length) {
|
|
90
|
-
urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
|
|
91
|
-
}
|
|
68
|
+
const hasOnSkippedRequest = onSkippedRequest !== undefined;
|
|
69
|
+
const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
|
|
70
|
+
const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
|
|
92
71
|
const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
|
|
93
72
|
page,
|
|
94
73
|
selector,
|
|
@@ -96,21 +75,21 @@ export async function enqueueLinksByClickingElements(options) {
|
|
|
96
75
|
maxWaitForPageIdleMillis,
|
|
97
76
|
clickOptions,
|
|
98
77
|
});
|
|
99
|
-
const requestOptions = createRequestOptions(interceptedRequests,
|
|
78
|
+
const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
|
|
100
79
|
const skippedByFilters = [];
|
|
101
|
-
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (
|
|
80
|
+
let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
|
|
102
81
|
if (onSkippedRequest && skippedByFilters.length > 0) {
|
|
103
|
-
await Promise.all(skippedByFilters.map((
|
|
82
|
+
await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
|
|
104
83
|
}
|
|
105
84
|
if (transformRequestFunction) {
|
|
106
85
|
const skippedByTransform = [];
|
|
107
|
-
filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
|
|
86
|
+
filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
|
|
108
87
|
if (onSkippedRequest && skippedByTransform.length > 0) {
|
|
109
|
-
await Promise.all(skippedByTransform.map((r) => onSkippedRequest(
|
|
88
|
+
await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
|
|
110
89
|
}
|
|
111
90
|
}
|
|
112
91
|
const requests = filteredOptions.map((opts) => new CrawleeRequest(opts));
|
|
113
|
-
const { addedRequests } = await
|
|
92
|
+
const { addedRequests } = await requestManager.addRequestsBatched(requests, { forefront });
|
|
114
93
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
115
94
|
}
|
|
116
95
|
/**
|
|
@@ -136,7 +115,7 @@ export async function clickElementsAndInterceptNavigationRequests(options) {
|
|
|
136
115
|
await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
|
|
137
116
|
// browser.off(BrowserEmittedEvents.TargetCreated, onTargetCreated);
|
|
138
117
|
page.off('framenavigated', onFrameNavigated);
|
|
139
|
-
await context.unroute('
|
|
118
|
+
await context.unroute('**', onInterceptedRequest);
|
|
140
119
|
const serializedRequests = Array.from(uniqueRequests);
|
|
141
120
|
return serializedRequests.map((r) => JSON.parse(r));
|
|
142
121
|
}
|
|
@@ -172,7 +151,9 @@ function createTargetCreatedHandler(requests) {
|
|
|
172
151
|
await popup.close();
|
|
173
152
|
}
|
|
174
153
|
catch (err) {
|
|
175
|
-
|
|
154
|
+
getLog().debug('enqueueLinksByClickingElements: Could not close spawned page.', {
|
|
155
|
+
error: err.stack,
|
|
156
|
+
});
|
|
176
157
|
}
|
|
177
158
|
};
|
|
178
159
|
}
|
|
@@ -180,7 +161,16 @@ function createTargetCreatedHandler(requests) {
|
|
|
180
161
|
* @ignore
|
|
181
162
|
*/
|
|
182
163
|
function isTopFrameNavigationRequest(page, req) {
|
|
183
|
-
|
|
164
|
+
try {
|
|
165
|
+
return req.isNavigationRequest() && req.frame() === page.mainFrame();
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
// `req.frame()` throws when the owning frame is unavailable - e.g. the request was
|
|
169
|
+
// issued by a service worker, or before/after its frame existed (see #3216). Such a
|
|
170
|
+
// request is not a top-frame navigation, so swallow the throw and let it pass through
|
|
171
|
+
// instead of crashing the route handler (which would leave the route unhandled).
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
184
174
|
}
|
|
185
175
|
/**
|
|
186
176
|
* @ignore
|
|
@@ -245,7 +235,7 @@ function updateElementCssToEnableMouseClick(el, zIndex) {
|
|
|
245
235
|
*/
|
|
246
236
|
export async function clickElements(page, selector, clickOptions) {
|
|
247
237
|
const elementHandles = await page.$$(selector);
|
|
248
|
-
|
|
238
|
+
getLog().debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
|
|
249
239
|
let clickedElementsCount = 0;
|
|
250
240
|
let zIndex = STARTING_Z_INDEX;
|
|
251
241
|
let shouldLogWarning = true;
|
|
@@ -258,15 +248,15 @@ export async function clickElements(page, selector, clickOptions) {
|
|
|
258
248
|
catch (err) {
|
|
259
249
|
const e = err;
|
|
260
250
|
if (shouldLogWarning && e.stack.includes('is detached from document')) {
|
|
261
|
-
|
|
251
|
+
getLog().warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
|
|
262
252
|
'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. ' +
|
|
263
253
|
'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
|
|
264
254
|
shouldLogWarning = false;
|
|
265
255
|
}
|
|
266
|
-
|
|
256
|
+
getLog().debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
|
|
267
257
|
}
|
|
268
258
|
}
|
|
269
|
-
|
|
259
|
+
getLog().debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
|
|
270
260
|
}
|
|
271
261
|
/**
|
|
272
262
|
* This function tracks whether any requests, frame navigations or targets were emitted
|
|
@@ -285,8 +275,6 @@ export async function clickElements(page, selector, clickOptions) {
|
|
|
285
275
|
async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis, }) {
|
|
286
276
|
return new Promise((resolve) => {
|
|
287
277
|
let timeout;
|
|
288
|
-
let maxTimeout;
|
|
289
|
-
page.on('popup', activityHandler);
|
|
290
278
|
function activityHandler() {
|
|
291
279
|
clearTimeout(timeout);
|
|
292
280
|
timeout = setTimeout(() => {
|
|
@@ -295,7 +283,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
|
|
|
295
283
|
}, waitForPageIdleMillis);
|
|
296
284
|
}
|
|
297
285
|
function maxTimeoutHandler() {
|
|
298
|
-
|
|
286
|
+
getLog().debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
|
|
299
287
|
'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
|
|
300
288
|
finish();
|
|
301
289
|
}
|
|
@@ -303,7 +291,8 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
|
|
|
303
291
|
page.off('request', activityHandler).off('framenavigated', activityHandler).off('popup', activityHandler);
|
|
304
292
|
resolve();
|
|
305
293
|
}
|
|
306
|
-
maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
|
|
294
|
+
const maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
|
|
295
|
+
page.on('popup', activityHandler);
|
|
307
296
|
activityHandler(); // We call this once manually in case there would be no requests at all.
|
|
308
297
|
page.on('request', activityHandler);
|
|
309
298
|
page.on('framenavigated', activityHandler);
|
|
@@ -326,8 +315,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
|
|
|
326
315
|
requests.add(JSON.stringify({ url }));
|
|
327
316
|
}
|
|
328
317
|
catch (err) {
|
|
329
|
-
|
|
318
|
+
getLog().debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
|
|
330
319
|
}
|
|
331
320
|
});
|
|
332
321
|
}
|
|
333
|
-
//# sourceMappingURL=click-elements.js.map
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { Configuration, CrawlerRemoteBrowserOptions } from '@crawlee/browser';
|
|
2
|
+
import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PlaywrightPlugin } from '@crawlee/browser-pool';
|
|
3
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
+
import type { Page } from 'playwright';
|
|
5
|
+
import type { PlaywrightLaunchContext } from './playwright-launcher.js';
|
|
6
|
+
/** {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`} for Playwright, with the protocol to connect over. */
|
|
7
|
+
export interface PlaywrightRemoteBrowserOptions extends CrawlerRemoteBrowserOptions {
|
|
8
|
+
/**
|
|
9
|
+
* `'cdp'` uses `connectOverCDP()` (the default), `'playwright'` uses `connect()` (Playwright's own WebSocket
|
|
10
|
+
* protocol, for endpoints started with `launchServer()`).
|
|
11
|
+
*/
|
|
12
|
+
protocol?: 'cdp' | 'playwright';
|
|
13
|
+
}
|
|
14
|
+
/** A {@link BrowserPool} of Playwright browsers, as built by {@link playwrightBrowserPool}. */
|
|
15
|
+
export type PlaywrightBrowserPool = BrowserPool<{
|
|
16
|
+
browserPlugins: [PlaywrightPlugin];
|
|
17
|
+
}, [PlaywrightPlugin]>;
|
|
18
|
+
export interface PlaywrightBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PlaywrightPlugin['createController']>, ReturnType<PlaywrightPlugin['createLaunchContext']>, Page> {
|
|
19
|
+
/** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
|
|
20
|
+
launchContext?: PlaywrightLaunchContext;
|
|
21
|
+
/**
|
|
22
|
+
* Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
|
|
23
|
+
* Defaults to `true`, and can also be set via {@link Configuration}.
|
|
24
|
+
*/
|
|
25
|
+
headless?: boolean;
|
|
26
|
+
/** Configuration to read the browser defaults from. Defaults to the global configuration. */
|
|
27
|
+
configuration?: Configuration;
|
|
28
|
+
/** Connect to a remote browser service instead of launching locally; see {@link BrowserCrawlerOptions.remoteBrowser}. */
|
|
29
|
+
remoteBrowser?: PlaywrightRemoteBrowserOptions;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
|
|
33
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
34
|
+
*
|
|
35
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
36
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
37
|
+
* given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
|
|
38
|
+
*
|
|
39
|
+
* **Example usage:**
|
|
40
|
+
*
|
|
41
|
+
* ```javascript
|
|
42
|
+
* const crawler = new PlaywrightCrawler({
|
|
43
|
+
* browserPool: playwrightBrowserPool({
|
|
44
|
+
* maxOpenPagesPerBrowser: 1,
|
|
45
|
+
* launchContext: { launcher: firefox },
|
|
46
|
+
* }),
|
|
47
|
+
* requestHandler: async ({ page }) => { ... },
|
|
48
|
+
* });
|
|
49
|
+
* ```
|
|
50
|
+
*
|
|
51
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
52
|
+
*
|
|
53
|
+
* @category Browser management
|
|
54
|
+
*/
|
|
55
|
+
export declare function playwrightBrowserPool(options?: PlaywrightBrowserPoolOptions): PlaywrightBrowserPool;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { PlaywrightLauncher } from './playwright-launcher.js';
|
|
2
|
+
/**
|
|
3
|
+
* Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
|
|
4
|
+
* {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
|
|
5
|
+
*
|
|
6
|
+
* It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
|
|
7
|
+
* `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
|
|
8
|
+
* given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
|
|
9
|
+
*
|
|
10
|
+
* **Example usage:**
|
|
11
|
+
*
|
|
12
|
+
* ```javascript
|
|
13
|
+
* const crawler = new PlaywrightCrawler({
|
|
14
|
+
* browserPool: playwrightBrowserPool({
|
|
15
|
+
* maxOpenPagesPerBrowser: 1,
|
|
16
|
+
* launchContext: { launcher: firefox },
|
|
17
|
+
* }),
|
|
18
|
+
* requestHandler: async ({ page }) => { ... },
|
|
19
|
+
* });
|
|
20
|
+
* ```
|
|
21
|
+
*
|
|
22
|
+
* The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
|
|
23
|
+
*
|
|
24
|
+
* @category Browser management
|
|
25
|
+
*/
|
|
26
|
+
export function playwrightBrowserPool(options = {}) {
|
|
27
|
+
const { launchContext, headless, configuration, ...poolOptions } = options;
|
|
28
|
+
return playwrightLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
|
|
29
|
+
}
|
|
30
|
+
function playwrightLauncher(launchContext = {}, headless, configuration) {
|
|
31
|
+
return new PlaywrightLauncher(headless == null
|
|
32
|
+
? launchContext
|
|
33
|
+
: { ...launchContext, launchOptions: { ...launchContext.launchOptions, headless } }, configuration);
|
|
34
|
+
}
|