@crawlee/playwright 4.0.0-beta.28 → 4.0.0-beta.280

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/README.md +15 -15
  2. package/index.d.ts +4 -3
  3. package/index.js +2 -2
  4. package/internals/adaptive-playwright-crawler.d.ts +145 -73
  5. package/internals/adaptive-playwright-crawler.js +505 -290
  6. package/internals/enqueue-links/click-elements.d.ts +21 -58
  7. package/internals/enqueue-links/click-elements.js +58 -70
  8. package/internals/playwright-browser-pool.d.ts +55 -0
  9. package/internals/playwright-browser-pool.js +34 -0
  10. package/internals/playwright-crawler.d.ts +178 -153
  11. package/internals/playwright-crawler.js +90 -74
  12. package/internals/playwright-launcher.d.ts +30 -20
  13. package/internals/playwright-launcher.js +24 -18
  14. package/internals/utils/playwright-utils.d.ts +64 -56
  15. package/internals/utils/playwright-utils.js +117 -138
  16. package/internals/utils/rendering-type-prediction.d.ts +34 -14
  17. package/internals/utils/rendering-type-prediction.js +87 -35
  18. package/package.json +19 -22
  19. package/index.d.ts.map +0 -1
  20. package/index.js.map +0 -1
  21. package/internals/adaptive-playwright-crawler.d.ts.map +0 -1
  22. package/internals/adaptive-playwright-crawler.js.map +0 -1
  23. package/internals/enqueue-links/click-elements.d.ts.map +0 -1
  24. package/internals/enqueue-links/click-elements.js.map +0 -1
  25. package/internals/playwright-crawler.d.ts.map +0 -1
  26. package/internals/playwright-crawler.js.map +0 -1
  27. package/internals/playwright-launcher.d.ts.map +0 -1
  28. package/internals/playwright-launcher.js.map +0 -1
  29. package/internals/utils/playwright-utils.d.ts.map +0 -1
  30. package/internals/utils/playwright-utils.js.map +0 -1
  31. package/internals/utils/rendering-type-prediction.d.ts.map +0 -1
  32. package/internals/utils/rendering-type-prediction.js.map +0 -1
@@ -1,4 +1,4 @@
1
- import type { GlobInput, PseudoUrlInput, RegExpInput, RequestProvider, RequestTransform, SkippedRequestCallback } from '@crawlee/browser';
1
+ import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
2
2
  import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
3
3
  // @ts-ignore optional peer dependency or compatibility with es2022
4
4
  import type { Page } from 'playwright';
@@ -9,9 +9,9 @@ export interface EnqueueLinksByClickingElementsOptions {
9
9
  */
10
10
  page: Page;
11
11
  /**
12
- * A request queue to which the URLs will be enqueued.
12
+ * * A request manager to which the URLs will be enqueued.
13
13
  */
14
- requestQueue: RequestProvider;
14
+ requestManager: IRequestManager;
15
15
  /**
16
16
  * A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
17
17
  * value. This is to prevent suboptimal use of this function by using it too broadly.
@@ -26,64 +26,29 @@ export interface EnqueueLinksByClickingElementsOptions {
26
26
  */
27
27
  clickOptions?: ClickOptions;
28
28
  /**
29
- * An array of glob pattern strings or plain objects
30
- * containing glob pattern strings matching the URLs to be enqueued.
29
+ * An array of URL patterns that URLs must match to be enqueued.
31
30
  *
32
- * The plain objects must include at least the `glob` property, which holds the glob pattern string.
33
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
34
- *
35
- * The matching is always case-insensitive.
36
- * If you need case-sensitive matching, use `regexps` property directly.
37
- *
38
- * If `globs` is an empty array or `undefined`, then the function
39
- * enqueues all the intercepted navigation requests produced by the page
40
- * after clicking on elements matching the provided CSS selector.
41
- */
42
- globs?: GlobInput[];
43
- /**
44
- * An array of glob pattern strings, regexp patterns or plain objects
45
- * containing patterns matching URLs that will **never** be enqueued.
46
- *
47
- * The plain objects must include either the `glob` property or the `regexp` property.
31
+ * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
48
32
  *
49
33
  * Glob matching is always case-insensitive.
50
- * If you need case-sensitive matching, provide a regexp.
51
- */
52
- exclude?: readonly (GlobInput | RegExpInput)[];
53
- /**
54
- * An array of regular expressions or plain objects
55
- * containing regular expressions matching the URLs to be enqueued.
56
- *
57
- * The plain objects must include at least the `regexp` property, which holds the regular expression.
58
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
34
+ * If you need case-sensitive matching, use a `RegExp`.
59
35
  *
60
- * If `regexps` is an empty array or `undefined`, then the function
36
+ * If `include` is an empty array or `undefined`, then the function
61
37
  * enqueues all the intercepted navigation requests produced by the page
62
38
  * after clicking on elements matching the provided CSS selector.
63
39
  */
64
- regexps?: RegExpInput[];
40
+ include?: UrlPatternInput[];
65
41
  /**
66
- * *NOTE:* In future versions of SDK the options will be removed.
67
- * Please use `globs` or `regexps` instead.
68
- *
69
- * An array of {@link PseudoUrl} strings or plain objects
70
- * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
71
- *
72
- * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
73
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
42
+ * An array of URL patterns. Matching URLs will **not** be enqueued.
74
43
  *
75
- * With a pseudo-URL string, the matching is always case-insensitive.
76
- * If you need case-sensitive matching, use `regexps` property directly.
44
+ * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
77
45
  *
78
- * If `pseudoUrls` is an empty array or `undefined`, then the function
79
- * enqueues all the intercepted navigation requests produced by the page
80
- * after clicking on elements matching the provided CSS selector.
81
- *
82
- * @deprecated prefer using `globs` or `regexps` instead
46
+ * Glob matching is always case-insensitive.
47
+ * If you need case-sensitive matching, use a `RegExp`.
83
48
  */
84
- pseudoUrls?: PseudoUrlInput[];
49
+ exclude?: readonly UrlPatternInput[];
85
50
  /**
86
- * After {@link Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
51
+ * After request options are filtered by `include`/`exclude` patterns,
87
52
  * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
88
53
  * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
89
54
  * but differ in methods or payloads, or to dynamically update or create `userData`.
@@ -98,8 +63,8 @@ export interface EnqueueLinksByClickingElementsOptions {
98
63
  * }
99
64
  * ```
100
65
  *
101
- * Note that `transformRequestFunction` has the highest priority and can overwrite request options
102
- * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
66
+ * Note that `transformRequestFunction` has the highest priority and can overwrite
67
+ * the global `label` option.
103
68
  *
104
69
  * The function receives a {@link RequestOptions} object and can return either:
105
70
  * - The modified {@link RequestOptions} object
@@ -160,8 +125,7 @@ export interface EnqueueLinksByClickingElementsOptions {
160
125
  * in `href` elements, but rather navigations are triggered in click handlers.
161
126
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
162
127
  *
163
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
164
- * and override settings of the enqueued {@link Request} objects.
128
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
165
129
  *
166
130
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
167
131
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -181,11 +145,11 @@ export interface EnqueueLinksByClickingElementsOptions {
181
145
  * ```javascript
182
146
  * await playwrightUtils.enqueueLinksByClickingElements({
183
147
  * page,
184
- * requestQueue,
148
+ * requestManager,
185
149
  * selector: 'a.product-detail',
186
- * pseudoUrls: [
187
- * 'https://www.example.com/handbags/[.*]'
188
- * 'https://www.example.com/purses/[.*]'
150
+ * include: [
151
+ * 'https://www.example.com/handbags/*',
152
+ * 'https://www.example.com/purses/*',
189
153
  * ],
190
154
  * });
191
155
  * ```
@@ -219,4 +183,3 @@ export declare function clickElementsAndInterceptNavigationRequests(options: Cli
219
183
  */
220
184
  export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
221
185
  export {};
222
- //# sourceMappingURL=click-elements.d.ts.map
@@ -1,9 +1,25 @@
1
1
  import { URL } from 'node:url';
2
- import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, Request as CrawleeRequest, } from '@crawlee/browser';
3
- import ow from 'ow';
4
- import log_ from '@apify/log';
2
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request as CrawleeRequest, serviceLocator, } from '@crawlee/browser';
3
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
5
5
  const STARTING_Z_INDEX = 2147400000;
6
- const log = log_.child({ prefix: 'Playwright Click Elements' });
6
+ const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
7
+ const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
8
+ page: schemas.objectWithKeys(['goto', 'evaluate']),
9
+ requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
10
+ selector: z.string(),
11
+ userData: schemas.anyObject.optional(),
12
+ clickOptions: schemas.anyObject.optional(),
13
+ include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
14
+ exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
15
+ transformRequestFunction: schemas.anyFunction.optional(),
16
+ waitForPageIdleSecs: schemas.anyNumber.default(1),
17
+ maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
18
+ label: z.string().optional(),
19
+ forefront: z.boolean().optional(),
20
+ skipNavigation: z.boolean().optional(),
21
+ onSkippedRequest: schemas.anyFunction.optional(),
22
+ });
7
23
  /**
8
24
  * The function finds elements matching a specific CSS selector in a Playwright page,
9
25
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -13,8 +29,7 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
13
29
  * in `href` elements, but rather navigations are triggered in click handlers.
14
30
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
15
31
  *
16
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
17
- * and override settings of the enqueued {@link Request} objects.
32
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
18
33
  *
19
34
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
20
35
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -34,11 +49,11 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
34
49
  * ```javascript
35
50
  * await playwrightUtils.enqueueLinksByClickingElements({
36
51
  * page,
37
- * requestQueue,
52
+ * requestManager,
38
53
  * selector: 'a.product-detail',
39
- * pseudoUrls: [
40
- * 'https://www.example.com/handbags/[.*]'
41
- * 'https://www.example.com/purses/[.*]'
54
+ * include: [
55
+ * 'https://www.example.com/handbags/*',
56
+ * 'https://www.example.com/purses/*',
42
57
  * ],
43
58
  * });
44
59
  * ```
@@ -46,49 +61,13 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
46
61
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
47
62
  */
48
63
  export async function enqueueLinksByClickingElements(options) {
49
- ow(options, ow.object.exactShape({
50
- page: ow.object.hasKeys('goto', 'evaluate'),
51
- requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
52
- selector: ow.string,
53
- userData: ow.optional.object,
54
- clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
55
- pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
56
- globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
57
- regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
58
- exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
59
- transformRequestFunction: ow.optional.function,
60
- waitForPageIdleSecs: ow.optional.number,
61
- maxWaitForPageIdleSecs: ow.optional.number,
62
- label: ow.optional.string,
63
- forefront: ow.optional.boolean,
64
- skipNavigation: ow.optional.boolean,
65
- onSkippedRequest: ow.optional.function,
66
- }));
67
- const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
64
+ const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
65
+ const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
68
66
  const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
69
67
  const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
70
- const urlExcludePatternObjects = [];
71
- const urlPatternObjects = [];
72
- if (exclude?.length) {
73
- for (const excl of exclude) {
74
- if (typeof excl === 'string' || 'glob' in excl) {
75
- urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
76
- }
77
- else if (excl instanceof RegExp || 'regexp' in excl) {
78
- urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
79
- }
80
- }
81
- }
82
- if (pseudoUrls?.length) {
83
- log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
84
- urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
85
- }
86
- if (globs?.length) {
87
- urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
88
- }
89
- if (regexps?.length) {
90
- urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
91
- }
68
+ const hasOnSkippedRequest = onSkippedRequest !== undefined;
69
+ const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
70
+ const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
92
71
  const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
93
72
  page,
94
73
  selector,
@@ -96,21 +75,21 @@ export async function enqueueLinksByClickingElements(options) {
96
75
  maxWaitForPageIdleMillis,
97
76
  clickOptions,
98
77
  });
99
- const requestOptions = createRequestOptions(interceptedRequests, options);
78
+ const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
100
79
  const skippedByFilters = [];
101
- let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
80
+ let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
102
81
  if (onSkippedRequest && skippedByFilters.length > 0) {
103
- await Promise.all(skippedByFilters.map((url) => onSkippedRequest({ url, reason: 'filters' })));
82
+ await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
104
83
  }
105
84
  if (transformRequestFunction) {
106
85
  const skippedByTransform = [];
107
- filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
86
+ filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
108
87
  if (onSkippedRequest && skippedByTransform.length > 0) {
109
- await Promise.all(skippedByTransform.map((r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
88
+ await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
110
89
  }
111
90
  }
112
91
  const requests = filteredOptions.map((opts) => new CrawleeRequest(opts));
113
- const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
92
+ const { addedRequests } = await requestManager.addRequestsBatched(requests, { forefront });
114
93
  return { processedRequests: addedRequests, unprocessedRequests: [] };
115
94
  }
116
95
  /**
@@ -136,7 +115,7 @@ export async function clickElementsAndInterceptNavigationRequests(options) {
136
115
  await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
137
116
  // browser.off(BrowserEmittedEvents.TargetCreated, onTargetCreated);
138
117
  page.off('framenavigated', onFrameNavigated);
139
- await context.unroute('*', onInterceptedRequest);
118
+ await context.unroute('**', onInterceptedRequest);
140
119
  const serializedRequests = Array.from(uniqueRequests);
141
120
  return serializedRequests.map((r) => JSON.parse(r));
142
121
  }
@@ -172,7 +151,9 @@ function createTargetCreatedHandler(requests) {
172
151
  await popup.close();
173
152
  }
174
153
  catch (err) {
175
- log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
154
+ getLog().debug('enqueueLinksByClickingElements: Could not close spawned page.', {
155
+ error: err.stack,
156
+ });
176
157
  }
177
158
  };
178
159
  }
@@ -180,7 +161,16 @@ function createTargetCreatedHandler(requests) {
180
161
  * @ignore
181
162
  */
182
163
  function isTopFrameNavigationRequest(page, req) {
183
- return req.isNavigationRequest() && req.frame() === page.mainFrame();
164
+ try {
165
+ return req.isNavigationRequest() && req.frame() === page.mainFrame();
166
+ }
167
+ catch {
168
+ // `req.frame()` throws when the owning frame is unavailable - e.g. the request was
169
+ // issued by a service worker, or before/after its frame existed (see #3216). Such a
170
+ // request is not a top-frame navigation, so swallow the throw and let it pass through
171
+ // instead of crashing the route handler (which would leave the route unhandled).
172
+ return false;
173
+ }
184
174
  }
185
175
  /**
186
176
  * @ignore
@@ -245,7 +235,7 @@ function updateElementCssToEnableMouseClick(el, zIndex) {
245
235
  */
246
236
  export async function clickElements(page, selector, clickOptions) {
247
237
  const elementHandles = await page.$$(selector);
248
- log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
238
+ getLog().debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
249
239
  let clickedElementsCount = 0;
250
240
  let zIndex = STARTING_Z_INDEX;
251
241
  let shouldLogWarning = true;
@@ -258,15 +248,15 @@ export async function clickElements(page, selector, clickOptions) {
258
248
  catch (err) {
259
249
  const e = err;
260
250
  if (shouldLogWarning && e.stack.includes('is detached from document')) {
261
- log.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
251
+ getLog().warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
262
252
  'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. ' +
263
253
  'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
264
254
  shouldLogWarning = false;
265
255
  }
266
- log.debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
256
+ getLog().debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
267
257
  }
268
258
  }
269
- log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
259
+ getLog().debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
270
260
  }
271
261
  /**
272
262
  * This function tracks whether any requests, frame navigations or targets were emitted
@@ -285,8 +275,6 @@ export async function clickElements(page, selector, clickOptions) {
285
275
  async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis, }) {
286
276
  return new Promise((resolve) => {
287
277
  let timeout;
288
- let maxTimeout;
289
- page.on('popup', activityHandler);
290
278
  function activityHandler() {
291
279
  clearTimeout(timeout);
292
280
  timeout = setTimeout(() => {
@@ -295,7 +283,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
295
283
  }, waitForPageIdleMillis);
296
284
  }
297
285
  function maxTimeoutHandler() {
298
- log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
286
+ getLog().debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
299
287
  'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
300
288
  finish();
301
289
  }
@@ -303,7 +291,8 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
303
291
  page.off('request', activityHandler).off('framenavigated', activityHandler).off('popup', activityHandler);
304
292
  resolve();
305
293
  }
306
- maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
294
+ const maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
295
+ page.on('popup', activityHandler);
307
296
  activityHandler(); // We call this once manually in case there would be no requests at all.
308
297
  page.on('request', activityHandler);
309
298
  page.on('framenavigated', activityHandler);
@@ -326,8 +315,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
326
315
  requests.add(JSON.stringify({ url }));
327
316
  }
328
317
  catch (err) {
329
- log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
318
+ getLog().debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
330
319
  }
331
320
  });
332
321
  }
333
- //# sourceMappingURL=click-elements.js.map
@@ -0,0 +1,55 @@
1
+ import type { Configuration, CrawlerRemoteBrowserOptions } from '@crawlee/browser';
2
+ import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PlaywrightPlugin } from '@crawlee/browser-pool';
3
+ // @ts-ignore optional peer dependency or compatibility with es2022
4
+ import type { Page } from 'playwright';
5
+ import type { PlaywrightLaunchContext } from './playwright-launcher.js';
6
+ /** {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`} for Playwright, with the protocol to connect over. */
7
+ export interface PlaywrightRemoteBrowserOptions extends CrawlerRemoteBrowserOptions {
8
+ /**
9
+ * `'cdp'` uses `connectOverCDP()` (the default), `'playwright'` uses `connect()` (Playwright's own WebSocket
10
+ * protocol, for endpoints started with `launchServer()`).
11
+ */
12
+ protocol?: 'cdp' | 'playwright';
13
+ }
14
+ /** A {@link BrowserPool} of Playwright browsers, as built by {@link playwrightBrowserPool}. */
15
+ export type PlaywrightBrowserPool = BrowserPool<{
16
+ browserPlugins: [PlaywrightPlugin];
17
+ }, [PlaywrightPlugin]>;
18
+ export interface PlaywrightBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PlaywrightPlugin['createController']>, ReturnType<PlaywrightPlugin['createLaunchContext']>, Page> {
19
+ /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
20
+ launchContext?: PlaywrightLaunchContext;
21
+ /**
22
+ * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
23
+ * Defaults to `true`, and can also be set via {@link Configuration}.
24
+ */
25
+ headless?: boolean;
26
+ /** Configuration to read the browser defaults from. Defaults to the global configuration. */
27
+ configuration?: Configuration;
28
+ /** Connect to a remote browser service instead of launching locally; see {@link BrowserCrawlerOptions.remoteBrowser}. */
29
+ remoteBrowser?: PlaywrightRemoteBrowserOptions;
30
+ }
31
+ /**
32
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
33
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
34
+ *
35
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
36
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
37
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
38
+ *
39
+ * **Example usage:**
40
+ *
41
+ * ```javascript
42
+ * const crawler = new PlaywrightCrawler({
43
+ * browserPool: playwrightBrowserPool({
44
+ * maxOpenPagesPerBrowser: 1,
45
+ * launchContext: { launcher: firefox },
46
+ * }),
47
+ * requestHandler: async ({ page }) => { ... },
48
+ * });
49
+ * ```
50
+ *
51
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
52
+ *
53
+ * @category Browser management
54
+ */
55
+ export declare function playwrightBrowserPool(options?: PlaywrightBrowserPoolOptions): PlaywrightBrowserPool;
@@ -0,0 +1,34 @@
1
+ import { PlaywrightLauncher } from './playwright-launcher.js';
2
+ /**
3
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
4
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
5
+ *
6
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
7
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
8
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
9
+ *
10
+ * **Example usage:**
11
+ *
12
+ * ```javascript
13
+ * const crawler = new PlaywrightCrawler({
14
+ * browserPool: playwrightBrowserPool({
15
+ * maxOpenPagesPerBrowser: 1,
16
+ * launchContext: { launcher: firefox },
17
+ * }),
18
+ * requestHandler: async ({ page }) => { ... },
19
+ * });
20
+ * ```
21
+ *
22
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
23
+ *
24
+ * @category Browser management
25
+ */
26
+ export function playwrightBrowserPool(options = {}) {
27
+ const { launchContext, headless, configuration, ...poolOptions } = options;
28
+ return playwrightLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
29
+ }
30
+ function playwrightLauncher(launchContext = {}, headless, configuration) {
31
+ return new PlaywrightLauncher(headless == null
32
+ ? launchContext
33
+ : { ...launchContext, launchOptions: { ...launchContext.launchOptions, headless } }, configuration);
34
+ }