@crawlee/playwright 4.0.0-beta.13 → 4.0.0-beta.131

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +17 -13
  2. package/index.d.ts +2 -2
  3. package/index.js +1 -1
  4. package/internals/adaptive-playwright-crawler.d.ts +128 -59
  5. package/internals/adaptive-playwright-crawler.js +317 -230
  6. package/internals/enqueue-links/click-elements.d.ts +37 -55
  7. package/internals/enqueue-links/click-elements.js +63 -55
  8. package/internals/playwright-browser-pool.d.ts +71 -0
  9. package/internals/playwright-browser-pool.js +61 -0
  10. package/internals/playwright-crawler.d.ts +193 -124
  11. package/internals/playwright-crawler.js +63 -62
  12. package/internals/playwright-launcher.d.ts +28 -18
  13. package/internals/playwright-launcher.js +19 -18
  14. package/internals/utils/playwright-utils.d.ts +61 -24
  15. package/internals/utils/playwright-utils.js +145 -95
  16. package/internals/utils/rendering-type-prediction.d.ts +28 -13
  17. package/internals/utils/rendering-type-prediction.js +87 -29
  18. package/package.json +18 -14
  19. package/index.d.ts.map +0 -1
  20. package/index.js.map +0 -1
  21. package/internals/adaptive-playwright-crawler.d.ts.map +0 -1
  22. package/internals/adaptive-playwright-crawler.js.map +0 -1
  23. package/internals/enqueue-links/click-elements.d.ts.map +0 -1
  24. package/internals/enqueue-links/click-elements.js.map +0 -1
  25. package/internals/playwright-crawler.d.ts.map +0 -1
  26. package/internals/playwright-crawler.js.map +0 -1
  27. package/internals/playwright-launcher.d.ts.map +0 -1
  28. package/internals/playwright-launcher.js.map +0 -1
  29. package/internals/utils/playwright-utils.d.ts.map +0 -1
  30. package/internals/utils/playwright-utils.js.map +0 -1
  31. package/internals/utils/rendering-type-prediction.d.ts.map +0 -1
  32. package/internals/utils/rendering-type-prediction.js.map +0 -1
  33. package/tsconfig.build.tsbuildinfo +0 -1
@@ -1,4 +1,4 @@
1
- import type { GlobInput, PseudoUrlInput, RegExpInput, RequestProvider, RequestTransform } from '@crawlee/browser';
1
+ import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
2
2
  import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
3
3
  // @ts-ignore optional peer dependency or compatibility with es2022
4
4
  import type { Page } from 'playwright';
@@ -9,9 +9,9 @@ export interface EnqueueLinksByClickingElementsOptions {
9
9
  */
10
10
  page: Page;
11
11
  /**
12
- * A request queue to which the URLs will be enqueued.
12
+ * * A request manager to which the URLs will be enqueued.
13
13
  */
14
- requestQueue: RequestProvider;
14
+ requestManager: IRequestManager;
15
15
  /**
16
16
  * A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
17
17
  * value. This is to prevent suboptimal use of this function by using it too broadly.
@@ -26,72 +26,50 @@ export interface EnqueueLinksByClickingElementsOptions {
26
26
  */
27
27
  clickOptions?: ClickOptions;
28
28
  /**
29
- * An array of glob pattern strings or plain objects
30
- * containing glob pattern strings matching the URLs to be enqueued.
29
+ * An array of URL patterns that URLs must match to be enqueued.
31
30
  *
32
- * The plain objects must include at least the `glob` property, which holds the glob pattern string.
33
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
31
+ * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
34
32
  *
35
- * The matching is always case-insensitive.
36
- * If you need case-sensitive matching, use `regexps` property directly.
33
+ * Glob matching is always case-insensitive.
34
+ * If you need case-sensitive matching, use a `RegExp`.
37
35
  *
38
- * If `globs` is an empty array or `undefined`, then the function
36
+ * If `include` is an empty array or `undefined`, then the function
39
37
  * enqueues all the intercepted navigation requests produced by the page
40
38
  * after clicking on elements matching the provided CSS selector.
41
39
  */
42
- globs?: GlobInput[];
40
+ include?: UrlPatternInput[];
43
41
  /**
44
- * An array of regular expressions or plain objects
45
- * containing regular expressions matching the URLs to be enqueued.
42
+ * An array of URL patterns. Matching URLs will **not** be enqueued.
46
43
  *
47
- * The plain objects must include at least the `regexp` property, which holds the regular expression.
48
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
44
+ * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
49
45
  *
50
- * If `regexps` is an empty array or `undefined`, then the function
51
- * enqueues all the intercepted navigation requests produced by the page
52
- * after clicking on elements matching the provided CSS selector.
53
- */
54
- regexps?: RegExpInput[];
55
- /**
56
- * *NOTE:* In future versions of SDK the options will be removed.
57
- * Please use `globs` or `regexps` instead.
58
- *
59
- * An array of {@link PseudoUrl} strings or plain objects
60
- * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
61
- *
62
- * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
63
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
64
- *
65
- * With a pseudo-URL string, the matching is always case-insensitive.
66
- * If you need case-sensitive matching, use `regexps` property directly.
67
- *
68
- * If `pseudoUrls` is an empty array or `undefined`, then the function
69
- * enqueues all the intercepted navigation requests produced by the page
70
- * after clicking on elements matching the provided CSS selector.
71
- *
72
- * @deprecated prefer using `globs` or `regexps` instead
46
+ * Glob matching is always case-insensitive.
47
+ * If you need case-sensitive matching, use a `RegExp`.
73
48
  */
74
- pseudoUrls?: PseudoUrlInput[];
49
+ exclude?: readonly UrlPatternInput[];
75
50
  /**
76
- * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
77
- * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
78
- * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
79
- * or to dynamically update or create `userData`.
80
- *
81
- * For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
82
- * a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
83
- * (POST requests).
51
+ * After request options are filtered by `include`/`exclude` patterns,
52
+ * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
53
+ * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
54
+ * but differ in methods or payloads, or to dynamically update or create `userData`.
84
55
  *
85
56
  * **Example:**
86
57
  * ```javascript
87
58
  * {
88
59
  * transformRequestFunction: (request) => {
89
60
  * request.userData.foo = 'bar';
90
- * request.useExtendedUniqueKey = true;
91
61
  * return request;
92
62
  * }
93
63
  * }
94
64
  * ```
65
+ *
66
+ * Note that `transformRequestFunction` has the highest priority and can overwrite
67
+ * the global `label` option.
68
+ *
69
+ * The function receives a {@link RequestOptions} object and can return either:
70
+ * - The modified {@link RequestOptions} object
71
+ * - `'unchanged'` to keep the original options as-is
72
+ * - A falsy value or `'skip'` to exclude the request from the queue
95
73
  */
96
74
  transformRequestFunction?: RequestTransform;
97
75
  /**
@@ -131,6 +109,12 @@ export interface EnqueueLinksByClickingElementsOptions {
131
109
  * @default false
132
110
  */
133
111
  skipNavigation?: boolean;
112
+ /**
113
+ * When a request is skipped for some reason, you can use this callback to act on it.
114
+ * This is fired for requests skipped because they don't match enqueueLinks filters
115
+ * or because they were removed by `transformRequestFunction`.
116
+ */
117
+ onSkippedRequest?: SkippedRequestCallback;
134
118
  }
135
119
  /**
136
120
  * The function finds elements matching a specific CSS selector in a Playwright page,
@@ -141,8 +125,7 @@ export interface EnqueueLinksByClickingElementsOptions {
141
125
  * in `href` elements, but rather navigations are triggered in click handlers.
142
126
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
143
127
  *
144
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
145
- * and override settings of the enqueued {@link Request} objects.
128
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
146
129
  *
147
130
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
148
131
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -162,11 +145,11 @@ export interface EnqueueLinksByClickingElementsOptions {
162
145
  * ```javascript
163
146
  * await playwrightUtils.enqueueLinksByClickingElements({
164
147
  * page,
165
- * requestQueue,
148
+ * requestManager,
166
149
  * selector: 'a.product-detail',
167
- * pseudoUrls: [
168
- * 'https://www.example.com/handbags/[.*]'
169
- * 'https://www.example.com/purses/[.*]'
150
+ * include: [
151
+ * 'https://www.example.com/handbags/*',
152
+ * 'https://www.example.com/purses/*',
170
153
  * ],
171
154
  * });
172
155
  * ```
@@ -200,4 +183,3 @@ export declare function clickElementsAndInterceptNavigationRequests(options: Cli
200
183
  */
201
184
  export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
202
185
  export {};
203
- //# sourceMappingURL=click-elements.d.ts.map
@@ -1,9 +1,24 @@
1
1
  import { URL } from 'node:url';
2
- import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, } from '@crawlee/browser';
3
- import ow from 'ow';
4
- import log_ from '@apify/log';
2
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, parseArgument, urlPatternSchema, Request as CrawleeRequest, schemas, serviceLocator, } from '@crawlee/browser';
3
+ import { z } from 'zod';
5
4
  const STARTING_Z_INDEX = 2147400000;
6
- const log = log_.child({ prefix: 'Playwright Click Elements' });
5
+ const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
6
+ const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
7
+ page: schemas.objectWithKeys(['goto', 'evaluate']),
8
+ requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
9
+ selector: z.string(),
10
+ userData: schemas.anyObject.optional(),
11
+ clickOptions: schemas.anyObject.optional(),
12
+ include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
13
+ exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
14
+ transformRequestFunction: schemas.anyFunction.optional(),
15
+ waitForPageIdleSecs: schemas.anyNumber.default(1),
16
+ maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
17
+ label: z.string().optional(),
18
+ forefront: z.boolean().optional(),
19
+ skipNavigation: z.boolean().optional(),
20
+ onSkippedRequest: schemas.anyFunction.optional(),
21
+ });
7
22
  /**
8
23
  * The function finds elements matching a specific CSS selector in a Playwright page,
9
24
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -13,8 +28,7 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
13
28
  * in `href` elements, but rather navigations are triggered in click handlers.
14
29
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
15
30
  *
16
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
17
- * and override settings of the enqueued {@link Request} objects.
31
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
18
32
  *
19
33
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
20
34
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -34,11 +48,11 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
34
48
  * ```javascript
35
49
  * await playwrightUtils.enqueueLinksByClickingElements({
36
50
  * page,
37
- * requestQueue,
51
+ * requestManager,
38
52
  * selector: 'a.product-detail',
39
- * pseudoUrls: [
40
- * 'https://www.example.com/handbags/[.*]'
41
- * 'https://www.example.com/purses/[.*]'
53
+ * include: [
54
+ * 'https://www.example.com/handbags/*',
55
+ * 'https://www.example.com/purses/*',
42
56
  * ],
43
57
  * });
44
58
  * ```
@@ -46,36 +60,12 @@ const log = log_.child({ prefix: 'Playwright Click Elements' });
46
60
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
47
61
  */
48
62
  export async function enqueueLinksByClickingElements(options) {
49
- ow(options, ow.object.exactShape({
50
- page: ow.object.hasKeys('goto', 'evaluate'),
51
- requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
52
- selector: ow.string,
53
- userData: ow.optional.object,
54
- clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
55
- pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
56
- globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
57
- regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
58
- transformRequestFunction: ow.optional.function,
59
- waitForPageIdleSecs: ow.optional.number,
60
- maxWaitForPageIdleSecs: ow.optional.number,
61
- label: ow.optional.string,
62
- forefront: ow.optional.boolean,
63
- skipNavigation: ow.optional.boolean,
64
- }));
65
- const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, } = options;
63
+ const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
64
+ const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
66
65
  const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
67
66
  const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
68
- const urlPatternObjects = [];
69
- if (pseudoUrls?.length) {
70
- log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
71
- urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
72
- }
73
- if (globs?.length) {
74
- urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
75
- }
76
- if (regexps?.length) {
77
- urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
78
- }
67
+ const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
68
+ const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
79
69
  const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
80
70
  page,
81
71
  selector,
@@ -83,12 +73,21 @@ export async function enqueueLinksByClickingElements(options) {
83
73
  maxWaitForPageIdleMillis,
84
74
  clickOptions,
85
75
  });
86
- let requestOptions = createRequestOptions(interceptedRequests, options);
76
+ const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
77
+ const skippedByFilters = [];
78
+ let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
79
+ if (onSkippedRequest && skippedByFilters.length > 0) {
80
+ await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
81
+ }
87
82
  if (transformRequestFunction) {
88
- requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
83
+ const skippedByTransform = [];
84
+ filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
85
+ if (onSkippedRequest && skippedByTransform.length > 0) {
86
+ await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
87
+ }
89
88
  }
90
- const requests = createRequests(requestOptions, urlPatternObjects);
91
- const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
89
+ const requests = filteredOptions.map((opts) => new CrawleeRequest(opts));
90
+ const { addedRequests } = await requestManager.addRequestsBatched(requests, { forefront });
92
91
  return { processedRequests: addedRequests, unprocessedRequests: [] };
93
92
  }
94
93
  /**
@@ -114,7 +113,7 @@ export async function clickElementsAndInterceptNavigationRequests(options) {
114
113
  await restoreHistoryNavigationAndSaveCapturedUrls(page, uniqueRequests);
115
114
  // browser.off(BrowserEmittedEvents.TargetCreated, onTargetCreated);
116
115
  page.off('framenavigated', onFrameNavigated);
117
- await context.unroute('*', onInterceptedRequest);
116
+ await context.unroute('**', onInterceptedRequest);
118
117
  const serializedRequests = Array.from(uniqueRequests);
119
118
  return serializedRequests.map((r) => JSON.parse(r));
120
119
  }
@@ -150,7 +149,9 @@ function createTargetCreatedHandler(requests) {
150
149
  await popup.close();
151
150
  }
152
151
  catch (err) {
153
- log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
152
+ getLog().debug('enqueueLinksByClickingElements: Could not close spawned page.', {
153
+ error: err.stack,
154
+ });
154
155
  }
155
156
  };
156
157
  }
@@ -158,7 +159,16 @@ function createTargetCreatedHandler(requests) {
158
159
  * @ignore
159
160
  */
160
161
  function isTopFrameNavigationRequest(page, req) {
161
- return req.isNavigationRequest() && req.frame() === page.mainFrame();
162
+ try {
163
+ return req.isNavigationRequest() && req.frame() === page.mainFrame();
164
+ }
165
+ catch {
166
+ // `req.frame()` throws when the owning frame is unavailable - e.g. the request was
167
+ // issued by a service worker, or before/after its frame existed (see #3216). Such a
168
+ // request is not a top-frame navigation, so swallow the throw and let it pass through
169
+ // instead of crashing the route handler (which would leave the route unhandled).
170
+ return false;
171
+ }
162
172
  }
163
173
  /**
164
174
  * @ignore
@@ -223,7 +233,7 @@ function updateElementCssToEnableMouseClick(el, zIndex) {
223
233
  */
224
234
  export async function clickElements(page, selector, clickOptions) {
225
235
  const elementHandles = await page.$$(selector);
226
- log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
236
+ getLog().debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
227
237
  let clickedElementsCount = 0;
228
238
  let zIndex = STARTING_Z_INDEX;
229
239
  let shouldLogWarning = true;
@@ -236,15 +246,15 @@ export async function clickElements(page, selector, clickOptions) {
236
246
  catch (err) {
237
247
  const e = err;
238
248
  if (shouldLogWarning && e.stack.includes('is detached from document')) {
239
- log.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
249
+ getLog().warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
240
250
  'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. ' +
241
251
  'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
242
252
  shouldLogWarning = false;
243
253
  }
244
- log.debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
254
+ getLog().debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
245
255
  }
246
256
  }
247
- log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
257
+ getLog().debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
248
258
  }
249
259
  /**
250
260
  * This function tracks whether any requests, frame navigations or targets were emitted
@@ -263,8 +273,6 @@ export async function clickElements(page, selector, clickOptions) {
263
273
  async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis, }) {
264
274
  return new Promise((resolve) => {
265
275
  let timeout;
266
- let maxTimeout;
267
- page.on('popup', activityHandler);
268
276
  function activityHandler() {
269
277
  clearTimeout(timeout);
270
278
  timeout = setTimeout(() => {
@@ -273,7 +281,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
273
281
  }, waitForPageIdleMillis);
274
282
  }
275
283
  function maxTimeoutHandler() {
276
- log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
284
+ getLog().debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
277
285
  'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
278
286
  finish();
279
287
  }
@@ -281,7 +289,8 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
281
289
  page.off('request', activityHandler).off('framenavigated', activityHandler).off('popup', activityHandler);
282
290
  resolve();
283
291
  }
284
- maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
292
+ const maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
293
+ page.on('popup', activityHandler);
285
294
  activityHandler(); // We call this once manually in case there would be no requests at all.
286
295
  page.on('request', activityHandler);
287
296
  page.on('framenavigated', activityHandler);
@@ -304,8 +313,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
304
313
  requests.add(JSON.stringify({ url }));
305
314
  }
306
315
  catch (err) {
307
- log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
316
+ getLog().debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
308
317
  }
309
318
  });
310
319
  }
311
- //# sourceMappingURL=click-elements.js.map
@@ -0,0 +1,71 @@
1
+ import type { Configuration } from '@crawlee/browser';
2
+ import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PlaywrightPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
3
+ // @ts-ignore optional peer dependency or compatibility with es2022
4
+ import type { Page } from 'playwright';
5
+ import type { PlaywrightLaunchContext } from './playwright-launcher.js';
6
+ /** A {@link BrowserPool} of Playwright browsers, as built by {@link playwrightBrowserPool}. */
7
+ export type PlaywrightBrowserPool = BrowserPool<{
8
+ browserPlugins: [PlaywrightPlugin];
9
+ }, [PlaywrightPlugin]>;
10
+ export interface PlaywrightBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PlaywrightPlugin['createController']>, ReturnType<PlaywrightPlugin['createLaunchContext']>, Page> {
11
+ /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
12
+ launchContext?: PlaywrightLaunchContext;
13
+ /**
14
+ * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
15
+ * Defaults to `true`, and can also be set via {@link Configuration}.
16
+ */
17
+ headless?: boolean;
18
+ /** Configuration to read the browser defaults from. Defaults to the global configuration. */
19
+ configuration?: Configuration;
20
+ }
21
+ export interface RemotePlaywrightBrowserPoolOptions extends Pick<PlaywrightBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
22
+ }
23
+ /**
24
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
25
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
26
+ *
27
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
28
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
29
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
30
+ *
31
+ * **Example usage:**
32
+ *
33
+ * ```javascript
34
+ * const crawler = new PlaywrightCrawler({
35
+ * browserPool: playwrightBrowserPool({
36
+ * maxOpenPagesPerBrowser: 1,
37
+ * launchContext: { launcher: firefox },
38
+ * }),
39
+ * requestHandler: async ({ page }) => { ... },
40
+ * });
41
+ * ```
42
+ *
43
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
44
+ *
45
+ * @category Browser management
46
+ */
47
+ export declare function playwrightBrowserPool(options?: PlaywrightBrowserPoolOptions): PlaywrightBrowserPool;
48
+ /**
49
+ * The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
50
+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
51
+ *
52
+ * A {@link PlaywrightCrawler} accepts the same connection details directly via
53
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
54
+ * tune the wrapping pool, or to share one remote pool between crawlers.
55
+ *
56
+ * **Example usage:**
57
+ *
58
+ * ```javascript
59
+ * const crawler = new PlaywrightCrawler({
60
+ * browserPool: remotePlaywrightBrowserPool({
61
+ * endpoint: 'wss://production-sfo.browserless.io?token=xxx',
62
+ * maxOpenBrowsers: 2,
63
+ * browserPoolOptions: { useFingerprints: false },
64
+ * }),
65
+ * requestHandler: async ({ page }) => { ... },
66
+ * });
67
+ * ```
68
+ *
69
+ * @category Browser management
70
+ */
71
+ export declare function remotePlaywrightBrowserPool(options: RemotePlaywrightBrowserPoolOptions): RemoteBrowserPool<Page>;
@@ -0,0 +1,61 @@
1
+ import { PlaywrightLauncher } from './playwright-launcher.js';
2
+ /**
3
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
4
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
5
+ *
6
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
7
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
8
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
9
+ *
10
+ * **Example usage:**
11
+ *
12
+ * ```javascript
13
+ * const crawler = new PlaywrightCrawler({
14
+ * browserPool: playwrightBrowserPool({
15
+ * maxOpenPagesPerBrowser: 1,
16
+ * launchContext: { launcher: firefox },
17
+ * }),
18
+ * requestHandler: async ({ page }) => { ... },
19
+ * });
20
+ * ```
21
+ *
22
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
23
+ *
24
+ * @category Browser management
25
+ */
26
+ export function playwrightBrowserPool(options = {}) {
27
+ const { launchContext, headless, configuration, ...poolOptions } = options;
28
+ return playwrightLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
29
+ }
30
+ /**
31
+ * The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
32
+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
33
+ *
34
+ * A {@link PlaywrightCrawler} accepts the same connection details directly via
35
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
36
+ * tune the wrapping pool, or to share one remote pool between crawlers.
37
+ *
38
+ * **Example usage:**
39
+ *
40
+ * ```javascript
41
+ * const crawler = new PlaywrightCrawler({
42
+ * browserPool: remotePlaywrightBrowserPool({
43
+ * endpoint: 'wss://production-sfo.browserless.io?token=xxx',
44
+ * maxOpenBrowsers: 2,
45
+ * browserPoolOptions: { useFingerprints: false },
46
+ * }),
47
+ * requestHandler: async ({ page }) => { ... },
48
+ * });
49
+ * ```
50
+ *
51
+ * @category Browser management
52
+ */
53
+ export function remotePlaywrightBrowserPool(options) {
54
+ const { launchContext, headless, configuration, ...remoteOptions } = options;
55
+ return playwrightLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
56
+ }
57
+ function playwrightLauncher(launchContext = {}, headless, configuration) {
58
+ return new PlaywrightLauncher(headless == null
59
+ ? launchContext
60
+ : { ...launchContext, launchOptions: { ...launchContext.launchOptions, headless } }, configuration);
61
+ }