@crawlee/puppeteer 4.0.0-beta.99 → 4.0.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
package/index.d.ts CHANGED
@@ -1,9 +1,8 @@
1
1
  export * from '@crawlee/browser';
2
+ export * from './internals/puppeteer-browser-pool.js';
2
3
  export * from './internals/puppeteer-crawler.js';
3
4
  export * from './internals/puppeteer-launcher.js';
4
- export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
5
5
  export type { InterceptHandler } from './internals/utils/puppeteer_request_interception.js';
6
6
  export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
7
- export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils.js';
8
- export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
7
+ export type { DirectNavigationOptions as PuppeteerDirectNavigationOptions } from './internals/utils/puppeteer_utils.js';
9
8
  export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
package/index.js CHANGED
@@ -1,6 +1,5 @@
1
1
  export * from '@crawlee/browser';
2
+ export * from './internals/puppeteer-browser-pool.js';
2
3
  export * from './internals/puppeteer-crawler.js';
3
4
  export * from './internals/puppeteer-launcher.js';
4
- export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
5
5
  export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
6
- export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
@@ -1,7 +1,7 @@
1
- import type { GlobInput, IRequestManager, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from '@crawlee/browser';
1
+ import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
2
2
  import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
3
3
  // @ts-ignore optional peer dependency or compatibility with es2022
4
- import type { ClickOptions, Page, Target } from 'puppeteer';
4
+ import type { ClickOptions, Page } from 'puppeteer';
5
5
  export interface EnqueueLinksByClickingElementsOptions {
6
6
  /**
7
7
  * Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
@@ -25,64 +25,29 @@ export interface EnqueueLinksByClickingElementsOptions {
25
25
  */
26
26
  clickOptions?: ClickOptions;
27
27
  /**
28
- * An array of glob pattern strings or plain objects
29
- * containing glob pattern strings matching the URLs to be enqueued.
28
+ * An array of URL patterns that URLs must match to be enqueued.
30
29
  *
31
- * The plain objects must include at least the `glob` property, which holds the glob pattern string.
32
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
33
- *
34
- * The matching is always case-insensitive.
35
- * If you need case-sensitive matching, use `regexps` property directly.
36
- *
37
- * If `globs` is an empty array or `undefined`, then the function
38
- * enqueues all the intercepted navigation requests produced by the page
39
- * after clicking on elements matching the provided CSS selector.
40
- */
41
- globs?: GlobInput[];
42
- /**
43
- * An array of glob pattern strings, regexp patterns or plain objects
44
- * containing patterns matching URLs that will **never** be enqueued.
45
- *
46
- * The plain objects must include either the `glob` property or the `regexp` property.
30
+ * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
47
31
  *
48
32
  * Glob matching is always case-insensitive.
49
- * If you need case-sensitive matching, provide a regexp.
50
- */
51
- exclude?: readonly (GlobInput | RegExpInput)[];
52
- /**
53
- * An array of regular expressions or plain objects
54
- * containing regular expressions matching the URLs to be enqueued.
55
- *
56
- * The plain objects must include at least the `regexp` property, which holds the regular expression.
57
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
33
+ * If you need case-sensitive matching, use a `RegExp`.
58
34
  *
59
- * If `regexps` is an empty array or `undefined`, then the function
35
+ * If `include` is an empty array or `undefined`, then the function
60
36
  * enqueues all the intercepted navigation requests produced by the page
61
37
  * after clicking on elements matching the provided CSS selector.
62
38
  */
63
- regexps?: RegExpInput[];
39
+ include?: UrlPatternInput[];
64
40
  /**
65
- * *NOTE:* In future versions of SDK the options will be removed.
66
- * Please use `globs` or `regexps` instead.
41
+ * An array of URL patterns. Matching URLs will **not** be enqueued.
67
42
  *
68
- * An array of {@link PseudoUrl} strings or plain objects
69
- * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
43
+ * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
70
44
  *
71
- * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
72
- * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
73
- *
74
- * With a pseudo-URL string, the matching is always case-insensitive.
75
- * If you need case-sensitive matching, use `regexps` property directly.
76
- *
77
- * If `pseudoUrls` is an empty array or `undefined`, then the function
78
- * enqueues all the intercepted navigation requests produced by the page
79
- * after clicking on elements matching the provided CSS selector.
80
- *
81
- * @deprecated prefer using `globs` or `regexps` instead
45
+ * Glob matching is always case-insensitive.
46
+ * If you need case-sensitive matching, use a `RegExp`.
82
47
  */
83
- pseudoUrls?: PseudoUrlInput[];
48
+ exclude?: readonly UrlPatternInput[];
84
49
  /**
85
- * After {@link Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
50
+ * After request options are filtered by `include`/`exclude` patterns,
86
51
  * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
87
52
  * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
88
53
  * but differ in methods or payloads, or to dynamically update or create `userData`.
@@ -97,8 +62,8 @@ export interface EnqueueLinksByClickingElementsOptions {
97
62
  * }
98
63
  * ```
99
64
  *
100
- * Note that `transformRequestFunction` has the highest priority and can overwrite request options
101
- * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
65
+ * Note that `transformRequestFunction` has the highest priority and can overwrite
66
+ * the global `label` option.
102
67
  *
103
68
  * The function receives a {@link RequestOptions} object and can return either:
104
69
  * - The modified {@link RequestOptions} object
@@ -159,8 +124,7 @@ export interface EnqueueLinksByClickingElementsOptions {
159
124
  * in `href` elements, but rather navigations are triggered in click handlers.
160
125
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
161
126
  *
162
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
163
- * and override settings of the enqueued {@link Request} objects.
127
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
164
128
  *
165
129
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
166
130
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -178,13 +142,13 @@ export interface EnqueueLinksByClickingElementsOptions {
178
142
  * **Example usage**
179
143
  *
180
144
  * ```javascript
181
- * await utils.puppeteer.enqueueLinksByClickingElements({
145
+ * await puppeteerUtils.enqueueLinksByClickingElements({
182
146
  * page,
183
147
  * requestManager,
184
148
  * selector: 'a.product-detail',
185
- * pseudoUrls: [
186
- * 'https://www.example.com/handbags/[.*]'
187
- * 'https://www.example.com/purses/[.*]'
149
+ * include: [
150
+ * 'https://www.example.com/handbags/*',
151
+ * 'https://www.example.com/purses/*',
188
152
  * ],
189
153
  * });
190
154
  * ```
@@ -205,20 +169,19 @@ interface ClickElementsAndInterceptNavigationRequestsOptions extends WaitForPage
205
169
  * Clicks all elements of given page matching given selector.
206
170
  * Catches and intercepts all initiated navigation requests and opened pages.
207
171
  * Returns a list of all target URLs.
172
+ *
173
+ * Not part of the public API — exported only so tests can import this module directly.
208
174
  * @ignore
209
175
  */
210
176
  export declare function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
211
- /**
212
- * We're only interested in pages created by the page we're currently clicking in.
213
- * There will generally be a lot of other targets being created in the browser.
214
- */
215
- export declare function isTargetRelevant(page: Page, target: Target): boolean;
216
177
  /**
217
178
  * Click all elements matching the given selector. To be able to do this using
218
179
  * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
219
180
  * so we first move them to the top of the page's stacking context and then click.
220
181
  * We do all in series to prevent elements from hiding one another. Therefore,
221
182
  * for large element sets, this will take considerable amount of time.
183
+ *
184
+ * Not part of the public API — exported only so tests can import this module directly.
222
185
  * @ignore
223
186
  */
224
187
  export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
@@ -1,9 +1,26 @@
1
1
  import { URL } from 'node:url';
2
- import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, Request, serviceLocator, } from '@crawlee/browser';
3
- import ow from 'ow';
2
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request, serviceLocator, } from '@crawlee/browser';
3
+ import { parseArgument, schemas } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
4
5
  import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
5
6
  const STARTING_Z_INDEX = 2147400000;
6
7
  const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
8
+ const enqueueLinksByClickingElementsOptionsSchema = z.strictObject({
9
+ page: schemas.objectWithKeys(['goto', 'evaluate']),
10
+ requestManager: schemas.objectWithKeys(['fetchNextRequest', 'addRequestsBatched']),
11
+ selector: z.string(),
12
+ userData: schemas.anyObject.optional(),
13
+ clickOptions: schemas.anyObject.optional(),
14
+ include: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
15
+ exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
16
+ transformRequestFunction: schemas.anyFunction.optional(),
17
+ waitForPageIdleSecs: schemas.anyNumber.default(1),
18
+ maxWaitForPageIdleSecs: schemas.anyNumber.default(5),
19
+ label: z.string().optional(),
20
+ forefront: z.boolean().optional(),
21
+ skipNavigation: z.boolean().optional(),
22
+ onSkippedRequest: schemas.anyFunction.optional(),
23
+ });
7
24
  /**
8
25
  * The function finds elements matching a specific CSS selector in a Puppeteer page,
9
26
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -13,8 +30,7 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
13
30
  * in `href` elements, but rather navigations are triggered in click handlers.
14
31
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
15
32
  *
16
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
17
- * and override settings of the enqueued {@link Request} objects.
33
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
18
34
  *
19
35
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
20
36
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -32,13 +48,13 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
32
48
  * **Example usage**
33
49
  *
34
50
  * ```javascript
35
- * await utils.puppeteer.enqueueLinksByClickingElements({
51
+ * await puppeteerUtils.enqueueLinksByClickingElements({
36
52
  * page,
37
53
  * requestManager,
38
54
  * selector: 'a.product-detail',
39
- * pseudoUrls: [
40
- * 'https://www.example.com/handbags/[.*]'
41
- * 'https://www.example.com/purses/[.*]'
55
+ * include: [
56
+ * 'https://www.example.com/handbags/*',
57
+ * 'https://www.example.com/purses/*',
42
58
  * ],
43
59
  * });
44
60
  * ```
@@ -46,51 +62,13 @@ const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
46
62
  * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
47
63
  */
48
64
  export async function enqueueLinksByClickingElements(options) {
49
- ow(options, ow.object.exactShape({
50
- page: ow.object.hasKeys('goto', 'evaluate'),
51
- requestManager: ow.object.hasKeys('fetchNextRequest', 'addRequestsBatched'),
52
- selector: ow.string,
53
- userData: ow.optional.object,
54
- clickOptions: ow.optional.object,
55
- pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
56
- globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
57
- regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
58
- exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
59
- transformRequestFunction: ow.optional.function,
60
- waitForPageIdleSecs: ow.optional.number,
61
- maxWaitForPageIdleSecs: ow.optional.number,
62
- label: ow.optional.string,
63
- forefront: ow.optional.boolean,
64
- skipNavigation: ow.optional.boolean,
65
- onSkippedRequest: ow.optional.function,
66
- }));
67
- const { page, requestManager, selector, clickOptions,
68
- // oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
69
- pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
65
+ const parsedOptions = parseArgument(options, enqueueLinksByClickingElementsOptionsSchema, 'EnqueueLinksByClickingElementsOptions');
66
+ const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
70
67
  const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
71
68
  const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
72
- const urlExcludePatternObjects = [];
73
- const urlPatternObjects = [];
74
- if (exclude?.length) {
75
- for (const excl of exclude) {
76
- if (typeof excl === 'string' || 'glob' in excl) {
77
- urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
78
- }
79
- else if (excl instanceof RegExp || 'regexp' in excl) {
80
- urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
81
- }
82
- }
83
- }
84
- if (pseudoUrls?.length) {
85
- getLog().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
86
- urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
87
- }
88
- if (globs?.length) {
89
- urlPatternObjects.push(...constructGlobObjectsFromGlobs(globs));
90
- }
91
- if (regexps?.length) {
92
- urlPatternObjects.push(...constructRegExpObjectsFromRegExps(regexps));
93
- }
69
+ const hasOnSkippedRequest = onSkippedRequest !== undefined;
70
+ const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
71
+ const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
94
72
  const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
95
73
  page,
96
74
  selector,
@@ -98,17 +76,17 @@ export async function enqueueLinksByClickingElements(options) {
98
76
  maxWaitForPageIdleMillis,
99
77
  clickOptions,
100
78
  });
101
- const requestOptions = createRequestOptions(interceptedRequests, options);
79
+ const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
102
80
  const skippedByFilters = [];
103
- let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
81
+ let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
104
82
  if (onSkippedRequest && skippedByFilters.length > 0) {
105
- await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
83
+ await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
106
84
  }
107
85
  if (transformRequestFunction) {
108
86
  const skippedByTransform = [];
109
- filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
87
+ filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
110
88
  if (onSkippedRequest && skippedByTransform.length > 0) {
111
- await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
89
+ await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
112
90
  }
113
91
  }
114
92
  const requests = filteredOptions.map((opts) => new Request(opts));
@@ -119,6 +97,8 @@ export async function enqueueLinksByClickingElements(options) {
119
97
  * Clicks all elements of given page matching given selector.
120
98
  * Catches and intercepts all initiated navigation requests and opened pages.
121
99
  * Returns a list of all target URLs.
100
+ *
101
+ * Not part of the public API — exported only so tests can import this module directly.
122
102
  * @ignore
123
103
  */
124
104
  export async function clickElementsAndInterceptNavigationRequests(options) {
@@ -197,7 +177,7 @@ function createTargetCreatedHandler(page, requests) {
197
177
  * We're only interested in pages created by the page we're currently clicking in.
198
178
  * There will generally be a lot of other targets being created in the browser.
199
179
  */
200
- export function isTargetRelevant(page, target) {
180
+ function isTargetRelevant(page, target) {
201
181
  // oxlint-disable-next-line typescript/no-deprecated -- the non-deprecated replacement (opener.page()) is async and would force every call site to await, including EventEmitter callbacks
202
182
  return target.type() === 'page' && page.target() === target.opener();
203
183
  }
@@ -242,6 +222,8 @@ async function preventHistoryNavigation(page) {
242
222
  * so we first move them to the top of the page's stacking context and then click.
243
223
  * We do all in series to prevent elements from hiding one another. Therefore,
244
224
  * for large element sets, this will take considerable amount of time.
225
+ *
226
+ * Not part of the public API — exported only so tests can import this module directly.
245
227
  * @ignore
246
228
  */
247
229
  export async function clickElements(page, selector, clickOptions) {
@@ -0,0 +1,55 @@
1
+ import type { Configuration } from '@crawlee/browser';
2
+ import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PuppeteerPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
3
+ // @ts-ignore optional peer dependency or compatibility with es2022
4
+ import type { Page } from 'puppeteer';
5
+ import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
6
+ /** A {@link BrowserPool} of Puppeteer browsers, as built by {@link puppeteerBrowserPool}. */
7
+ export type PuppeteerBrowserPool = BrowserPool<{
8
+ browserPlugins: [PuppeteerPlugin];
9
+ }, [PuppeteerPlugin]>;
10
+ export interface PuppeteerBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PuppeteerPlugin['createController']>, ReturnType<PuppeteerPlugin['createLaunchContext']>, Page> {
11
+ /** How to launch the browser: proxy, user data dir, whether to use full Chrome, ... */
12
+ launchContext?: PuppeteerLaunchContext;
13
+ /**
14
+ * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
15
+ * Defaults to `true`, and can also be set via {@link Configuration}.
16
+ */
17
+ headless?: boolean | 'new' | 'old';
18
+ /** Configuration to read the browser defaults from. Defaults to the global configuration. */
19
+ configuration?: Configuration;
20
+ }
21
+ export interface RemotePuppeteerBrowserPoolOptions extends Pick<PuppeteerBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
22
+ }
23
+ /**
24
+ * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
25
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
26
+ *
27
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
28
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
29
+ * given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
30
+ *
31
+ * **Example usage:**
32
+ *
33
+ * ```javascript
34
+ * const crawler = new PuppeteerCrawler({
35
+ * browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
36
+ * requestHandler: async ({ page }) => { ... },
37
+ * });
38
+ * ```
39
+ *
40
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
41
+ *
42
+ * @category Browser management
43
+ */
44
+ export declare function puppeteerBrowserPool(options?: PuppeteerBrowserPoolOptions): PuppeteerBrowserPool;
45
+ /**
46
+ * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
47
+ * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
48
+ *
49
+ * A {@link PuppeteerCrawler} accepts the same connection details directly via
50
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
51
+ * tune the wrapping pool, or to share one remote pool between crawlers.
52
+ *
53
+ * @category Browser management
54
+ */
55
+ export declare function remotePuppeteerBrowserPool(options: RemotePuppeteerBrowserPoolOptions): RemoteBrowserPool<Page>;
@@ -0,0 +1,48 @@
1
+ import { PuppeteerLauncher } from './puppeteer-launcher.js';
2
+ /**
3
+ * Builds a {@link BrowserPool} of Puppeteer browsers to pass to a {@link PuppeteerCrawler} as its
4
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
5
+ *
6
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
7
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
8
+ * given to, and configuring one never means assembling a {@link PuppeteerPlugin} by hand.
9
+ *
10
+ * **Example usage:**
11
+ *
12
+ * ```javascript
13
+ * const crawler = new PuppeteerCrawler({
14
+ * browserPool: puppeteerBrowserPool({ maxOpenPagesPerBrowser: 1 }),
15
+ * requestHandler: async ({ page }) => { ... },
16
+ * });
17
+ * ```
18
+ *
19
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
20
+ *
21
+ * @category Browser management
22
+ */
23
+ export function puppeteerBrowserPool(options = {}) {
24
+ const { launchContext, headless, configuration, ...poolOptions } = options;
25
+ return puppeteerLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
26
+ }
27
+ /**
28
+ * The {@link RemoteBrowserPool} counterpart of {@link puppeteerBrowserPool}: connects to a remote browser
29
+ * service (Browserbase, Browserless, Steel, ...) with a Puppeteer plugin derived from `launchContext`.
30
+ *
31
+ * A {@link PuppeteerCrawler} accepts the same connection details directly via
32
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
33
+ * tune the wrapping pool, or to share one remote pool between crawlers.
34
+ *
35
+ * @category Browser management
36
+ */
37
+ export function remotePuppeteerBrowserPool(options) {
38
+ const { launchContext, headless, configuration, ...remoteOptions } = options;
39
+ return puppeteerLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
40
+ }
41
+ function puppeteerLauncher(launchContext = {}, headless, configuration) {
42
+ return new PuppeteerLauncher(headless == null
43
+ ? launchContext
44
+ : {
45
+ ...launchContext,
46
+ launchOptions: { ...launchContext.launchOptions, headless: headless },
47
+ }, configuration);
48
+ }