@crawlee/browser 4.0.0-rc.0 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
34
34
 
35
35
  We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
36
36
 
37
- > Crawlee requires **Node.js 16 or higher**.
37
+ > Crawlee requires **Node.js 22.13 or higher**.
38
38
 
39
39
  ### With Crawlee CLI
40
40
 
@@ -1,9 +1,9 @@
1
- import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, Request, RequestHandler, RouterHandler } from '@crawlee/basic';
1
+ import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
2
2
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
- import type { CommonPage, CrawlerRemoteBrowserOptions } from '@crawlee/browser-pool';
3
+ import type { CommonPage } from '@crawlee/browser-pool';
4
4
  import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
5
5
  import { z } from 'zod';
6
- import type { BrowserLaunchContext } from './browser-launcher.js';
6
+ import type { CrawlerRemoteBrowserOptions } from './browser-launcher.js';
7
7
  interface BaseResponse {
8
8
  status(): number;
9
9
  /** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
@@ -11,18 +11,14 @@ interface BaseResponse {
11
11
  }
12
12
  /**
13
13
  * The type of a browser pool the crawler builds (and therefore owns) for itself. It's an {@link IBrowserPool} that
14
- * additionally exposes `destroy()` — the crawler only ever tears down pools it created, which is why {@link IBrowserPool}
15
- * itself intentionally omits `destroy`.
14
+ * additionally exposes the lifecycle hooks a crawler only ever calls on a pool it created — which is why
15
+ * {@link IBrowserPool} itself intentionally omits them: `releaseAllBrowsers()` at the end of every run, and
16
+ * `destroy()` once the crawler itself is destroyed.
16
17
  */
17
18
  export type OwnedBrowserPool<Page> = IBrowserPool<Page> & {
19
+ releaseAllBrowsers: () => Promise<void>;
18
20
  destroy: () => Promise<void>;
19
21
  };
20
- /**
21
- * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
22
- * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
23
- * into a second, half-working way of configuring the same pool.
24
- */
25
- export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;
26
22
  export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
27
23
  GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
28
24
  /**
@@ -32,7 +28,7 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
32
28
  /**
33
29
  * The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
34
30
  */
35
- request: LoadedRequest<Request<UserData>>;
31
+ request: LoadedRequest<CrawlingRequest<UserData>>;
36
32
  /**
37
33
  * The HTTP response object returned by the browser's navigation.
38
34
  */
@@ -52,8 +48,7 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
52
48
  enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
53
49
  }
54
50
  export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
55
- export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
56
- launchContext?: BrowserLaunchContext<any, any>;
51
+ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler' | 'contextPipelineBuilder'> {
57
52
  /**
58
53
  * The browser pool the crawler should serve its pages from. This is the single way to run a pool with
59
54
  * non-default options: build one with the factory that matches your crawler
@@ -71,14 +66,13 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
71
66
  /**
72
67
  * Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
73
68
  *
74
- * The crawler builds a {@link RemoteBrowserPool} around its own browser plugin, so the connection is
75
- * always for the right browser — there is no plugin to construct and no way to mismatch the pool with the
76
- * crawler. Supply the connection details only: a static `endpoint` URL, a function returning one per launch,
77
- * or a {@link RemoteBrowserProvider}.
69
+ * The crawler builds its pool around a remote variant of its own browser plugin, so the connection is always
70
+ * for the right browser. Supply the connection details only: a static `endpoint` URL, a function returning one
71
+ * per launch, or a {@link RemoteBrowserProvider}.
78
72
  *
79
- * Cannot be combined with `browserPool`. To tune the pool wrapping the remote connection, or to share it
80
- * across crawlers, build it with the remote factory for your crawler ({@link remotePlaywrightBrowserPool},
81
- * {@link remotePuppeteerBrowserPool}, {@link remoteStagehandBrowserPool}) and pass it as `browserPool`.
73
+ * Cannot be combined with `browserPool`. To tune the pool, or to share it across crawlers, pass the same
74
+ * `remoteBrowser` to the pool factory for your crawler ({@link playwrightBrowserPool},
75
+ * {@link puppeteerBrowserPool}) and pass the result as `browserPool`.
82
76
  */
83
77
  remoteBrowser?: CrawlerRemoteBrowserOptions;
84
78
  /**
@@ -105,7 +99,7 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
105
99
  * To make this work, we should **always**
106
100
  * let our function throw exceptions rather than catch them.
107
101
  * The exceptions are logged to the request using the
108
- * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
102
+ * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
109
103
  */
110
104
  requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
111
105
  /**
@@ -244,28 +238,19 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
244
238
  *
245
239
  * @category Crawlers
246
240
  */
247
- export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
241
+ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
248
242
  #private;
249
243
  /**
250
244
  * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
251
245
  * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
252
246
  */
253
247
  get browserPool(): IBrowserPool<Page>;
254
- launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
255
248
  protected readonly ignoreShadowRoots: boolean;
256
249
  protected readonly ignoreIframes: boolean;
250
+ /**
251
+ * @internal
252
+ */
257
253
  protected static optionsShape: {
258
- navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
259
- preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
260
- postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
261
- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
262
- browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
263
- browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
264
- remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
265
- saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
266
- proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
267
- ignoreIframes: z.ZodDefault<z.ZodBoolean>;
268
- ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
269
254
  contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
270
255
  extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
271
256
  requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -297,25 +282,23 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
297
282
  }, z.core.$strict>]>>;
298
283
  onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
299
284
  // @ts-ignore optional peer dependency or compatibility with es2022
300
- httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
285
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
301
286
  // @ts-ignore optional peer dependency or compatibility with es2022
302
- configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
287
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
303
288
  storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
304
289
  // @ts-ignore optional peer dependency or compatibility with es2022
305
- eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
290
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
306
291
  logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
307
292
  minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
308
293
  maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
294
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
309
295
  maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
310
296
  keepAlive: z.ZodOptional<z.ZodBoolean>;
311
297
  statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
312
298
  id: z.ZodOptional<z.ZodString>;
313
- };
314
- protected static optionsSchema: z.ZodObject<{
315
299
  navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
316
300
  preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
317
301
  postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
318
- launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
319
302
  browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
320
303
  browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
321
304
  remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
@@ -323,6 +306,9 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
323
306
  proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
324
307
  ignoreIframes: z.ZodDefault<z.ZodBoolean>;
325
308
  ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
309
+ };
310
+ /** @internal */
311
+ protected static optionsSchema: z.ZodObject<{
326
312
  contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
327
313
  extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
328
314
  requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -354,22 +340,34 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
354
340
  }, z.core.$strict>]>>;
355
341
  onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
356
342
  // @ts-ignore optional peer dependency or compatibility with es2022
357
- httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
343
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
358
344
  // @ts-ignore optional peer dependency or compatibility with es2022
359
- configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
345
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
360
346
  storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
361
347
  // @ts-ignore optional peer dependency or compatibility with es2022
362
- eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
348
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
363
349
  logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
364
350
  minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
365
351
  maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
352
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
366
353
  maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
367
354
  keepAlive: z.ZodOptional<z.ZodBoolean>;
368
355
  statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
369
356
  id: z.ZodOptional<z.ZodString>;
357
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
358
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
359
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
360
+ browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
361
+ browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
362
+ remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
363
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
364
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
365
+ ignoreIframes: z.ZodDefault<z.ZodBoolean>;
366
+ ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
370
367
  }, z.core.$strict>;
371
368
  /**
372
369
  * All `BrowserCrawler` parameters are passed via an options object.
370
+ * @internal
373
371
  */
374
372
  protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & {
375
373
  contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
@@ -380,41 +378,23 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
380
378
  */
381
379
  browserPoolBuilder: (remoteBrowser?: CrawlerRemoteBrowserOptions) => OwnedBrowserPool<Page>;
382
380
  });
381
+ /** @internal */
383
382
  protected getNavigationTimeoutMillis(): number;
384
383
  protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
385
- private containsSelectors;
386
- private isRequestBlocked;
387
- private preparePage;
388
- private prepareNavigation;
389
384
  private navigate;
390
- private finalizeNavigation;
391
- /**
392
- * Copies cookies from the live browser page into the session cookie jar.
393
- */
394
- private persistCookiesFromPage;
395
385
  /**
396
386
  * Runs the user request handler, then re-reads browser cookies so login flows /
397
387
  * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
388
+ * @internal
398
389
  */
399
390
  protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
400
- private handleBlockedRequestByContent;
401
- private restoreRequestState;
402
- private applyCookies;
403
- /**
404
- * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
405
- */
406
- private handleNavigationTimeout;
407
- /**
408
- * Transforms proxy-related errors to `SessionError`.
409
- */
410
- private throwIfProxyError;
411
391
  protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
412
- private processResponse;
413
392
  /**
414
- * Function for cleaning up after all requests are processed.
415
- * @ignore
393
+ * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
394
+ * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
416
395
  */
417
396
  teardown(): Promise<void>;
397
+ destroy(): Promise<void>;
418
398
  }
419
399
  /**
420
400
  * Extracts URLs from a given page.
@@ -1,22 +1,8 @@
1
- import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, parseArgument, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, schemas, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
2
- import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
1
+ import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, skippedNavigationRequest, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
2
+ import { assertBrowserPoolNotConfigured, AWS_WAF_RETRY_CSS_SELECTORS, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
3
3
  import { sleep } from '@crawlee/utils';
4
4
  import { z } from 'zod';
5
5
  import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
6
- /**
7
- * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
8
- * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
9
- * into a second, half-working way of configuring the same pool.
10
- */
11
- export function assertBrowserPoolNotConfigured(crawlerName, ignoredOptions) {
12
- const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);
13
- if (names.length === 0) {
14
- return;
15
- }
16
- throw new Error(`${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` +
17
- `${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` +
18
- 'itself. Configure the pool you pass in instead.');
19
- }
20
6
  const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
21
7
  const readContextField = (ctx, key) => ctx[key];
22
8
  /**
@@ -80,13 +66,15 @@ export class BrowserCrawler extends BasicCrawler {
80
66
  get browserPool() {
81
67
  return this.#browserPoolDep.value;
82
68
  }
83
- launchContext;
84
69
  ignoreShadowRoots;
85
70
  ignoreIframes;
86
71
  #navigationTimeoutMillis;
87
72
  #preNavigationHooks;
88
73
  #postNavigationHooks;
89
74
  #saveResponseCookies;
75
+ /**
76
+ * @internal
77
+ */
90
78
  static optionsShape = {
91
79
  ...BasicCrawler.optionsShape,
92
80
  navigationTimeoutSecs: schemas.anyNumber
@@ -94,7 +82,6 @@ export class BrowserCrawler extends BasicCrawler {
94
82
  .default(60),
95
83
  preNavigationHooks: schemas.anyArray.default(() => []),
96
84
  postNavigationHooks: schemas.anyArray.default(() => []),
97
- launchContext: schemas.anyObject.default(() => ({})),
98
85
  browserPool: validators.browserPool.optional(),
99
86
  browserPoolBuilder: schemas.anyFunction.optional(),
100
87
  remoteBrowser: schemas.anyObject.optional(),
@@ -103,18 +90,18 @@ export class BrowserCrawler extends BasicCrawler {
103
90
  ignoreIframes: z.boolean().default(false),
104
91
  ignoreShadowRoots: z.boolean().default(false),
105
92
  };
93
+ /** @internal */
106
94
  static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
107
95
  /**
108
96
  * All `BrowserCrawler` parameters are passed via an options object.
97
+ * @internal
109
98
  */
110
99
  constructor(options) {
111
- const { navigationTimeoutSecs, saveResponseCookies, launchContext, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
100
+ const { navigationTimeoutSecs, saveResponseCookies, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
112
101
  if (browserPool) {
113
102
  assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
114
103
  }
115
- const skipGuard = (action) => ({
116
- action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
117
- });
104
+ const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
118
105
  super({
119
106
  ...basicCrawlerOptions,
120
107
  contextPipelineBuilder: () => {
@@ -129,7 +116,7 @@ export class BrowserCrawler extends BasicCrawler {
129
116
  }
130
117
  return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
131
118
  });
132
- let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
119
+ let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));
133
120
  for (const hook of this.#preNavigationHooks) {
134
121
  pipeline = pipeline.compose(windowGuard(hook));
135
122
  }
@@ -138,13 +125,12 @@ export class BrowserCrawler extends BasicCrawler {
138
125
  pipeline = pipeline.compose(windowGuard(hook));
139
126
  }
140
127
  return pipeline
141
- .compose(skipGuard(this.finalizeNavigation.bind(this)))
142
- .compose({ action: this.handleBlockedRequestByContent.bind(this) })
143
- .compose({ action: this.restoreRequestState.bind(this) });
128
+ .compose(skipGuard(this.#finalizeNavigation.bind(this)))
129
+ .compose(this.#handleBlockedRequestByContent.bind(this))
130
+ .compose(this.#restoreRequestState.bind(this));
144
131
  },
145
132
  extendContext,
146
133
  });
147
- this.launchContext = launchContext;
148
134
  this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
149
135
  // The public option hooks are extension-aware; internal storage uses the base context type
150
136
  // (the pipeline composes hooks against the concrete context, which does not statically carry
@@ -156,43 +142,39 @@ export class BrowserCrawler extends BasicCrawler {
156
142
  this.#saveResponseCookies = saveResponseCookies;
157
143
  this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
158
144
  }
145
+ /** @internal */
159
146
  getNavigationTimeoutMillis() {
160
147
  return this.#navigationTimeoutMillis;
161
148
  }
162
149
  buildContextPipeline() {
163
- return ContextPipeline.create().compose({
164
- action: this.preparePage.bind(this),
165
- cleanup: async (context) => {
166
- context.registerDeferredCleanup(async () => {
167
- const error = !context.session.isUsable()
168
- ? new SessionError('Session is no longer usable')
169
- : undefined;
170
- await this.browserPool
171
- .closePage(context.page, { error })
172
- .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
173
- });
174
- },
175
- });
150
+ return ContextPipeline.create().compose(this.#preparePage.bind(this));
176
151
  }
177
- async containsSelectors(page, selectors) {
178
- const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
152
+ async #containsSelectors(page, selectors) {
153
+ const query = async () => Promise.all(selectors.map((selector) => page.$(selector)));
154
+ // a challenge page navigates itself once it is solved, which destroys the context of an in-flight query
155
+ const elements = await query().catch(async () => {
156
+ await sleep(1000);
157
+ return query();
158
+ });
159
+ const foundSelectors = elements
179
160
  .map((x, i) => [x, selectors[i]])
180
161
  .filter(([x]) => x !== null)
181
162
  .map(([, selector]) => selector);
182
163
  return foundSelectors.length > 0 ? foundSelectors : null;
183
164
  }
184
- async isRequestBlocked(crawlingContext) {
185
- const { page, response } = crawlingContext;
186
- // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
187
- if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
188
- await sleep(5000);
189
- // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
190
- const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
191
- if (!foundSelectors)
192
- return false;
193
- return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
165
+ async #detectSelfResolvingChallenge({ page, response, }) {
166
+ const statusCode = response?.status();
167
+ if (statusCode === 403 && (await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS))) {
168
+ return 'Cloudflare';
169
+ }
170
+ if (statusCode === 202 && (await this.#containsSelectors(page, AWS_WAF_RETRY_CSS_SELECTORS))) {
171
+ return 'AWS WAF';
194
172
  }
195
- const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
173
+ return null;
174
+ }
175
+ async #isRequestBlocked(crawlingContext) {
176
+ const { page, response } = crawlingContext;
177
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
196
178
  const statusCode = response?.status() ?? 0;
197
179
  if (foundSelectors)
198
180
  return `Found selectors: ${foundSelectors.join(', ')}`;
@@ -200,12 +182,34 @@ export class BrowserCrawler extends BasicCrawler {
200
182
  return `Received blocked status code: ${statusCode}`;
201
183
  return false;
202
184
  }
203
- async preparePage(crawlingContext) {
185
+ async #preparePage(crawlingContext, onCleanup) {
204
186
  const page = await this.browserPool.newPage({
205
187
  id: crawlingContext.id,
206
188
  session: crawlingContext.session,
207
189
  });
208
190
  tryCancel();
191
+ onCleanup((failure) => {
192
+ crawlingContext.registerDeferredCleanup(async () => {
193
+ const cause = failure instanceof RequestHandlerError || failure instanceof ContextPipelineInitializationError
194
+ ? failure.cause
195
+ : failure;
196
+ // Only tells the pool what to do with the page's browser state - the request's failure, if any, has
197
+ // already been handled. A thrown `SessionError` or a blocked session counts as a block; a session
198
+ // unusable for any other reason is reported as retired. `retire()`/`markBad()` also bump the usage
199
+ // count, so the block check must come first.
200
+ const { session } = crawlingContext;
201
+ const closeReason = cause instanceof SessionError
202
+ ? cause
203
+ : session.isBlocked()
204
+ ? new SessionError()
205
+ : session.isUsable()
206
+ ? undefined
207
+ : new SessionRetiredError();
208
+ await this.browserPool
209
+ .closePage(page, { error: closeReason })
210
+ .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
211
+ });
212
+ });
209
213
  const addRequests = crawlingContext.addRequests;
210
214
  const extractLinks = async (options) => {
211
215
  return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
@@ -235,17 +239,10 @@ export class BrowserCrawler extends BasicCrawler {
235
239
  },
236
240
  };
237
241
  }
238
- async prepareNavigation(crawlingContext) {
242
+ async #prepareNavigation(crawlingContext) {
239
243
  if (crawlingContext.request.skipNavigation) {
240
244
  return {
241
- request: new Proxy(crawlingContext.request, {
242
- get(target, propertyName, receiver) {
243
- if (propertyName === 'loadedUrl') {
244
- throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
245
- }
246
- return Reflect.get(target, propertyName, receiver);
247
- },
248
- }),
245
+ request: skippedNavigationRequest(crawlingContext.request),
249
246
  get response() {
250
247
  throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
251
248
  },
@@ -259,6 +256,7 @@ export class BrowserCrawler extends BasicCrawler {
259
256
  [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
260
257
  };
261
258
  }
259
+ // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
262
260
  async navigate(crawlingContext) {
263
261
  tryCancel();
264
262
  const gotoOptions = crawlingContext.gotoOptions;
@@ -276,22 +274,22 @@ export class BrowserCrawler extends BasicCrawler {
276
274
  }
277
275
  const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
278
276
  const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
279
- await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
277
+ await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
280
278
  let response;
281
279
  try {
282
280
  response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
283
281
  }
284
282
  catch (error) {
285
- await this.handleNavigationTimeout(crawlingContext, error);
283
+ await this.#handleNavigationTimeout(crawlingContext, error);
286
284
  crawlingContext.request.state = RequestState.ERROR;
287
- this.throwIfProxyError(error);
285
+ this.#throwIfProxyError(error);
288
286
  throw error;
289
287
  }
290
288
  tryCancel();
291
289
  crawlingContext.request.state = RequestState.AFTER_NAV;
292
290
  return { response };
293
291
  }
294
- async finalizeNavigation(crawlingContext) {
292
+ async #finalizeNavigation(crawlingContext) {
295
293
  tryCancel();
296
294
  let response;
297
295
  try {
@@ -301,17 +299,17 @@ export class BrowserCrawler extends BasicCrawler {
301
299
  // `preparePage` installs a throwing getter for `response`; reaching this branch means
302
300
  // navigation produced no response and no hook overrode it. Treat as undefined.
303
301
  }
304
- await this.processResponse(response, crawlingContext);
302
+ await this.#processResponse(response, crawlingContext);
305
303
  tryCancel();
306
304
  // Persist cookies from the navigation response before the user handler runs.
307
305
  // Cookies set during `requestHandler` are saved again afterwards.
308
- await this.persistCookiesFromPage(crawlingContext);
306
+ await this.#persistCookiesFromPage(crawlingContext);
309
307
  return { request: crawlingContext.request };
310
308
  }
311
309
  /**
312
310
  * Copies cookies from the live browser page into the session cookie jar.
313
311
  */
314
- async persistCookiesFromPage(crawlingContext) {
312
+ async #persistCookiesFromPage(crawlingContext) {
315
313
  if (!this.#saveResponseCookies || !crawlingContext.session) {
316
314
  return;
317
315
  }
@@ -333,6 +331,7 @@ export class BrowserCrawler extends BasicCrawler {
333
331
  /**
334
332
  * Runs the user request handler, then re-reads browser cookies so login flows /
335
333
  * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
334
+ * @internal
336
335
  */
337
336
  async runRequestHandler(crawlingContext) {
338
337
  try {
@@ -341,7 +340,7 @@ export class BrowserCrawler extends BasicCrawler {
341
340
  finally {
342
341
  if (!crawlingContext.request.skipNavigation) {
343
342
  try {
344
- await this.persistCookiesFromPage(crawlingContext);
343
+ await this.#persistCookiesFromPage(crawlingContext);
345
344
  }
346
345
  catch {
347
346
  // Page may already be closed on some failure paths; ignore.
@@ -349,19 +348,44 @@ export class BrowserCrawler extends BasicCrawler {
349
348
  }
350
349
  }
351
350
  }
352
- async handleBlockedRequestByContent(crawlingContext) {
353
- if (this.retryOnBlocked) {
354
- const error = await this.isRequestBlocked(crawlingContext);
355
- if (error)
356
- throw new SessionError(error);
351
+ async #handleBlockedRequestByContent(crawlingContext) {
352
+ if (!this.retryOnBlocked)
353
+ return {};
354
+ const page = crawlingContext.page;
355
+ // A passed challenge navigates to the actual page, whose response replaces the one of the challenge.
356
+ let navigationResponse;
357
+ const onResponse = (response) => {
358
+ if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
359
+ navigationResponse = response;
360
+ }
361
+ };
362
+ page.on('response', onResponse);
363
+ try {
364
+ // Cloudflare and AWS WAF serve challenges a browser can pass on its own, with a 403 and a 202 status respectively.
365
+ // Wait 5 seconds for the JS challenge to load / resolve before judging the page.
366
+ const challenge = await this.#detectSelfResolvingChallenge(crawlingContext);
367
+ if (challenge) {
368
+ await sleep(5000);
369
+ const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
370
+ if (foundSelectors) {
371
+ throw new SessionError(`${challenge} challenge failed, found selectors: ${foundSelectors.join(', ')}`);
372
+ }
373
+ return navigationResponse ? { response: navigationResponse } : {};
374
+ }
375
+ }
376
+ finally {
377
+ page.off('response', onResponse);
357
378
  }
379
+ const error = await this.#isRequestBlocked(crawlingContext);
380
+ if (error)
381
+ throw new SessionError(error);
358
382
  return {};
359
383
  }
360
- async restoreRequestState(crawlingContext) {
384
+ async #restoreRequestState(crawlingContext) {
361
385
  crawlingContext.request.state = RequestState.REQUEST_HANDLER;
362
386
  return {};
363
387
  }
364
- async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
388
+ async #applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
365
389
  const sessionCookie = session
366
390
  ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
367
391
  : [];
@@ -375,7 +399,7 @@ export class BrowserCrawler extends BasicCrawler {
375
399
  /**
376
400
  * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
377
401
  */
378
- async handleNavigationTimeout(crawlingContext, error) {
402
+ async #handleNavigationTimeout(crawlingContext, error) {
379
403
  const { session, page } = crawlingContext;
380
404
  // Fire-and-forget: no user code will run on this page after a failed navigation.
381
405
  // Swallow rejections: the page may already be detached.
@@ -390,12 +414,12 @@ export class BrowserCrawler extends BasicCrawler {
390
414
  /**
391
415
  * Transforms proxy-related errors to `SessionError`.
392
416
  */
393
- throwIfProxyError(error) {
417
+ #throwIfProxyError(error) {
394
418
  if (this.isProxyError(error)) {
395
419
  throw new SessionError(this.getMessageFromError(error));
396
420
  }
397
421
  }
398
- async processResponse(response, crawlingContext) {
422
+ async #processResponse(response, crawlingContext) {
399
423
  const { session, request, page } = crawlingContext;
400
424
  if (typeof response === 'object' && typeof response.status === 'function') {
401
425
  const status = response.status();
@@ -427,13 +451,17 @@ export class BrowserCrawler extends BasicCrawler {
427
451
  request.loadedUrl = await page.url();
428
452
  }
429
453
  /**
430
- * Function for cleaning up after all requests are processed.
431
- * @ignore
454
+ * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
455
+ * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
432
456
  */
433
457
  async teardown() {
434
- await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
458
+ await this.#browserPoolDep.ifOwned((pool) => pool.releaseAllBrowsers());
435
459
  await super.teardown();
436
460
  }
461
+ async destroy() {
462
+ await super.destroy();
463
+ await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
464
+ }
437
465
  }
438
466
  /**
439
467
  * Extracts URLs from a given page.
@@ -1,6 +1,5 @@
1
1
  import { Configuration } from '@crawlee/basic';
2
- import type { BrowserPlugin, BrowserPluginOptions, BrowserPoolHooks, BrowserPoolOptions, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
3
- import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
2
+ import { type BrowserPlugin, type BrowserPluginOptions, BrowserPool, type BrowserPoolHooks, type BrowserPoolOptions, type RemoteBrowserOptions } from '@crawlee/browser-pool';
4
3
  import type { Constructor, Dictionary } from '@crawlee/types';
5
4
  import { z } from 'zod';
6
5
  export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
@@ -72,18 +71,26 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
72
71
  * the browser they run against is only known to the concrete `*BrowserPool()` factory, which is where the
73
72
  * caller-facing types are pinned down.
74
73
  */
75
- export type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
74
+ type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
76
75
  [Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];
77
76
  };
78
77
  /**
79
- * The {@link RemoteBrowserPool} counterpart of {@link LauncherBrowserPoolOptions}.
78
+ * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option. Library-specific
79
+ * connect settings (e.g. Playwright's `protocol`) are added by the concrete crawler's options.
80
80
  */
81
- export type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins'>;
81
+ export interface CrawlerRemoteBrowserOptions extends RemoteBrowserOptions {
82
+ /**
83
+ * Maximum number of remote browsers open at once; see {@link BrowserPoolOptions.maxOpenBrowsers}. Defaults
84
+ * to the {@link RemoteBrowserProvider.maxOpenBrowsers|provider's value}, or `Infinity`.
85
+ */
86
+ maxOpenBrowsers?: number;
87
+ }
82
88
  /**
83
89
  * Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
84
90
  * @ignore
85
91
  */
86
92
  export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Launcher = Plugin['library'], T extends Constructor<Plugin> = Constructor<Plugin>, LaunchOptions extends Dictionary<any> | undefined = Partial<Parameters<Plugin['launch']>[0]>, LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>> {
93
+ #private;
87
94
  readonly configuration: Configuration;
88
95
  launcher: Launcher;
89
96
  proxyUrl?: string;
@@ -91,7 +98,11 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
91
98
  launchOptions: Dictionary;
92
99
  otherLaunchContextProps: Dictionary;
93
100
  Plugin: T;
101
+ RemotePlugin?: T;
94
102
  userAgent?: string;
103
+ /**
104
+ * @internal
105
+ */
95
106
  protected static optionsShape: {
96
107
  proxyUrl: z.ZodOptional<z.ZodURL>;
97
108
  useChrome: z.ZodOptional<z.ZodBoolean>;
@@ -102,6 +113,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
102
113
  launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
103
114
  userAgent: z.ZodOptional<z.ZodString>;
104
115
  };
116
+ /** @internal */
105
117
  protected static optionsSchema: z.ZodObject<{
106
118
  proxyUrl: z.ZodOptional<z.ZodURL>;
107
119
  useChrome: z.ZodOptional<z.ZodBoolean>;
@@ -120,27 +132,21 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
120
132
  /**
121
133
  * @ignore
122
134
  */
123
- createBrowserPlugin(): Plugin;
135
+ createBrowserPlugin(remoteBrowser?: CrawlerRemoteBrowserOptions): Plugin;
124
136
  /**
125
137
  * Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
126
138
  * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
127
139
  * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
140
+ * With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
128
141
  * @internal
129
142
  */
130
- createBrowserPool(options?: LauncherBrowserPoolOptions): BrowserPool<{
143
+ createBrowserPool({ remoteBrowser, ...options }?: LauncherBrowserPoolOptions & {
144
+ remoteBrowser?: CrawlerRemoteBrowserOptions;
145
+ }): BrowserPool<{
131
146
  browserPlugins: [Plugin];
132
- }, [Plugin]>;
133
- /**
134
- * The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
135
- * supplies the plugin, the caller supplies the remote connection details.
136
- * @internal
137
- */
138
- createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
139
- /**
140
- * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
141
- * explicitly requested user agent wins.
142
- */
143
- private resolveFingerprinting;
147
+ }, [
148
+ Plugin
149
+ ]>;
144
150
  /**
145
151
  * Launches a browser instance based on the plugin.
146
152
  * @returns Browser instance.
@@ -148,10 +154,5 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
148
154
  launch(): LaunchResult;
149
155
  createLaunchOptions(): Dictionary;
150
156
  protected getDefaultHeadlessOption(): boolean;
151
- private getChromeExecutablePath;
152
- /**
153
- * Gets a typical path to Chrome executable, depending on the current operating system.
154
- */
155
- private getTypicalChromeExecutablePath;
156
- private validateProxyUrlProtocol;
157
157
  }
158
+ export {};
@@ -1,8 +1,9 @@
1
1
  import fs from 'node:fs';
2
2
  import { createRequire } from 'node:module';
3
3
  import os from 'node:os';
4
- import { Configuration, schemas, serviceLocator } from '@crawlee/basic';
5
- import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
4
+ import { Configuration, serviceLocator } from '@crawlee/basic';
5
+ import { BrowserPool, RemoteBrowserProvider, } from '@crawlee/browser-pool';
6
+ import { schemas } from '@crawlee/utils/internal';
6
7
  import { z } from 'zod';
7
8
  const DEFAULT_VIEWPORT = {
8
9
  width: 1366,
@@ -20,9 +21,13 @@ export class BrowserLauncher {
20
21
  useChrome;
21
22
  launchOptions;
22
23
  otherLaunchContextProps;
23
- // to be provided by child classes;
24
+ // to be provided by child classes; `RemotePlugin` only by those supporting remote browsers
24
25
  Plugin;
26
+ RemotePlugin;
25
27
  userAgent;
28
+ /**
29
+ * @internal
30
+ */
26
31
  static optionsShape = {
27
32
  proxyUrl: z.url().optional(),
28
33
  useChrome: z.boolean().optional(),
@@ -33,6 +38,7 @@ export class BrowserLauncher {
33
38
  launchOptions: schemas.anyObject.optional(),
34
39
  userAgent: z.string().optional(),
35
40
  };
41
+ /** @internal */
36
42
  static optionsSchema = z.strictObject(BrowserLauncher.optionsShape);
37
43
  static requireLauncherOrThrow(launcher, apifyImageName) {
38
44
  try {
@@ -56,7 +62,7 @@ export class BrowserLauncher {
56
62
  constructor(launchContext, configuration = Configuration.getGlobalConfiguration()) {
57
63
  this.configuration = configuration;
58
64
  const { launcher, proxyUrl, useChrome, userAgent, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
59
- this.validateProxyUrlProtocol(proxyUrl);
65
+ this.#validateProxyUrlProtocol(proxyUrl);
60
66
  // those need to be reassigned otherwise they are {} in types
61
67
  this.launcher = launcher;
62
68
  this.proxyUrl = proxyUrl;
@@ -68,45 +74,46 @@ export class BrowserLauncher {
68
74
  /**
69
75
  * @ignore
70
76
  */
71
- createBrowserPlugin() {
72
- return new this.Plugin(this.launcher, {
77
+ createBrowserPlugin(remoteBrowser) {
78
+ const options = {
73
79
  proxyUrl: this.proxyUrl,
74
80
  launchOptions: this.createLaunchOptions(),
75
81
  ...this.otherLaunchContextProps,
76
- });
82
+ };
83
+ if (!remoteBrowser) {
84
+ return new this.Plugin(this.launcher, options);
85
+ }
86
+ if (!this.RemotePlugin) {
87
+ throw new Error(`${this.constructor.name} does not support remote browsers.`);
88
+ }
89
+ const { maxOpenBrowsers: _, ...remoteOptions } = remoteBrowser;
90
+ return new this.RemotePlugin(this.launcher, { ...options, ...remoteOptions });
77
91
  }
78
92
  /**
79
93
  * Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
80
94
  * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
81
95
  * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
96
+ * With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
82
97
  * @internal
83
98
  */
84
- createBrowserPool(options = {}) {
99
+ createBrowserPool({ remoteBrowser, ...options } = {}) {
100
+ const provider = remoteBrowser?.endpoint instanceof RemoteBrowserProvider ? remoteBrowser.endpoint : undefined;
101
+ const maxOpenBrowsers = options.maxOpenBrowsers ?? remoteBrowser?.maxOpenBrowsers ?? provider?.maxOpenBrowsers;
85
102
  // The hook types `BrowserPool` derives from `Plugin` are unresolvable while `Plugin` is still a free type
86
103
  // parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
87
104
  // caller-facing hook types get pinned down.
88
105
  return new BrowserPool({
89
- ...this.resolveFingerprinting(options),
90
- browserPlugins: [this.createBrowserPlugin()],
91
- });
92
- }
93
- /**
94
- * The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
95
- * supplies the plugin, the caller supplies the remote connection details.
96
- * @internal
97
- */
98
- createRemoteBrowserPool(options) {
99
- return new RemoteBrowserPool({
100
- ...options,
101
- browserPlugins: [this.createBrowserPlugin()],
102
- browserPoolOptions: this.resolveFingerprinting(options.browserPoolOptions ?? {}),
106
+ ...this.#resolveFingerprinting(options),
107
+ ...(maxOpenBrowsers !== undefined && { maxOpenBrowsers }),
108
+ browserPlugins: [this.createBrowserPlugin(remoteBrowser)],
103
109
  });
104
110
  }
105
111
  /**
112
+
106
113
  * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
107
114
  * explicitly requested user agent wins.
108
115
  */
109
- resolveFingerprinting(options) {
116
+ #resolveFingerprinting(options) {
110
117
  if (!this.userAgent) {
111
118
  return options;
112
119
  }
@@ -143,20 +150,20 @@ export class BrowserLauncher {
143
150
  launchOptions.headless = this.getDefaultHeadlessOption();
144
151
  }
145
152
  if (this.useChrome && !launchOptions.executablePath) {
146
- launchOptions.executablePath = this.getChromeExecutablePath();
153
+ launchOptions.executablePath = this.#getChromeExecutablePath();
147
154
  }
148
155
  return launchOptions;
149
156
  }
150
157
  getDefaultHeadlessOption() {
151
158
  return this.configuration.headless && !this.configuration.xvfb;
152
159
  }
153
- getChromeExecutablePath() {
154
- return this.configuration.chromeExecutablePath ?? this.getTypicalChromeExecutablePath();
160
+ #getChromeExecutablePath() {
161
+ return this.configuration.chromeExecutablePath ?? this.#getTypicalChromeExecutablePath();
155
162
  }
156
163
  /**
157
164
  * Gets a typical path to Chrome executable, depending on the current operating system.
158
165
  */
159
- getTypicalChromeExecutablePath() {
166
+ #getTypicalChromeExecutablePath() {
160
167
  /**
161
168
  * Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.
162
169
  * Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).
@@ -184,7 +191,7 @@ export class BrowserLauncher {
184
191
  return '/usr/bin/google-chrome';
185
192
  }
186
193
  }
187
- validateProxyUrlProtocol(proxyUrl) {
194
+ #validateProxyUrlProtocol(proxyUrl) {
188
195
  if (!proxyUrl)
189
196
  return;
190
197
  if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {
package/package.json CHANGED
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "name": "@crawlee/browser",
3
- "version": "4.0.0-rc.0",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
- "node": ">=22.0.0"
6
+ "node": ">=22.13.0"
7
7
  },
8
8
  "type": "module",
9
9
  "exports": {
@@ -47,14 +47,13 @@
47
47
  "access": "public"
48
48
  },
49
49
  "dependencies": {
50
- "@apify/timeout": "^0.4.4",
51
- "@crawlee/basic": "4.0.0-rc.0",
52
- "@crawlee/browser-pool": "4.0.0-rc.0",
53
- "@crawlee/types": "4.0.0-rc.0",
54
- "@crawlee/utils": "4.0.0-rc.0",
55
- "tslib": "^2.8.1",
56
- "type-fest": "^4.41.0",
57
- "zod": "^4.4.3"
50
+ "@apify/timeout": "^1.0.1",
51
+ "@crawlee/basic": "4.0.0-rc.2",
52
+ "@crawlee/browser-pool": "4.0.0-rc.2",
53
+ "@crawlee/types": "4.0.0-rc.2",
54
+ "@crawlee/utils": "4.0.0-rc.2",
55
+ "tslib": "^2.9.1",
56
+ "zod": "^4.6.7"
58
57
  },
59
58
  "peerDependencies": {
60
59
  "playwright": "*",
@@ -75,5 +74,5 @@
75
74
  }
76
75
  }
77
76
  },
78
- "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
77
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
79
78
  }