@crawlee/browser 4.0.0-rc.0 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/internals/browser-crawler.d.ts +47 -67
- package/internals/browser-crawler.js +112 -84
- package/internals/browser-launcher.d.ts +26 -25
- package/internals/browser-launcher.js +35 -28
- package/package.json +10 -11
package/README.md
CHANGED
|
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
34
34
|
|
|
35
35
|
We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
|
|
36
36
|
|
|
37
|
-
> Crawlee requires **Node.js
|
|
37
|
+
> Crawlee requires **Node.js 22.13 or higher**.
|
|
38
38
|
|
|
39
39
|
### With Crawlee CLI
|
|
40
40
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest,
|
|
1
|
+
import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequestHandler, RouterHandler } from '@crawlee/basic';
|
|
2
2
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
3
|
-
import type { CommonPage
|
|
3
|
+
import type { CommonPage } from '@crawlee/browser-pool';
|
|
4
4
|
import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
|
|
5
5
|
import { z } from 'zod';
|
|
6
|
-
import type {
|
|
6
|
+
import type { CrawlerRemoteBrowserOptions } from './browser-launcher.js';
|
|
7
7
|
interface BaseResponse {
|
|
8
8
|
status(): number;
|
|
9
9
|
/** Optional because only Playwright and Puppeteer responses are guaranteed to carry it. */
|
|
@@ -11,18 +11,14 @@ interface BaseResponse {
|
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
13
13
|
* The type of a browser pool the crawler builds (and therefore owns) for itself. It's an {@link IBrowserPool} that
|
|
14
|
-
* additionally exposes
|
|
15
|
-
* itself intentionally omits `
|
|
14
|
+
* additionally exposes the lifecycle hooks a crawler only ever calls on a pool it created — which is why
|
|
15
|
+
* {@link IBrowserPool} itself intentionally omits them: `releaseAllBrowsers()` at the end of every run, and
|
|
16
|
+
* `destroy()` once the crawler itself is destroyed.
|
|
16
17
|
*/
|
|
17
18
|
export type OwnedBrowserPool<Page> = IBrowserPool<Page> & {
|
|
19
|
+
releaseAllBrowsers: () => Promise<void>;
|
|
18
20
|
destroy: () => Promise<void>;
|
|
19
21
|
};
|
|
20
|
-
/**
|
|
21
|
-
* Rejects options that exist only to configure the browser pool the crawler would have built for itself.
|
|
22
|
-
* Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
|
|
23
|
-
* into a second, half-working way of configuring the same pool.
|
|
24
|
-
*/
|
|
25
|
-
export declare function assertBrowserPoolNotConfigured(crawlerName: string, ignoredOptions: Dictionary): void;
|
|
26
22
|
export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
27
23
|
GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
28
24
|
/**
|
|
@@ -32,7 +28,7 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
|
32
28
|
/**
|
|
33
29
|
* The request object that was successfully loaded and navigated to, including the {@link Request.loadedUrl|`loadedUrl`} property.
|
|
34
30
|
*/
|
|
35
|
-
request: LoadedRequest<
|
|
31
|
+
request: LoadedRequest<CrawlingRequest<UserData>>;
|
|
36
32
|
/**
|
|
37
33
|
* The HTTP response object returned by the browser's navigation.
|
|
38
34
|
*/
|
|
@@ -52,8 +48,7 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
|
52
48
|
enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
|
|
53
49
|
}
|
|
54
50
|
export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
|
|
55
|
-
export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
|
|
56
|
-
launchContext?: BrowserLaunchContext<any, any>;
|
|
51
|
+
export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler' | 'contextPipelineBuilder'> {
|
|
57
52
|
/**
|
|
58
53
|
* The browser pool the crawler should serve its pages from. This is the single way to run a pool with
|
|
59
54
|
* non-default options: build one with the factory that matches your crawler
|
|
@@ -71,14 +66,13 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
71
66
|
/**
|
|
72
67
|
* Connect to a remote browser service (Browserbase, Browserless, Steel, …) instead of launching locally.
|
|
73
68
|
*
|
|
74
|
-
* The crawler builds a
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
* or a {@link RemoteBrowserProvider}.
|
|
69
|
+
* The crawler builds its pool around a remote variant of its own browser plugin, so the connection is always
|
|
70
|
+
* for the right browser. Supply the connection details only: a static `endpoint` URL, a function returning one
|
|
71
|
+
* per launch, or a {@link RemoteBrowserProvider}.
|
|
78
72
|
*
|
|
79
|
-
* Cannot be combined with `browserPool`. To tune the pool
|
|
80
|
-
*
|
|
81
|
-
* {@link
|
|
73
|
+
* Cannot be combined with `browserPool`. To tune the pool, or to share it across crawlers, pass the same
|
|
74
|
+
* `remoteBrowser` to the pool factory for your crawler ({@link playwrightBrowserPool},
|
|
75
|
+
* {@link puppeteerBrowserPool}) and pass the result as `browserPool`.
|
|
82
76
|
*/
|
|
83
77
|
remoteBrowser?: CrawlerRemoteBrowserOptions;
|
|
84
78
|
/**
|
|
@@ -105,7 +99,7 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
105
99
|
* To make this work, we should **always**
|
|
106
100
|
* let our function throw exceptions rather than catch them.
|
|
107
101
|
* The exceptions are logged to the request using the
|
|
108
|
-
* {@link
|
|
102
|
+
* {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
|
|
109
103
|
*/
|
|
110
104
|
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
|
|
111
105
|
/**
|
|
@@ -244,28 +238,19 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
244
238
|
*
|
|
245
239
|
* @category Crawlers
|
|
246
240
|
*/
|
|
247
|
-
export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse,
|
|
241
|
+
export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
248
242
|
#private;
|
|
249
243
|
/**
|
|
250
244
|
* A reference to the underlying browser pool that manages the crawler's browsers. Typed as
|
|
251
245
|
* {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
|
|
252
246
|
*/
|
|
253
247
|
get browserPool(): IBrowserPool<Page>;
|
|
254
|
-
launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
|
|
255
248
|
protected readonly ignoreShadowRoots: boolean;
|
|
256
249
|
protected readonly ignoreIframes: boolean;
|
|
250
|
+
/**
|
|
251
|
+
* @internal
|
|
252
|
+
*/
|
|
257
253
|
protected static optionsShape: {
|
|
258
|
-
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
259
|
-
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
260
|
-
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
261
|
-
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
262
|
-
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
263
|
-
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
264
|
-
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
265
|
-
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
266
|
-
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
267
|
-
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
268
|
-
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
269
254
|
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
270
255
|
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
271
256
|
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -297,25 +282,23 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
|
|
|
297
282
|
}, z.core.$strict>]>>;
|
|
298
283
|
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
299
284
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
300
|
-
httpClient: z.ZodOptional<z.
|
|
285
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
301
286
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
302
|
-
configuration: z.ZodOptional<z.
|
|
287
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
303
288
|
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
304
289
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
305
|
-
eventManager: z.ZodOptional<z.
|
|
290
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
306
291
|
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
307
292
|
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
308
293
|
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
294
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
309
295
|
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
310
296
|
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
311
297
|
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
312
298
|
id: z.ZodOptional<z.ZodString>;
|
|
313
|
-
};
|
|
314
|
-
protected static optionsSchema: z.ZodObject<{
|
|
315
299
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
316
300
|
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
317
301
|
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
318
|
-
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
319
302
|
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
320
303
|
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
321
304
|
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
@@ -323,6 +306,9 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
|
|
|
323
306
|
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
324
307
|
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
325
308
|
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
309
|
+
};
|
|
310
|
+
/** @internal */
|
|
311
|
+
protected static optionsSchema: z.ZodObject<{
|
|
326
312
|
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
327
313
|
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
328
314
|
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
@@ -354,22 +340,34 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
|
|
|
354
340
|
}, z.core.$strict>]>>;
|
|
355
341
|
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
356
342
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
357
|
-
httpClient: z.ZodOptional<z.
|
|
343
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
358
344
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
359
|
-
configuration: z.ZodOptional<z.
|
|
345
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
360
346
|
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
361
347
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
362
|
-
eventManager: z.ZodOptional<z.
|
|
348
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
363
349
|
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
364
350
|
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
365
351
|
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
352
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
366
353
|
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
367
354
|
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
368
355
|
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
369
356
|
id: z.ZodOptional<z.ZodString>;
|
|
357
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
358
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
359
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
360
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
361
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
362
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
363
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
364
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
365
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
366
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
370
367
|
}, z.core.$strict>;
|
|
371
368
|
/**
|
|
372
369
|
* All `BrowserCrawler` parameters are passed via an options object.
|
|
370
|
+
* @internal
|
|
373
371
|
*/
|
|
374
372
|
protected constructor(options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & {
|
|
375
373
|
contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
|
|
@@ -380,41 +378,23 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
|
|
|
380
378
|
*/
|
|
381
379
|
browserPoolBuilder: (remoteBrowser?: CrawlerRemoteBrowserOptions) => OwnedBrowserPool<Page>;
|
|
382
380
|
});
|
|
381
|
+
/** @internal */
|
|
383
382
|
protected getNavigationTimeoutMillis(): number;
|
|
384
383
|
protected buildContextPipeline(): ContextPipeline<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>;
|
|
385
|
-
private containsSelectors;
|
|
386
|
-
private isRequestBlocked;
|
|
387
|
-
private preparePage;
|
|
388
|
-
private prepareNavigation;
|
|
389
384
|
private navigate;
|
|
390
|
-
private finalizeNavigation;
|
|
391
|
-
/**
|
|
392
|
-
* Copies cookies from the live browser page into the session cookie jar.
|
|
393
|
-
*/
|
|
394
|
-
private persistCookiesFromPage;
|
|
395
385
|
/**
|
|
396
386
|
* Runs the user request handler, then re-reads browser cookies so login flows /
|
|
397
387
|
* `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
|
|
388
|
+
* @internal
|
|
398
389
|
*/
|
|
399
390
|
protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
|
|
400
|
-
private handleBlockedRequestByContent;
|
|
401
|
-
private restoreRequestState;
|
|
402
|
-
private applyCookies;
|
|
403
|
-
/**
|
|
404
|
-
* Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
|
|
405
|
-
*/
|
|
406
|
-
private handleNavigationTimeout;
|
|
407
|
-
/**
|
|
408
|
-
* Transforms proxy-related errors to `SessionError`.
|
|
409
|
-
*/
|
|
410
|
-
private throwIfProxyError;
|
|
411
391
|
protected abstract navigationHandler(crawlingContext: BrowserCrawlingContext<Page, Response>, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
|
|
412
|
-
private processResponse;
|
|
413
392
|
/**
|
|
414
|
-
*
|
|
415
|
-
*
|
|
393
|
+
* Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
|
|
394
|
+
* crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
|
|
416
395
|
*/
|
|
417
396
|
teardown(): Promise<void>;
|
|
397
|
+
destroy(): Promise<void>;
|
|
418
398
|
}
|
|
419
399
|
/**
|
|
420
400
|
* Extracts URLs from a given page.
|
|
@@ -1,22 +1,8 @@
|
|
|
1
|
-
import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError,
|
|
2
|
-
import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
|
|
1
|
+
import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, skippedNavigationRequest, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
|
|
2
|
+
import { assertBrowserPoolNotConfigured, AWS_WAF_RETRY_CSS_SELECTORS, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
|
|
3
3
|
import { sleep } from '@crawlee/utils';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
|
|
6
|
-
/**
|
|
7
|
-
* Rejects options that exist only to configure the browser pool the crawler would have built for itself.
|
|
8
|
-
* Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
|
|
9
|
-
* into a second, half-working way of configuring the same pool.
|
|
10
|
-
*/
|
|
11
|
-
export function assertBrowserPoolNotConfigured(crawlerName, ignoredOptions) {
|
|
12
|
-
const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);
|
|
13
|
-
if (names.length === 0) {
|
|
14
|
-
return;
|
|
15
|
-
}
|
|
16
|
-
throw new Error(`${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` +
|
|
17
|
-
`${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` +
|
|
18
|
-
'itself. Configure the pool you pass in instead.');
|
|
19
|
-
}
|
|
20
6
|
const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
|
|
21
7
|
const readContextField = (ctx, key) => ctx[key];
|
|
22
8
|
/**
|
|
@@ -80,13 +66,15 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
80
66
|
get browserPool() {
|
|
81
67
|
return this.#browserPoolDep.value;
|
|
82
68
|
}
|
|
83
|
-
launchContext;
|
|
84
69
|
ignoreShadowRoots;
|
|
85
70
|
ignoreIframes;
|
|
86
71
|
#navigationTimeoutMillis;
|
|
87
72
|
#preNavigationHooks;
|
|
88
73
|
#postNavigationHooks;
|
|
89
74
|
#saveResponseCookies;
|
|
75
|
+
/**
|
|
76
|
+
* @internal
|
|
77
|
+
*/
|
|
90
78
|
static optionsShape = {
|
|
91
79
|
...BasicCrawler.optionsShape,
|
|
92
80
|
navigationTimeoutSecs: schemas.anyNumber
|
|
@@ -94,7 +82,6 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
94
82
|
.default(60),
|
|
95
83
|
preNavigationHooks: schemas.anyArray.default(() => []),
|
|
96
84
|
postNavigationHooks: schemas.anyArray.default(() => []),
|
|
97
|
-
launchContext: schemas.anyObject.default(() => ({})),
|
|
98
85
|
browserPool: validators.browserPool.optional(),
|
|
99
86
|
browserPoolBuilder: schemas.anyFunction.optional(),
|
|
100
87
|
remoteBrowser: schemas.anyObject.optional(),
|
|
@@ -103,18 +90,18 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
103
90
|
ignoreIframes: z.boolean().default(false),
|
|
104
91
|
ignoreShadowRoots: z.boolean().default(false),
|
|
105
92
|
};
|
|
93
|
+
/** @internal */
|
|
106
94
|
static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
|
|
107
95
|
/**
|
|
108
96
|
* All `BrowserCrawler` parameters are passed via an options object.
|
|
97
|
+
* @internal
|
|
109
98
|
*/
|
|
110
99
|
constructor(options) {
|
|
111
|
-
const { navigationTimeoutSecs, saveResponseCookies,
|
|
100
|
+
const { navigationTimeoutSecs, saveResponseCookies, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
|
|
112
101
|
if (browserPool) {
|
|
113
102
|
assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
|
|
114
103
|
}
|
|
115
|
-
const skipGuard = (action) => ({
|
|
116
|
-
action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
|
|
117
|
-
});
|
|
104
|
+
const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
|
|
118
105
|
super({
|
|
119
106
|
...basicCrawlerOptions,
|
|
120
107
|
contextPipelineBuilder: () => {
|
|
@@ -129,7 +116,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
129
116
|
}
|
|
130
117
|
return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
131
118
|
});
|
|
132
|
-
let pipeline = contextPipelineBuilder().compose(
|
|
119
|
+
let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));
|
|
133
120
|
for (const hook of this.#preNavigationHooks) {
|
|
134
121
|
pipeline = pipeline.compose(windowGuard(hook));
|
|
135
122
|
}
|
|
@@ -138,13 +125,12 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
138
125
|
pipeline = pipeline.compose(windowGuard(hook));
|
|
139
126
|
}
|
|
140
127
|
return pipeline
|
|
141
|
-
.compose(skipGuard(this
|
|
142
|
-
.compose(
|
|
143
|
-
.compose(
|
|
128
|
+
.compose(skipGuard(this.#finalizeNavigation.bind(this)))
|
|
129
|
+
.compose(this.#handleBlockedRequestByContent.bind(this))
|
|
130
|
+
.compose(this.#restoreRequestState.bind(this));
|
|
144
131
|
},
|
|
145
132
|
extendContext,
|
|
146
133
|
});
|
|
147
|
-
this.launchContext = launchContext;
|
|
148
134
|
this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
|
|
149
135
|
// The public option hooks are extension-aware; internal storage uses the base context type
|
|
150
136
|
// (the pipeline composes hooks against the concrete context, which does not statically carry
|
|
@@ -156,43 +142,39 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
156
142
|
this.#saveResponseCookies = saveResponseCookies;
|
|
157
143
|
this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
|
|
158
144
|
}
|
|
145
|
+
/** @internal */
|
|
159
146
|
getNavigationTimeoutMillis() {
|
|
160
147
|
return this.#navigationTimeoutMillis;
|
|
161
148
|
}
|
|
162
149
|
buildContextPipeline() {
|
|
163
|
-
return ContextPipeline.create().compose(
|
|
164
|
-
action: this.preparePage.bind(this),
|
|
165
|
-
cleanup: async (context) => {
|
|
166
|
-
context.registerDeferredCleanup(async () => {
|
|
167
|
-
const error = !context.session.isUsable()
|
|
168
|
-
? new SessionError('Session is no longer usable')
|
|
169
|
-
: undefined;
|
|
170
|
-
await this.browserPool
|
|
171
|
-
.closePage(context.page, { error })
|
|
172
|
-
.catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
|
|
173
|
-
});
|
|
174
|
-
},
|
|
175
|
-
});
|
|
150
|
+
return ContextPipeline.create().compose(this.#preparePage.bind(this));
|
|
176
151
|
}
|
|
177
|
-
async containsSelectors(page, selectors) {
|
|
178
|
-
const
|
|
152
|
+
async #containsSelectors(page, selectors) {
|
|
153
|
+
const query = async () => Promise.all(selectors.map((selector) => page.$(selector)));
|
|
154
|
+
// a challenge page navigates itself once it is solved, which destroys the context of an in-flight query
|
|
155
|
+
const elements = await query().catch(async () => {
|
|
156
|
+
await sleep(1000);
|
|
157
|
+
return query();
|
|
158
|
+
});
|
|
159
|
+
const foundSelectors = elements
|
|
179
160
|
.map((x, i) => [x, selectors[i]])
|
|
180
161
|
.filter(([x]) => x !== null)
|
|
181
162
|
.map(([, selector]) => selector);
|
|
182
163
|
return foundSelectors.length > 0 ? foundSelectors : null;
|
|
183
164
|
}
|
|
184
|
-
async
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
if (!foundSelectors)
|
|
192
|
-
return false;
|
|
193
|
-
return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
|
|
165
|
+
async #detectSelfResolvingChallenge({ page, response, }) {
|
|
166
|
+
const statusCode = response?.status();
|
|
167
|
+
if (statusCode === 403 && (await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS))) {
|
|
168
|
+
return 'Cloudflare';
|
|
169
|
+
}
|
|
170
|
+
if (statusCode === 202 && (await this.#containsSelectors(page, AWS_WAF_RETRY_CSS_SELECTORS))) {
|
|
171
|
+
return 'AWS WAF';
|
|
194
172
|
}
|
|
195
|
-
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
async #isRequestBlocked(crawlingContext) {
|
|
176
|
+
const { page, response } = crawlingContext;
|
|
177
|
+
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
196
178
|
const statusCode = response?.status() ?? 0;
|
|
197
179
|
if (foundSelectors)
|
|
198
180
|
return `Found selectors: ${foundSelectors.join(', ')}`;
|
|
@@ -200,12 +182,34 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
200
182
|
return `Received blocked status code: ${statusCode}`;
|
|
201
183
|
return false;
|
|
202
184
|
}
|
|
203
|
-
async preparePage(crawlingContext) {
|
|
185
|
+
async #preparePage(crawlingContext, onCleanup) {
|
|
204
186
|
const page = await this.browserPool.newPage({
|
|
205
187
|
id: crawlingContext.id,
|
|
206
188
|
session: crawlingContext.session,
|
|
207
189
|
});
|
|
208
190
|
tryCancel();
|
|
191
|
+
onCleanup((failure) => {
|
|
192
|
+
crawlingContext.registerDeferredCleanup(async () => {
|
|
193
|
+
const cause = failure instanceof RequestHandlerError || failure instanceof ContextPipelineInitializationError
|
|
194
|
+
? failure.cause
|
|
195
|
+
: failure;
|
|
196
|
+
// Only tells the pool what to do with the page's browser state - the request's failure, if any, has
|
|
197
|
+
// already been handled. A thrown `SessionError` or a blocked session counts as a block; a session
|
|
198
|
+
// unusable for any other reason is reported as retired. `retire()`/`markBad()` also bump the usage
|
|
199
|
+
// count, so the block check must come first.
|
|
200
|
+
const { session } = crawlingContext;
|
|
201
|
+
const closeReason = cause instanceof SessionError
|
|
202
|
+
? cause
|
|
203
|
+
: session.isBlocked()
|
|
204
|
+
? new SessionError()
|
|
205
|
+
: session.isUsable()
|
|
206
|
+
? undefined
|
|
207
|
+
: new SessionRetiredError();
|
|
208
|
+
await this.browserPool
|
|
209
|
+
.closePage(page, { error: closeReason })
|
|
210
|
+
.catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
|
|
211
|
+
});
|
|
212
|
+
});
|
|
209
213
|
const addRequests = crawlingContext.addRequests;
|
|
210
214
|
const extractLinks = async (options) => {
|
|
211
215
|
return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
|
|
@@ -235,17 +239,10 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
235
239
|
},
|
|
236
240
|
};
|
|
237
241
|
}
|
|
238
|
-
async prepareNavigation(crawlingContext) {
|
|
242
|
+
async #prepareNavigation(crawlingContext) {
|
|
239
243
|
if (crawlingContext.request.skipNavigation) {
|
|
240
244
|
return {
|
|
241
|
-
request:
|
|
242
|
-
get(target, propertyName, receiver) {
|
|
243
|
-
if (propertyName === 'loadedUrl') {
|
|
244
|
-
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
245
|
-
}
|
|
246
|
-
return Reflect.get(target, propertyName, receiver);
|
|
247
|
-
},
|
|
248
|
-
}),
|
|
245
|
+
request: skippedNavigationRequest(crawlingContext.request),
|
|
249
246
|
get response() {
|
|
250
247
|
throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
|
|
251
248
|
},
|
|
@@ -259,6 +256,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
259
256
|
[COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
|
|
260
257
|
};
|
|
261
258
|
}
|
|
259
|
+
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
262
260
|
async navigate(crawlingContext) {
|
|
263
261
|
tryCancel();
|
|
264
262
|
const gotoOptions = crawlingContext.gotoOptions;
|
|
@@ -276,22 +274,22 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
276
274
|
}
|
|
277
275
|
const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
|
|
278
276
|
const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
|
|
279
|
-
await this
|
|
277
|
+
await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
|
|
280
278
|
let response;
|
|
281
279
|
try {
|
|
282
280
|
response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
|
|
283
281
|
}
|
|
284
282
|
catch (error) {
|
|
285
|
-
await this
|
|
283
|
+
await this.#handleNavigationTimeout(crawlingContext, error);
|
|
286
284
|
crawlingContext.request.state = RequestState.ERROR;
|
|
287
|
-
this
|
|
285
|
+
this.#throwIfProxyError(error);
|
|
288
286
|
throw error;
|
|
289
287
|
}
|
|
290
288
|
tryCancel();
|
|
291
289
|
crawlingContext.request.state = RequestState.AFTER_NAV;
|
|
292
290
|
return { response };
|
|
293
291
|
}
|
|
294
|
-
async finalizeNavigation(crawlingContext) {
|
|
292
|
+
async #finalizeNavigation(crawlingContext) {
|
|
295
293
|
tryCancel();
|
|
296
294
|
let response;
|
|
297
295
|
try {
|
|
@@ -301,17 +299,17 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
301
299
|
// `preparePage` installs a throwing getter for `response`; reaching this branch means
|
|
302
300
|
// navigation produced no response and no hook overrode it. Treat as undefined.
|
|
303
301
|
}
|
|
304
|
-
await this
|
|
302
|
+
await this.#processResponse(response, crawlingContext);
|
|
305
303
|
tryCancel();
|
|
306
304
|
// Persist cookies from the navigation response before the user handler runs.
|
|
307
305
|
// Cookies set during `requestHandler` are saved again afterwards.
|
|
308
|
-
await this
|
|
306
|
+
await this.#persistCookiesFromPage(crawlingContext);
|
|
309
307
|
return { request: crawlingContext.request };
|
|
310
308
|
}
|
|
311
309
|
/**
|
|
312
310
|
* Copies cookies from the live browser page into the session cookie jar.
|
|
313
311
|
*/
|
|
314
|
-
async persistCookiesFromPage(crawlingContext) {
|
|
312
|
+
async #persistCookiesFromPage(crawlingContext) {
|
|
315
313
|
if (!this.#saveResponseCookies || !crawlingContext.session) {
|
|
316
314
|
return;
|
|
317
315
|
}
|
|
@@ -333,6 +331,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
333
331
|
/**
|
|
334
332
|
* Runs the user request handler, then re-reads browser cookies so login flows /
|
|
335
333
|
* `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
|
|
334
|
+
* @internal
|
|
336
335
|
*/
|
|
337
336
|
async runRequestHandler(crawlingContext) {
|
|
338
337
|
try {
|
|
@@ -341,7 +340,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
341
340
|
finally {
|
|
342
341
|
if (!crawlingContext.request.skipNavigation) {
|
|
343
342
|
try {
|
|
344
|
-
await this
|
|
343
|
+
await this.#persistCookiesFromPage(crawlingContext);
|
|
345
344
|
}
|
|
346
345
|
catch {
|
|
347
346
|
// Page may already be closed on some failure paths; ignore.
|
|
@@ -349,19 +348,44 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
349
348
|
}
|
|
350
349
|
}
|
|
351
350
|
}
|
|
352
|
-
async handleBlockedRequestByContent(crawlingContext) {
|
|
353
|
-
if (this.retryOnBlocked)
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
351
|
+
async #handleBlockedRequestByContent(crawlingContext) {
|
|
352
|
+
if (!this.retryOnBlocked)
|
|
353
|
+
return {};
|
|
354
|
+
const page = crawlingContext.page;
|
|
355
|
+
// A passed challenge navigates to the actual page, whose response replaces the one of the challenge.
|
|
356
|
+
let navigationResponse;
|
|
357
|
+
const onResponse = (response) => {
|
|
358
|
+
if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
|
|
359
|
+
navigationResponse = response;
|
|
360
|
+
}
|
|
361
|
+
};
|
|
362
|
+
page.on('response', onResponse);
|
|
363
|
+
try {
|
|
364
|
+
// Cloudflare and AWS WAF serve challenges a browser can pass on its own, with a 403 and a 202 status respectively.
|
|
365
|
+
// Wait 5 seconds for the JS challenge to load / resolve before judging the page.
|
|
366
|
+
const challenge = await this.#detectSelfResolvingChallenge(crawlingContext);
|
|
367
|
+
if (challenge) {
|
|
368
|
+
await sleep(5000);
|
|
369
|
+
const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
370
|
+
if (foundSelectors) {
|
|
371
|
+
throw new SessionError(`${challenge} challenge failed, found selectors: ${foundSelectors.join(', ')}`);
|
|
372
|
+
}
|
|
373
|
+
return navigationResponse ? { response: navigationResponse } : {};
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
finally {
|
|
377
|
+
page.off('response', onResponse);
|
|
357
378
|
}
|
|
379
|
+
const error = await this.#isRequestBlocked(crawlingContext);
|
|
380
|
+
if (error)
|
|
381
|
+
throw new SessionError(error);
|
|
358
382
|
return {};
|
|
359
383
|
}
|
|
360
|
-
async restoreRequestState(crawlingContext) {
|
|
384
|
+
async #restoreRequestState(crawlingContext) {
|
|
361
385
|
crawlingContext.request.state = RequestState.REQUEST_HANDLER;
|
|
362
386
|
return {};
|
|
363
387
|
}
|
|
364
|
-
async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
|
|
388
|
+
async #applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
|
|
365
389
|
const sessionCookie = session
|
|
366
390
|
? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
|
|
367
391
|
: [];
|
|
@@ -375,7 +399,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
375
399
|
/**
|
|
376
400
|
* Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
|
|
377
401
|
*/
|
|
378
|
-
async handleNavigationTimeout(crawlingContext, error) {
|
|
402
|
+
async #handleNavigationTimeout(crawlingContext, error) {
|
|
379
403
|
const { session, page } = crawlingContext;
|
|
380
404
|
// Fire-and-forget: no user code will run on this page after a failed navigation.
|
|
381
405
|
// Swallow rejections: the page may already be detached.
|
|
@@ -390,12 +414,12 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
390
414
|
/**
|
|
391
415
|
* Transforms proxy-related errors to `SessionError`.
|
|
392
416
|
*/
|
|
393
|
-
throwIfProxyError(error) {
|
|
417
|
+
#throwIfProxyError(error) {
|
|
394
418
|
if (this.isProxyError(error)) {
|
|
395
419
|
throw new SessionError(this.getMessageFromError(error));
|
|
396
420
|
}
|
|
397
421
|
}
|
|
398
|
-
async processResponse(response, crawlingContext) {
|
|
422
|
+
async #processResponse(response, crawlingContext) {
|
|
399
423
|
const { session, request, page } = crawlingContext;
|
|
400
424
|
if (typeof response === 'object' && typeof response.status === 'function') {
|
|
401
425
|
const status = response.status();
|
|
@@ -427,13 +451,17 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
427
451
|
request.loadedUrl = await page.url();
|
|
428
452
|
}
|
|
429
453
|
/**
|
|
430
|
-
*
|
|
431
|
-
*
|
|
454
|
+
* Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
|
|
455
|
+
* crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
|
|
432
456
|
*/
|
|
433
457
|
async teardown() {
|
|
434
|
-
await this.#browserPoolDep.ifOwned((pool) => pool.
|
|
458
|
+
await this.#browserPoolDep.ifOwned((pool) => pool.releaseAllBrowsers());
|
|
435
459
|
await super.teardown();
|
|
436
460
|
}
|
|
461
|
+
async destroy() {
|
|
462
|
+
await super.destroy();
|
|
463
|
+
await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
|
|
464
|
+
}
|
|
437
465
|
}
|
|
438
466
|
/**
|
|
439
467
|
* Extracts URLs from a given page.
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { Configuration } from '@crawlee/basic';
|
|
2
|
-
import type
|
|
3
|
-
import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
|
|
2
|
+
import { type BrowserPlugin, type BrowserPluginOptions, BrowserPool, type BrowserPoolHooks, type BrowserPoolOptions, type RemoteBrowserOptions } from '@crawlee/browser-pool';
|
|
4
3
|
import type { Constructor, Dictionary } from '@crawlee/types';
|
|
5
4
|
import { z } from 'zod';
|
|
6
5
|
export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
|
|
@@ -72,18 +71,26 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
|
|
|
72
71
|
* the browser they run against is only known to the concrete `*BrowserPool()` factory, which is where the
|
|
73
72
|
* caller-facing types are pinned down.
|
|
74
73
|
*/
|
|
75
|
-
|
|
74
|
+
type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
|
|
76
75
|
[Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];
|
|
77
76
|
};
|
|
78
77
|
/**
|
|
79
|
-
* The
|
|
78
|
+
* The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option. Library-specific
|
|
79
|
+
* connect settings (e.g. Playwright's `protocol`) are added by the concrete crawler's options.
|
|
80
80
|
*/
|
|
81
|
-
export
|
|
81
|
+
export interface CrawlerRemoteBrowserOptions extends RemoteBrowserOptions {
|
|
82
|
+
/**
|
|
83
|
+
* Maximum number of remote browsers open at once; see {@link BrowserPoolOptions.maxOpenBrowsers}. Defaults
|
|
84
|
+
* to the {@link RemoteBrowserProvider.maxOpenBrowsers|provider's value}, or `Infinity`.
|
|
85
|
+
*/
|
|
86
|
+
maxOpenBrowsers?: number;
|
|
87
|
+
}
|
|
82
88
|
/**
|
|
83
89
|
* Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
|
|
84
90
|
* @ignore
|
|
85
91
|
*/
|
|
86
92
|
export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Launcher = Plugin['library'], T extends Constructor<Plugin> = Constructor<Plugin>, LaunchOptions extends Dictionary<any> | undefined = Partial<Parameters<Plugin['launch']>[0]>, LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>> {
|
|
93
|
+
#private;
|
|
87
94
|
readonly configuration: Configuration;
|
|
88
95
|
launcher: Launcher;
|
|
89
96
|
proxyUrl?: string;
|
|
@@ -91,7 +98,11 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
91
98
|
launchOptions: Dictionary;
|
|
92
99
|
otherLaunchContextProps: Dictionary;
|
|
93
100
|
Plugin: T;
|
|
101
|
+
RemotePlugin?: T;
|
|
94
102
|
userAgent?: string;
|
|
103
|
+
/**
|
|
104
|
+
* @internal
|
|
105
|
+
*/
|
|
95
106
|
protected static optionsShape: {
|
|
96
107
|
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
97
108
|
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -102,6 +113,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
102
113
|
launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
103
114
|
userAgent: z.ZodOptional<z.ZodString>;
|
|
104
115
|
};
|
|
116
|
+
/** @internal */
|
|
105
117
|
protected static optionsSchema: z.ZodObject<{
|
|
106
118
|
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
107
119
|
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -120,27 +132,21 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
120
132
|
/**
|
|
121
133
|
* @ignore
|
|
122
134
|
*/
|
|
123
|
-
createBrowserPlugin(): Plugin;
|
|
135
|
+
createBrowserPlugin(remoteBrowser?: CrawlerRemoteBrowserOptions): Plugin;
|
|
124
136
|
/**
|
|
125
137
|
* Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
|
|
126
138
|
* per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
|
|
127
139
|
* a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
|
|
140
|
+
* With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
|
|
128
141
|
* @internal
|
|
129
142
|
*/
|
|
130
|
-
createBrowserPool(options?: LauncherBrowserPoolOptions
|
|
143
|
+
createBrowserPool({ remoteBrowser, ...options }?: LauncherBrowserPoolOptions & {
|
|
144
|
+
remoteBrowser?: CrawlerRemoteBrowserOptions;
|
|
145
|
+
}): BrowserPool<{
|
|
131
146
|
browserPlugins: [Plugin];
|
|
132
|
-
}, [
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
* supplies the plugin, the caller supplies the remote connection details.
|
|
136
|
-
* @internal
|
|
137
|
-
*/
|
|
138
|
-
createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
|
|
139
|
-
/**
|
|
140
|
-
* A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
|
|
141
|
-
* explicitly requested user agent wins.
|
|
142
|
-
*/
|
|
143
|
-
private resolveFingerprinting;
|
|
147
|
+
}, [
|
|
148
|
+
Plugin
|
|
149
|
+
]>;
|
|
144
150
|
/**
|
|
145
151
|
* Launches a browser instance based on the plugin.
|
|
146
152
|
* @returns Browser instance.
|
|
@@ -148,10 +154,5 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
148
154
|
launch(): LaunchResult;
|
|
149
155
|
createLaunchOptions(): Dictionary;
|
|
150
156
|
protected getDefaultHeadlessOption(): boolean;
|
|
151
|
-
private getChromeExecutablePath;
|
|
152
|
-
/**
|
|
153
|
-
* Gets a typical path to Chrome executable, depending on the current operating system.
|
|
154
|
-
*/
|
|
155
|
-
private getTypicalChromeExecutablePath;
|
|
156
|
-
private validateProxyUrlProtocol;
|
|
157
157
|
}
|
|
158
|
+
export {};
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import { createRequire } from 'node:module';
|
|
3
3
|
import os from 'node:os';
|
|
4
|
-
import { Configuration,
|
|
5
|
-
import { BrowserPool,
|
|
4
|
+
import { Configuration, serviceLocator } from '@crawlee/basic';
|
|
5
|
+
import { BrowserPool, RemoteBrowserProvider, } from '@crawlee/browser-pool';
|
|
6
|
+
import { schemas } from '@crawlee/utils/internal';
|
|
6
7
|
import { z } from 'zod';
|
|
7
8
|
const DEFAULT_VIEWPORT = {
|
|
8
9
|
width: 1366,
|
|
@@ -20,9 +21,13 @@ export class BrowserLauncher {
|
|
|
20
21
|
useChrome;
|
|
21
22
|
launchOptions;
|
|
22
23
|
otherLaunchContextProps;
|
|
23
|
-
// to be provided by child classes;
|
|
24
|
+
// to be provided by child classes; `RemotePlugin` only by those supporting remote browsers
|
|
24
25
|
Plugin;
|
|
26
|
+
RemotePlugin;
|
|
25
27
|
userAgent;
|
|
28
|
+
/**
|
|
29
|
+
* @internal
|
|
30
|
+
*/
|
|
26
31
|
static optionsShape = {
|
|
27
32
|
proxyUrl: z.url().optional(),
|
|
28
33
|
useChrome: z.boolean().optional(),
|
|
@@ -33,6 +38,7 @@ export class BrowserLauncher {
|
|
|
33
38
|
launchOptions: schemas.anyObject.optional(),
|
|
34
39
|
userAgent: z.string().optional(),
|
|
35
40
|
};
|
|
41
|
+
/** @internal */
|
|
36
42
|
static optionsSchema = z.strictObject(BrowserLauncher.optionsShape);
|
|
37
43
|
static requireLauncherOrThrow(launcher, apifyImageName) {
|
|
38
44
|
try {
|
|
@@ -56,7 +62,7 @@ export class BrowserLauncher {
|
|
|
56
62
|
constructor(launchContext, configuration = Configuration.getGlobalConfiguration()) {
|
|
57
63
|
this.configuration = configuration;
|
|
58
64
|
const { launcher, proxyUrl, useChrome, userAgent, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
|
|
59
|
-
this
|
|
65
|
+
this.#validateProxyUrlProtocol(proxyUrl);
|
|
60
66
|
// those need to be reassigned otherwise they are {} in types
|
|
61
67
|
this.launcher = launcher;
|
|
62
68
|
this.proxyUrl = proxyUrl;
|
|
@@ -68,45 +74,46 @@ export class BrowserLauncher {
|
|
|
68
74
|
/**
|
|
69
75
|
* @ignore
|
|
70
76
|
*/
|
|
71
|
-
createBrowserPlugin() {
|
|
72
|
-
|
|
77
|
+
createBrowserPlugin(remoteBrowser) {
|
|
78
|
+
const options = {
|
|
73
79
|
proxyUrl: this.proxyUrl,
|
|
74
80
|
launchOptions: this.createLaunchOptions(),
|
|
75
81
|
...this.otherLaunchContextProps,
|
|
76
|
-
}
|
|
82
|
+
};
|
|
83
|
+
if (!remoteBrowser) {
|
|
84
|
+
return new this.Plugin(this.launcher, options);
|
|
85
|
+
}
|
|
86
|
+
if (!this.RemotePlugin) {
|
|
87
|
+
throw new Error(`${this.constructor.name} does not support remote browsers.`);
|
|
88
|
+
}
|
|
89
|
+
const { maxOpenBrowsers: _, ...remoteOptions } = remoteBrowser;
|
|
90
|
+
return new this.RemotePlugin(this.launcher, { ...options, ...remoteOptions });
|
|
77
91
|
}
|
|
78
92
|
/**
|
|
79
93
|
* Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
|
|
80
94
|
* per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
|
|
81
95
|
* a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
|
|
96
|
+
* With `remoteBrowser`, the plugin connects to a remote browser service instead of launching locally.
|
|
82
97
|
* @internal
|
|
83
98
|
*/
|
|
84
|
-
createBrowserPool(options = {}) {
|
|
99
|
+
createBrowserPool({ remoteBrowser, ...options } = {}) {
|
|
100
|
+
const provider = remoteBrowser?.endpoint instanceof RemoteBrowserProvider ? remoteBrowser.endpoint : undefined;
|
|
101
|
+
const maxOpenBrowsers = options.maxOpenBrowsers ?? remoteBrowser?.maxOpenBrowsers ?? provider?.maxOpenBrowsers;
|
|
85
102
|
// The hook types `BrowserPool` derives from `Plugin` are unresolvable while `Plugin` is still a free type
|
|
86
103
|
// parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
|
|
87
104
|
// caller-facing hook types get pinned down.
|
|
88
105
|
return new BrowserPool({
|
|
89
|
-
...this
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
}
|
|
93
|
-
/**
|
|
94
|
-
* The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
|
|
95
|
-
* supplies the plugin, the caller supplies the remote connection details.
|
|
96
|
-
* @internal
|
|
97
|
-
*/
|
|
98
|
-
createRemoteBrowserPool(options) {
|
|
99
|
-
return new RemoteBrowserPool({
|
|
100
|
-
...options,
|
|
101
|
-
browserPlugins: [this.createBrowserPlugin()],
|
|
102
|
-
browserPoolOptions: this.resolveFingerprinting(options.browserPoolOptions ?? {}),
|
|
106
|
+
...this.#resolveFingerprinting(options),
|
|
107
|
+
...(maxOpenBrowsers !== undefined && { maxOpenBrowsers }),
|
|
108
|
+
browserPlugins: [this.createBrowserPlugin(remoteBrowser)],
|
|
103
109
|
});
|
|
104
110
|
}
|
|
105
111
|
/**
|
|
112
|
+
|
|
106
113
|
* A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
|
|
107
114
|
* explicitly requested user agent wins.
|
|
108
115
|
*/
|
|
109
|
-
resolveFingerprinting(options) {
|
|
116
|
+
#resolveFingerprinting(options) {
|
|
110
117
|
if (!this.userAgent) {
|
|
111
118
|
return options;
|
|
112
119
|
}
|
|
@@ -143,20 +150,20 @@ export class BrowserLauncher {
|
|
|
143
150
|
launchOptions.headless = this.getDefaultHeadlessOption();
|
|
144
151
|
}
|
|
145
152
|
if (this.useChrome && !launchOptions.executablePath) {
|
|
146
|
-
launchOptions.executablePath = this
|
|
153
|
+
launchOptions.executablePath = this.#getChromeExecutablePath();
|
|
147
154
|
}
|
|
148
155
|
return launchOptions;
|
|
149
156
|
}
|
|
150
157
|
getDefaultHeadlessOption() {
|
|
151
158
|
return this.configuration.headless && !this.configuration.xvfb;
|
|
152
159
|
}
|
|
153
|
-
getChromeExecutablePath() {
|
|
154
|
-
return this.configuration.chromeExecutablePath ?? this
|
|
160
|
+
#getChromeExecutablePath() {
|
|
161
|
+
return this.configuration.chromeExecutablePath ?? this.#getTypicalChromeExecutablePath();
|
|
155
162
|
}
|
|
156
163
|
/**
|
|
157
164
|
* Gets a typical path to Chrome executable, depending on the current operating system.
|
|
158
165
|
*/
|
|
159
|
-
getTypicalChromeExecutablePath() {
|
|
166
|
+
#getTypicalChromeExecutablePath() {
|
|
160
167
|
/**
|
|
161
168
|
* Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.
|
|
162
169
|
* Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).
|
|
@@ -184,7 +191,7 @@ export class BrowserLauncher {
|
|
|
184
191
|
return '/usr/bin/google-chrome';
|
|
185
192
|
}
|
|
186
193
|
}
|
|
187
|
-
validateProxyUrlProtocol(proxyUrl) {
|
|
194
|
+
#validateProxyUrlProtocol(proxyUrl) {
|
|
188
195
|
if (!proxyUrl)
|
|
189
196
|
return;
|
|
190
197
|
if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-rc.
|
|
3
|
+
"version": "4.0.0-rc.2",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=22.
|
|
6
|
+
"node": ">=22.13.0"
|
|
7
7
|
},
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
@@ -47,14 +47,13 @@
|
|
|
47
47
|
"access": "public"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@apify/timeout": "^0.
|
|
51
|
-
"@crawlee/basic": "4.0.0-rc.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-rc.
|
|
53
|
-
"@crawlee/types": "4.0.0-rc.
|
|
54
|
-
"@crawlee/utils": "4.0.0-rc.
|
|
55
|
-
"tslib": "^2.
|
|
56
|
-
"
|
|
57
|
-
"zod": "^4.4.3"
|
|
50
|
+
"@apify/timeout": "^1.0.1",
|
|
51
|
+
"@crawlee/basic": "4.0.0-rc.2",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-rc.2",
|
|
53
|
+
"@crawlee/types": "4.0.0-rc.2",
|
|
54
|
+
"@crawlee/utils": "4.0.0-rc.2",
|
|
55
|
+
"tslib": "^2.9.1",
|
|
56
|
+
"zod": "^4.6.7"
|
|
58
57
|
},
|
|
59
58
|
"peerDependencies": {
|
|
60
59
|
"playwright": "*",
|
|
@@ -75,5 +74,5 @@
|
|
|
75
74
|
}
|
|
76
75
|
}
|
|
77
76
|
},
|
|
78
|
-
"gitHead": "
|
|
77
|
+
"gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
|
|
79
78
|
}
|