@crawlee/browser 4.0.0-beta.63 → 4.0.0-beta.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -9,7 +9,7 @@ import type { BrowserLaunchContext } from './browser-launcher.js';
|
|
|
9
9
|
interface BaseResponse {
|
|
10
10
|
status(): number;
|
|
11
11
|
}
|
|
12
|
-
export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
12
|
+
export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController = BrowserController, UserData extends Dictionary = Dictionary, GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
|
13
13
|
/**
|
|
14
14
|
* An instance of the {@link BrowserController} that manages the browser instance and provides access to its API.
|
|
15
15
|
*/
|
|
@@ -26,12 +26,17 @@ export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Re
|
|
|
26
26
|
* The HTTP response object returned by the browser's navigation.
|
|
27
27
|
*/
|
|
28
28
|
response: Response;
|
|
29
|
+
/**
|
|
30
|
+
* Options object passed to the underlying `page.goto()` call. `preNavigationHooks` can mutate this
|
|
31
|
+
* object (or return `{ gotoOptions: ... }`) to influence the navigation.
|
|
32
|
+
*/
|
|
33
|
+
gotoOptions: GoToOptions;
|
|
29
34
|
/**
|
|
30
35
|
* Helper function for extracting URLs from the current page and adding them to the request queue.
|
|
31
36
|
*/
|
|
32
37
|
enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
|
|
33
38
|
}
|
|
34
|
-
export type BrowserHook<Context = BrowserCrawlingContext
|
|
39
|
+
export type BrowserHook<Context = BrowserCrawlingContext> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
|
|
35
40
|
export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, ProvidedController extends BrowserController = BrowserController, Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
|
|
36
41
|
launchContext?: BrowserLaunchContext<any, any>;
|
|
37
42
|
/**
|
|
@@ -90,15 +95,14 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
90
95
|
browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
|
|
91
96
|
/**
|
|
92
97
|
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
|
|
93
|
-
* or browser properties before navigation. The function
|
|
94
|
-
*
|
|
98
|
+
* or browser properties before navigation. The function receives the `crawlingContext`; the options object
|
|
99
|
+
* forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
|
|
95
100
|
*
|
|
96
101
|
* **Example:**
|
|
97
102
|
*
|
|
98
103
|
* ```js
|
|
99
104
|
* preNavigationHooks: [
|
|
100
|
-
* async (
|
|
101
|
-
* const { page } = crawlingContext;
|
|
105
|
+
* async ({ page, gotoOptions }) => {
|
|
102
106
|
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
|
|
103
107
|
* gotoOptions.timeout = 60_000;
|
|
104
108
|
* gotoOptions.waitUntil = 'domcontentloaded';
|
|
@@ -106,14 +110,17 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
106
110
|
* ]
|
|
107
111
|
* ```
|
|
108
112
|
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
113
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context,
|
|
114
|
+
* allowing the hook to override context members for subsequent hooks and pipeline stages.
|
|
111
115
|
*/
|
|
112
116
|
preNavigationHooks?: BrowserHook<Context>[];
|
|
113
117
|
/**
|
|
114
118
|
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
|
|
115
119
|
* The function accepts `crawlingContext` as the only parameter.
|
|
116
120
|
*
|
|
121
|
+
* A hook may optionally return a partial object whose properties are merged into the crawling context.
|
|
122
|
+
* This is useful for overriding context members (e.g. `response`) after solving a challenge.
|
|
123
|
+
*
|
|
117
124
|
* **Example:**
|
|
118
125
|
*
|
|
119
126
|
* ```js
|
|
@@ -124,6 +131,11 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
|
|
|
124
131
|
* await solveCaptcha(page);
|
|
125
132
|
* }
|
|
126
133
|
* },
|
|
134
|
+
* async (crawlingContext) => {
|
|
135
|
+
* if (await needsRevalidation(crawlingContext)) {
|
|
136
|
+
* return { response: await crawlingContext.page.reload() };
|
|
137
|
+
* }
|
|
138
|
+
* },
|
|
127
139
|
* ]
|
|
128
140
|
* ```
|
|
129
141
|
*/
|
|
@@ -299,7 +311,9 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
|
|
|
299
311
|
private containsSelectors;
|
|
300
312
|
protected isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>): Promise<string | false>;
|
|
301
313
|
private preparePage;
|
|
302
|
-
private
|
|
314
|
+
private prepareNavigation;
|
|
315
|
+
private navigate;
|
|
316
|
+
private finalizeNavigation;
|
|
303
317
|
private handleBlockedRequestByContent;
|
|
304
318
|
private restoreRequestState;
|
|
305
319
|
protected _applyCookies({ session, request, page, browserController }: BrowserCrawlingContext, preHooksCookies: string, postHooksCookies: string): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,SAAS,EACT,mBAAmB,EACnB,oBAAoB,
|
|
1
|
+
{"version":3,"file":"browser-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,SAAS,EACT,mBAAmB,EACnB,oBAAoB,EAEpB,eAAe,EACf,UAAU,EACV,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,OAAO,EACP,cAAc,EACd,eAAe,EACf,sBAAsB,EACzB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,eAAe,EAWlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EACR,iBAAiB,EACjB,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,uBAAuB,EACvB,aAAa,EAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,EAAE,sBAAsB,EAAoC,MAAM,gBAAgB,CAAC;AAC/F,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAI9C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAElE,UAAU,YAAY;IAClB,MAAM,IAAI,MAAM,CAAC;CACpB;AAID,MAAM,WAAW,sBAAsB,CACnC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,SAAS,YAAY,GAAG,YAAY,EAC5C,kBAAkB,GAAG,iBAAiB,EACtC,QAAQ,SAAS,UAAU,GAAG,UAAU,EACxC,WAAW,SAAS,UAAU,GAAG,UAAU,CAC7C,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,iBAAiB,EAAE,kBAAkB,CAAC;IAEtC;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE1C;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;;OAGG;IACH,WAAW,EAAE,WAAW,CAAC;IAEzB;;OAEG;IACH,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,mBAAmB,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;CACpF;AAED,MAAM,MAAM,WAAW,CAAC,OAAO,GAAG,sBAAsB,IAAI,CACxD,eAAe,EAAE,OAAO,KACvB,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAMxC,MAAM,WAAW,qBAAqB,CAClC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,SAAS,YAAY,GAAG,YAAY,EAC5C,kBAAkB,SAAS,iBAAiB,GAAG,iBAAiB,EAChE,OAAO,SAAS,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,UAAU,CAAC,GAAG,sBAAsB,CAC3G,IAAI,EACJ,QAAQ,EACR,kBAAkB,EAClB,UAAU,CACb,EACD,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,EAC5D,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,gBAAgB,SAAS,aAAa,EAAE,GAAG,uBAAuB,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,EAChH,yBAAyB,SAAS,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,EAC9G,qBAAqB,SAAS,aAAa,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAC3G,SAAQ,IAAI,CACV,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC,EAE7C,gBAAgB,GAAG,sBAAsB,GAAG,cAAc,CAC7D;IACG,aAAa,CAAC,EAAE,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAE/C;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,eAAe,CAAC,CAAC;IAEjD;;;;;;;;;OASG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAE9D;;;;;;;;OAQG;IACH,oBAAoB,CAAC,EAAE,YAAY,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAEtE;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAC5C,OAAO,CAAC,gBAAgB,CAAC,yBAAyB,EAAE,qBAAqB,CAAC,CAAC,CAAC;IAEhF;;;;;;;;;;;;;;;;;;;OAmBG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE5C;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE7C;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,KAAK,GAAG,KAAK,CAAC;IAEnC;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,8BAAsB,cAAc,CAChC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,SAAS,YAAY,GAAG,YAAY,EAC5C,kBAAkB,SAAS,iBAAiB,GAAG,iBAAiB,EAChE,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,aAAa,SAAS,UAAU,GAAG,SAAS,GAAG,UAAU,EACzD,OAAO,SAAS,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,UAAU,CAAC,GAAG,sBAAsB,CAC3G,IAAI,EACJ,QAAQ,EACR,kBAAkB,EAClB,UAAU,CACb,EACD,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,EACpC,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,EAC5D,WAAW,SAAS,UAAU,GAAG,UAAU,CAC7C,SAAQ,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,eAAe,CAAC;IAC9D;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC,0BAA0B,CAAC,CAAC;IAErD,aAAa,EAAE,oBAAoB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAE5D,SAAS,CAAC,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IAC9C,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAE1C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACrD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;IAEvC,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAYpC;IAEF;;OAEG;IACH,SAAS,aACL,OAAO,EAAE,qBAAqB,CAC1B,IAAI,EACJ,QAAQ,EACR,kBAAkB,EAClB,OAAO,EACP,gBAAgB,EAChB,eAAe,CAClB,GAAG;QACA,sBAAsB,EAAE,MAAM,eAAe,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;KAC3E;cAwEc,oBAAoB,IAAI,eAAe,CACtD,eAAe,EACf,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,UAAU,CAAC,CACzE;YA4Ba,iBAAiB;cASf,gBAAgB,CAC5B,eAAe,EAAE,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,CAAC,GAC5E,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;YAuBZ,WAAW;YAmDX,iBAAiB;YA6BjB,QAAQ;YAyBR,kBAAkB;YAiClB,6BAA6B;YAW7B,mBAAmB;cAKjB,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,sBAAsB,EACrE,eAAe,EAAE,MAAM,EACvB,gBAAgB,EAAE,MAAM;IAc5B;;OAEG;cACa,wBAAwB,CAAC,eAAe,EAAE,sBAAsB,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9G;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAK;IAMzC,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CACjC,eAAe,EAAE,sBAAsB,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,CAAC,EAC3E,WAAW,EAAE,WAAW,GACzB,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;YAEpC,eAAe;IA+B7B;;;OAGG;IACY,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAI3C;AAED,gBAAgB;AAChB,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;IAC9G,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,eAAe,CAAC;IAC9B,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,UAAU,gCAAgC;IACtC,YAAY,EAAE,oBAAoB,CAAC,cAAc,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;IAC9G,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,EAAE,UAAU,CAAC;CACpB;AASD,gBAAgB;AAChB,wBAAsB,0BAA0B,CAC5C,OAAO,EAAE,2BAA2B,GAAG,gCAAgC,oBAiC1E;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CAErC,IAAI,EAAE;IAAE,MAAM,EAAE,QAAQ,CAAA;CAAE,EAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC,CA0BnB"}
|
|
@@ -3,6 +3,8 @@ import { BrowserPool } from '@crawlee/browser-pool';
|
|
|
3
3
|
import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
|
|
4
4
|
import ow from 'ow';
|
|
5
5
|
import { tryCancel } from '@apify/timeout';
|
|
6
|
+
const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
|
|
7
|
+
const readContextField = (ctx, key) => ctx[key];
|
|
6
8
|
/**
|
|
7
9
|
* Provides a simple framework for parallel crawling of web pages
|
|
8
10
|
* using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
|
|
@@ -71,12 +73,25 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
71
73
|
constructor(options) {
|
|
72
74
|
ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
|
|
73
75
|
const { navigationTimeoutSecs = 60, saveResponseCookies = true, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = options;
|
|
76
|
+
const skipGuard = (action) => ({
|
|
77
|
+
action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
|
|
78
|
+
});
|
|
74
79
|
super({
|
|
75
80
|
...basicCrawlerOptions,
|
|
76
|
-
contextPipelineBuilder: () =>
|
|
77
|
-
.compose({ action: this.
|
|
78
|
-
|
|
79
|
-
|
|
81
|
+
contextPipelineBuilder: () => {
|
|
82
|
+
let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
|
|
83
|
+
for (const hook of this.preNavigationHooks) {
|
|
84
|
+
pipeline = pipeline.compose(skipGuard(hook));
|
|
85
|
+
}
|
|
86
|
+
pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
|
|
87
|
+
for (const hook of this.postNavigationHooks) {
|
|
88
|
+
pipeline = pipeline.compose(skipGuard(hook));
|
|
89
|
+
}
|
|
90
|
+
return pipeline
|
|
91
|
+
.compose(skipGuard(this.finalizeNavigation.bind(this)))
|
|
92
|
+
.compose({ action: this.handleBlockedRequestByContent.bind(this) })
|
|
93
|
+
.compose({ action: this.restoreRequestState.bind(this) });
|
|
94
|
+
},
|
|
80
95
|
extendContext: extendContext,
|
|
81
96
|
});
|
|
82
97
|
this.launchContext = launchContext;
|
|
@@ -161,6 +176,9 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
161
176
|
get response() {
|
|
162
177
|
throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
|
|
163
178
|
},
|
|
179
|
+
get gotoOptions() {
|
|
180
|
+
throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
|
|
181
|
+
},
|
|
164
182
|
browserController: browserControllerInstance,
|
|
165
183
|
enqueueLinks: async (enqueueOptions = {}) => {
|
|
166
184
|
return (await browserCrawlerEnqueueLinks({
|
|
@@ -176,7 +194,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
176
194
|
},
|
|
177
195
|
};
|
|
178
196
|
}
|
|
179
|
-
async
|
|
197
|
+
async prepareNavigation(crawlingContext) {
|
|
180
198
|
if (crawlingContext.request.skipNavigation) {
|
|
181
199
|
return {
|
|
182
200
|
request: new Proxy(crawlingContext.request, {
|
|
@@ -192,13 +210,18 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
192
210
|
},
|
|
193
211
|
};
|
|
194
212
|
}
|
|
195
|
-
const gotoOptions = { timeout: this.navigationTimeoutMillis };
|
|
196
|
-
const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
|
|
197
213
|
crawlingContext.request.state = RequestState.BEFORE_NAV;
|
|
198
|
-
|
|
214
|
+
return {
|
|
215
|
+
gotoOptions: { timeout: this.navigationTimeoutMillis },
|
|
216
|
+
[COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
async navigate(crawlingContext) {
|
|
199
220
|
tryCancel();
|
|
200
|
-
const
|
|
201
|
-
|
|
221
|
+
const gotoOptions = crawlingContext.gotoOptions;
|
|
222
|
+
const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
|
|
223
|
+
const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
|
|
224
|
+
await this._applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
|
|
202
225
|
let response;
|
|
203
226
|
try {
|
|
204
227
|
response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
|
|
@@ -211,10 +234,20 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
211
234
|
}
|
|
212
235
|
tryCancel();
|
|
213
236
|
crawlingContext.request.state = RequestState.AFTER_NAV;
|
|
214
|
-
|
|
237
|
+
return { response };
|
|
238
|
+
}
|
|
239
|
+
async finalizeNavigation(crawlingContext) {
|
|
240
|
+
tryCancel();
|
|
241
|
+
let response;
|
|
242
|
+
try {
|
|
243
|
+
response = crawlingContext.response;
|
|
244
|
+
}
|
|
245
|
+
catch {
|
|
246
|
+
// `preparePage` installs a throwing getter for `response`; reaching this branch means
|
|
247
|
+
// navigation produced no response and no hook overrode it. Treat as undefined.
|
|
248
|
+
}
|
|
215
249
|
await this.processResponse(response, crawlingContext);
|
|
216
250
|
tryCancel();
|
|
217
|
-
// save cookies
|
|
218
251
|
// TODO: Should we save the cookies also after/only the handle page?
|
|
219
252
|
if (this.saveResponseCookies && crawlingContext.session) {
|
|
220
253
|
const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
|
|
@@ -231,15 +264,7 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
231
264
|
}
|
|
232
265
|
}
|
|
233
266
|
}
|
|
234
|
-
|
|
235
|
-
return {
|
|
236
|
-
request: crawlingContext.request,
|
|
237
|
-
response,
|
|
238
|
-
};
|
|
239
|
-
}
|
|
240
|
-
return {
|
|
241
|
-
request: crawlingContext.request,
|
|
242
|
-
};
|
|
267
|
+
return { request: crawlingContext.request };
|
|
243
268
|
}
|
|
244
269
|
async handleBlockedRequestByContent(crawlingContext) {
|
|
245
270
|
if (this.retryOnBlocked) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-crawler.js","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAcA,OAAO,EACH,YAAY,EACZ,8BAA8B,EAC9B,eAAe,EACf,yBAAyB,EACzB,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EACZ,sCAAsC,EACtC,YAAY,EACZ,8BAA8B,EAC9B,cAAc,EACd,UAAU,GACb,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,EAAE,8BAA8B,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5F,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAwM3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,OAAgB,cAepB,SAAQ,YAAwD;IAC9D;;OAEG;IACH,WAAW,CAA0C;IAErD,aAAa,CAA+C;IAEzC,iBAAiB,CAAU;IAC3B,aAAa,CAAU;IAEhC,uBAAuB,CAAS;IAChC,kBAAkB,CAAyB;IAC3C,mBAAmB,CAAyB;IAC5C,mBAAmB,CAAU;IAE7B,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC;QAChD,kBAAkB,EAAE,EAAE,CAAC,MAAM;QAC7B,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACxC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC;KACjF,CAAC;IAEF;;OAEG;IACH,YACI,OASC;QAED,EAAE,CAAC,OAAO,EAAE,uBAAuB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,GAAG,IAAI,EAC1B,aAAa,GAAG,EAAE,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,QAAQ,EACR,aAAa,GAAG,KAAK,EACrB,iBAAiB,GAAG,KAAK,EACzB,sBAAsB,EACtB,aAAa,EACb,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,sBAAsB,EAAE,GAAG,EAAE,CACzB,sBAAsB,EAAE;iBACnB,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBACtD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBAClE,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,aAAa,EAAE,aAAkE;SACpF,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,aAA4B,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzE,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,aAAa,EAAE,SAAS,EAAE,CAAC;YAC3B,IAAI,kBAAkB,CAAC,eAAe;gBAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;YACpG,kBAAkB,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAA6B;YAC3D,GAAI,kBAA0B;SACjC,CAAC,CAAC;IACP,CAAC;IAEkB,oBAAoB;QAInC,OAAO,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;YACrD,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YACnC,OAAO,EAAE,KAAK,EAAE,OAKf,EAAE,EAAE;gBACD,OAAO,CAAC,uBAAuB,CAAC,KAAK,IAAI,EAAE;oBACvC,6EAA6E;oBAC7E,2EAA2E;oBAC3E,8EAA8E;oBAC9E,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;wBAC9B,IAAI,CAAC,WAAW,CAAC,uBAAuB,CACpC,OAAO,CAAC,iBAEJ,CACP,CAAC;oBACN,CAAC;oBACD,MAAM,OAAO,CAAC,IAAI;yBACb,KAAK,EAAE;yBACP,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;gBACxF,CAAC,CAAC,CAAC;YACP,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,IAAgB,EAAE,SAAmB;QACjE,MAAM,cAAc,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAE,IAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC7F,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAErC,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC5B,eAA2E;QAE3E,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC;QAE3C,yGAAyG;QACzG,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC,IAAI,QAAQ,EAAE,MAAM,EAAE,KAAK,GAAG,EAAE,CAAC;YACrG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAElB,+IAA+I;YAC/I,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAE/E,IAAI,CAAC,cAAc;gBAAE,OAAO,KAAK,CAAC;YAClC,OAAO,iDAAiD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxF,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAC/E,MAAM,UAAU,GAAG,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3C,IAAI,cAAc;YAAE,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,OAAO,iCAAiC,UAAU,EAAE,CAAC;QAElG,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CACrB,eAAgC;QAIhC,MAAM,cAAc,GAAe;YAC/B,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,IAAI,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC;YACrC,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC;YAEpD,cAAc,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACzC,cAAc,CAAC,eAAe,GAAG,SAAS,EAAE,eAAe,CAAC;QAChE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAS,CAAC;QACtE,SAAS,EAAE,CAAC;QAEZ,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CACzE,IAAW,CACQ,CAAC;QAExB,MAAM,mBAAmB,GAAG,eAAe,CAAC,YAAY,CAAC;QAEzD,OAAO;YACH,IAAI;YACJ,IAAI,QAAQ;gBACR,MAAM,IAAI,KAAK,CACX,gMAAgM,CACnM,CAAC;YACN,CAAC;YACD,iBAAiB,EAAE,yBAAyB;YAC5C,YAAY,EAAE,KAAK,EAAE,iBAAsC,EAAE,EAAE,EAAE;gBAC7D,OAAO,CAAC,MAAM,0BAA0B,CAAC;oBACrC,OAAO,EAAE,EAAE,GAAG,cAAc,EAAE,KAAK,EAAE,IAAI,CAAC,6BAA6B,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE;oBAChG,IAAI;oBACJ,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7E,gBAAgB,EAAE,IAAI,CAAC,oBAAoB;oBAC3C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;oBAClD,YAAY,EAAE,mBAAmB;iBACpC,CAAC,CAA2B,CAAC,CAAC,2BAA2B;YAC9D,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QAIpD,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE;oBACxC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,sBAAsB,CAC5B,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAA2B;gBAC5B,IAAI,QAAQ;oBACR,MAAM,IAAI,sBAAsB,CAC5B,sEAAsE,CACzE,CAAC;gBACN,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAA4B,CAAC;QAExF,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE5F,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE7F,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEjG,IAAI,QAA8B,CAAC;QAEnC,IAAI,CAAC;YACD,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC,IAAI,SAAS,CAAC;QAC1F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAc,CAAC,CAAC;YAErE,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YAEnD,IAAI,CAAC,kBAAkB,CAAC,KAAc,CAAC,CAAC;YACxC,MAAM,KAAK,CAAC;QAChB,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAEjF,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QACtD,SAAS,EAAE,CAAC;QAEZ,eAAe;QACf,oEAAoE;QACpE,IAAI,IAAI,CAAC,mBAAmB,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;YACtD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YACzF,SAAS,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,eAAe,CAAC,OAAO,CAAC,SAAU,CAAC;YAC/C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACD,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE;wBACzF,WAAW,EAAE,KAAK;qBACrB,CAAC,CAAC;gBACP,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAA0B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;gBACpE,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,eAAe,CAAC,OAAiC;gBAC1D,QAAQ;aACX,CAAC;QACN,CAAC;QAED,OAAO;YACH,OAAO,EAAE,eAAe,CAAC,OAAiC;SAC7D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CACvC,eAA2E;QAE3E,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,EAAE,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,eAAgC;QAC9D,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,eAAe,CAAC;QAC7D,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAA0B,EACrE,eAAuB,EACvB,gBAAwB;QAExB,MAAM,aAAa,GAAG,OAAO,EAAE,SAAS,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,8BAA8B,CAAC,IAAI,EAAE,CAAC;QAC/G,MAAM,qBAAqB,GAAG,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtG,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QAExG,MAAM,iBAAiB,CAAC,UAAU,CAC9B,IAAI,EACJ,CAAC,GAAG,aAAa,EAAE,GAAG,qBAAqB,EAAE,GAAG,sBAAsB,CAAC;aAClE,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,OAAO,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,IAAI,CAAC;aACxE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CACvE,CAAC;IACN,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,wBAAwB,CAAC,eAAuC,EAAE,KAAY;QAC1F,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,EAAE,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC7C,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,KAAY;QACrC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAW,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAOO,KAAK,CAAC,eAAe,CACzB,QAA8B,EAC9B,eAAuC;QAEvC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAEnD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxE,MAAM,MAAM,GAAW,QAAQ,CAAC,MAAM,EAAE,CAAC;YAEzC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAEtC,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,IAAI,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBAClD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;gBACtE,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,0BAA0B,CAAC,CAAC;YACzD,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC1C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACxE,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC/E,CAAC;QACL,CAAC;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AAuBL,gBAAgB;AAChB,SAAS,oBAAoB,CACzB,OAAuE;IAEvE,OAAO,CAAC,CAAE,OAA4C,CAAC,YAAY,CAAC;AACxE,CAAC;AAED,gBAAgB;AAChB,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC5C,OAAuE;IAEvE,MAAM,EAAE,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,kBAAkB,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;IAE5F,MAAM,OAAO,GAAG,sCAAsC,CAAC;QACnD,eAAe,EAAE,mBAAmB,EAAE,QAAQ;QAC9C,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,mBAAmB,EAAE,OAAO;KACpD,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAClC,IAAW,EACX,mBAAmB,EAAE,QAAQ,IAAI,GAAG,EACpC,mBAAmB,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CACxE,CAAC;IAEF,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,YAAY,CAAC;YACxB,IAAI;YACJ,OAAO;YACP,GAAG,mBAAmB;SACzB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,YAAY,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,IAAI;QACJ,OAAO;QACP,GAAI,mBAA2C;KAClD,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;AACrC,sEAAsE;AACtE,IAA0B,EAC1B,QAAgB,EAChB,OAAe;IAEf,MAAM,IAAI,GACN,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAA0B,EAAE,EAAE,CACxD,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5E,CAAC,IAAI,EAAE,CAAC;IACb,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAsB,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI;SACN,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;QAClB,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,+CAA+C;gBACpE,+EAA+E,CACtF,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC"}
|
|
1
|
+
{"version":3,"file":"browser-crawler.js","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAeA,OAAO,EACH,YAAY,EACZ,8BAA8B,EAC9B,eAAe,EACf,yBAAyB,EACzB,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EACtB,YAAY,EACZ,sCAAsC,EACtC,YAAY,EACZ,8BAA8B,EAC9B,cAAc,EACd,UAAU,GACb,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAGpD,OAAO,EAAE,8BAA8B,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5F,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAqD3C,MAAM,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;AAE1D,MAAM,gBAAgB,GAAG,CAAI,GAAW,EAAE,GAAW,EAAK,EAAE,CAAE,GAA+B,CAAC,GAAG,CAAM,CAAC;AAkKxG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,OAAgB,cAepB,SAAQ,YAAwD;IAC9D;;OAEG;IACH,WAAW,CAA0C;IAErD,aAAa,CAA+C;IAEzC,iBAAiB,CAAU;IAC3B,aAAa,CAAU;IAEhC,uBAAuB,CAAS;IAChC,kBAAkB,CAAyB;IAC3C,mBAAmB,CAAyB;IAC5C,mBAAmB,CAAU;IAE7B,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC;QAChD,kBAAkB,EAAE,EAAE,CAAC,MAAM;QAC7B,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACxC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC;KACjF,CAAC;IAEF;;OAEG;IACH,YACI,OASC;QAED,EAAE,CAAC,OAAO,EAAE,uBAAuB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,mBAAmB,GAAG,IAAI,EAC1B,aAAa,GAAG,EAAE,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,QAAQ,EACR,aAAa,GAAG,KAAK,EACrB,iBAAiB,GAAG,KAAK,EACzB,sBAAsB,EACtB,aAAa,EACb,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,MAAM,SAAS,GAAG,CACd,MAAoD,EAChB,EAAE,CAAC,CAAC;YACxC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SACzF,CAAC,CAAC;QAEH,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,sBAAsB,EAAE,GAAG,EAAE;gBACzB,IAAI,QAAQ,GAAG,sBAAsB,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAE/F,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBACzC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC;gBAED,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAEjE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBAC1C,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC;gBAED,OAAO,QAAQ;qBACV,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;qBACtD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;qBAClE,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;YACD,aAAa,EAAE,aAAkE;SACpF,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,aAA4B,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzE,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,aAAa,EAAE,SAAS,EAAE,CAAC;YAC3B,IAAI,kBAAkB,CAAC,eAAe;gBAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;YACpG,kBAAkB,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAA6B;YAC3D,GAAI,kBAA0B;SACjC,CAAC,CAAC;IACP,CAAC;IAEkB,oBAAoB;QAInC,OAAO,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;YACrD,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YACnC,OAAO,EAAE,KAAK,EAAE,OAKf,EAAE,EAAE;gBACD,OAAO,CAAC,uBAAuB,CAAC,KAAK,IAAI,EAAE;oBACvC,6EAA6E;oBAC7E,2EAA2E;oBAC3E,8EAA8E;oBAC9E,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;wBAC9B,IAAI,CAAC,WAAW,CAAC,uBAAuB,CACpC,OAAO,CAAC,iBAEJ,CACP,CAAC;oBACN,CAAC;oBACD,MAAM,OAAO,CAAC,IAAI;yBACb,KAAK,EAAE;yBACP,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;gBACxF,CAAC,CAAC,CAAC;YACP,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,IAAgB,EAAE,SAAmB;QACjE,MAAM,cAAc,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAE,IAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC7F,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAErC,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC5B,eAA2E;QAE3E,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC;QAE3C,yGAAyG;QACzG,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC,IAAI,QAAQ,EAAE,MAAM,EAAE,KAAK,GAAG,EAAE,CAAC;YACrG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAElB,+IAA+I;YAC/I,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAE/E,IAAI,CAAC,cAAc;gBAAE,OAAO,KAAK,CAAC;YAClC,OAAO,iDAAiD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxF,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAC/E,MAAM,UAAU,GAAG,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3C,IAAI,cAAc;YAAE,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,OAAO,iCAAiC,UAAU,EAAE,CAAC;QAElG,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CACrB,eAAgC;QAIhC,MAAM,cAAc,GAAe;YAC/B,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,IAAI,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC;YACrC,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC;YAEpD,cAAc,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACzC,cAAc,CAAC,eAAe,GAAG,SAAS,EAAE,eAAe,CAAC;QAChE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAS,CAAC;QACtE,SAAS,EAAE,CAAC;QAEZ,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CACzE,IAAW,CACQ,CAAC;QAExB,MAAM,mBAAmB,GAAG,eAAe,CAAC,YAAY,CAAC;QAEzD,OAAO;YACH,IAAI;YACJ,IAAI,QAAQ;gBACR,MAAM,IAAI,KAAK,CACX,gMAAgM,CACnM,CAAC;YACN,CAAC;YACD,IAAI,WAAW;gBACX,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;YACnG,CAAC;YACD,iBAAiB,EAAE,yBAAyB;YAC5C,YAAY,EAAE,KAAK,EAAE,iBAAsC,EAAE,EAAE,EAAE;gBAC7D,OAAO,CAAC,MAAM,0BAA0B,CAAC;oBACrC,OAAO,EAAE,EAAE,GAAG,cAAc,EAAE,KAAK,EAAE,IAAI,CAAC,6BAA6B,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE;oBAChG,IAAI;oBACJ,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7E,gBAAgB,EAAE,IAAI,CAAC,oBAAoB;oBAC3C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;oBAClD,YAAY,EAAE,mBAAmB;iBACpC,CAAC,CAA2B,CAAC,CAAC,2BAA2B;YAC9D,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACpD,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE;oBACxC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,sBAAsB,CAC5B,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAA2B;gBAC5B,IAAI,QAAQ;oBACR,MAAM,IAAI,sBAAsB,CAC5B,sEAAsE,CACzE,CAAC;gBACN,CAAC;aACgB,CAAC;QAC1B,CAAC;QAED,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QAExD,OAAO;YACH,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAA4B;YAChF,CAAC,oBAAoB,CAAC,EAAE,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC;SACrD,CAAC;IACrC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,eAAwB;QAC3C,SAAS,EAAE,CAAC;QAEZ,MAAM,WAAW,GAAG,eAAe,CAAC,WAA0B,CAAC;QAC/D,MAAM,kBAAkB,GAAG,gBAAgB,CAAS,eAAe,EAAE,oBAAoB,CAAC,CAAC;QAC3F,MAAM,iBAAiB,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAEpF,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,kBAAkB,EAAE,iBAAiB,CAAC,CAAC;QAEjF,IAAI,QAA8B,CAAC;QACnC,IAAI,CAAC;YACD,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC,IAAI,SAAS,CAAC;QAC1F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAc,CAAC,CAAC;YACrE,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YACnD,IAAI,CAAC,kBAAkB,CAAC,KAAc,CAAC,CAAC;YACxC,MAAM,KAAK,CAAC;QAChB,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvD,OAAO,EAAE,QAAQ,EAAsB,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QACrD,SAAS,EAAE,CAAC;QAEZ,IAAI,QAA8B,CAAC;QACnC,IAAI,CAAC;YACD,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACL,sFAAsF;YACtF,+EAA+E;QACnF,CAAC;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QACtD,SAAS,EAAE,CAAC;QAEZ,oEAAoE;QACpE,IAAI,IAAI,CAAC,mBAAmB,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;YACtD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YACzF,SAAS,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,eAAe,CAAC,OAAO,CAAC,SAAU,CAAC;YAC/C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACD,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE;wBACzF,WAAW,EAAE,KAAK;qBACrB,CAAC,CAAC;gBACP,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAA0B,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;gBACpE,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,eAAe,CAAC,OAAiC,EAAsB,CAAC;IAC9F,CAAC;IAEO,KAAK,CAAC,6BAA6B,CACvC,eAA2E;QAE3E,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,EAAE,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,eAAgC;QAC9D,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,eAAe,CAAC;QAC7D,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAA0B,EACrE,eAAuB,EACvB,gBAAwB;QAExB,MAAM,aAAa,GAAG,OAAO,EAAE,SAAS,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,8BAA8B,CAAC,IAAI,EAAE,CAAC;QAC/G,MAAM,qBAAqB,GAAG,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtG,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QAExG,MAAM,iBAAiB,CAAC,UAAU,CAC9B,IAAI,EACJ,CAAC,GAAG,aAAa,EAAE,GAAG,qBAAqB,EAAE,GAAG,sBAAsB,CAAC;aAClE,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,OAAO,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,IAAI,CAAC;aACxE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CACvE,CAAC;IACN,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,wBAAwB,CAAC,eAAuC,EAAE,KAAY;QAC1F,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,EAAE,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC7C,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,KAAY;QACrC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAW,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAOO,KAAK,CAAC,eAAe,CACzB,QAA8B,EAC9B,eAAuC;QAEvC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAEnD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxE,MAAM,MAAM,GAAW,QAAQ,CAAC,MAAM,EAAE,CAAC;YAEzC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAEtC,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,IAAI,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBAClD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;gBACtE,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,0BAA0B,CAAC,CAAC;YACzD,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC1C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACxE,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC/E,CAAC;QACL,CAAC;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AAuBL,gBAAgB;AAChB,SAAS,oBAAoB,CACzB,OAAuE;IAEvE,OAAO,CAAC,CAAE,OAA4C,CAAC,YAAY,CAAC;AACxE,CAAC;AAED,gBAAgB;AAChB,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC5C,OAAuE;IAEvE,MAAM,EAAE,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,kBAAkB,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;IAE5F,MAAM,OAAO,GAAG,sCAAsC,CAAC;QACnD,eAAe,EAAE,mBAAmB,EAAE,QAAQ;QAC9C,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,mBAAmB,EAAE,OAAO;KACpD,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAClC,IAAW,EACX,mBAAmB,EAAE,QAAQ,IAAI,GAAG,EACpC,mBAAmB,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CACxE,CAAC;IAEF,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,YAAY,CAAC;YACxB,IAAI;YACJ,OAAO;YACP,GAAG,mBAAmB;SACzB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,YAAY,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,IAAI;QACJ,OAAO;QACP,GAAI,mBAA2C;KAClD,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;AACrC,sEAAsE;AACtE,IAA0B,EAC1B,QAAgB,EAChB,OAAe;IAEf,MAAM,IAAI,GACN,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAA0B,EAAE,EAAE,CACxD,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5E,CAAC,IAAI,EAAE,CAAC;IACb,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAsB,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI;SACN,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;QAClB,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,+CAA+C;gBACpE,+EAA+E,CACtF,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.64",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.3.2",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.64",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.64",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.64",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.64",
|
|
55
55
|
"ow": "^2.0.0",
|
|
56
56
|
"tslib": "^2.8.1",
|
|
57
57
|
"type-fest": "^4.41.0"
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "76ba0fe1101b1e14fb5e1962540107775ad4e513"
|
|
79
79
|
}
|