@crawlee/browser 4.0.0-beta.4 → 4.0.0-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/internals/browser-crawler.d.ts +78 -53
- package/internals/browser-crawler.d.ts.map +1 -1
- package/internals/browser-crawler.js +150 -189
- package/internals/browser-crawler.js.map +1 -1
- package/internals/browser-launcher.d.ts +7 -0
- package/internals/browser-launcher.d.ts.map +1 -1
- package/internals/browser-launcher.js +1 -0
- package/internals/browser-launcher.js.map +1 -1
- package/package.json +6 -6
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { BasicCrawler, ContextPipeline, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
|
|
2
2
|
import { BrowserPool } from '@crawlee/browser-pool';
|
|
3
3
|
import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
|
|
4
4
|
import ow from 'ow';
|
|
5
|
-
import {
|
|
5
|
+
import { tryCancel } from '@apify/timeout';
|
|
6
6
|
/**
|
|
7
7
|
* Provides a simple framework for parallel crawling of web pages
|
|
8
8
|
* using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
|
|
@@ -43,20 +43,14 @@ import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
|
43
43
|
* @category Crawlers
|
|
44
44
|
*/
|
|
45
45
|
export class BrowserCrawler extends BasicCrawler {
|
|
46
|
-
config;
|
|
47
|
-
/**
|
|
48
|
-
* A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
|
|
49
|
-
* Only available if used by the crawler.
|
|
50
|
-
*/
|
|
51
|
-
proxyConfiguration;
|
|
52
46
|
/**
|
|
53
47
|
* A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
|
|
54
48
|
*/
|
|
55
49
|
browserPool;
|
|
56
50
|
launchContext;
|
|
57
|
-
|
|
51
|
+
ignoreShadowRoots;
|
|
52
|
+
ignoreIframes;
|
|
58
53
|
navigationTimeoutMillis;
|
|
59
|
-
requestHandlerTimeoutInnerMillis;
|
|
60
54
|
preNavigationHooks;
|
|
61
55
|
postNavigationHooks;
|
|
62
56
|
persistCookiesPerSession;
|
|
@@ -70,64 +64,54 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
70
64
|
browserPoolOptions: ow.object,
|
|
71
65
|
sessionPoolOptions: ow.optional.object,
|
|
72
66
|
persistCookiesPerSession: ow.optional.boolean,
|
|
73
|
-
useSessionPool: ow.optional.boolean,
|
|
74
67
|
proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
|
|
75
|
-
ignoreShadowRoots: ow.optional.boolean,
|
|
76
|
-
ignoreIframes: ow.optional.boolean,
|
|
77
68
|
};
|
|
78
69
|
/**
|
|
79
70
|
* All `BrowserCrawler` parameters are passed via an options object.
|
|
80
71
|
*/
|
|
81
|
-
constructor(options
|
|
72
|
+
constructor(options) {
|
|
82
73
|
ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
|
|
83
|
-
const { navigationTimeoutSecs = 60,
|
|
74
|
+
const { navigationTimeoutSecs = 60, persistCookiesPerSession = true, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, proxyConfiguration, ...basicCrawlerOptions } = options;
|
|
84
75
|
super({
|
|
85
76
|
...basicCrawlerOptions,
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
this.failedRequestHandler = failedRequestHandler; // FIXME is this even needed?
|
|
93
|
-
// Cookies should be persisted per session only if session pool is used
|
|
94
|
-
if (!this.useSessionPool && persistCookiesPerSession) {
|
|
95
|
-
throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
|
|
96
|
-
}
|
|
77
|
+
contextPipelineBuilder: () => contextPipelineBuilder()
|
|
78
|
+
.compose({ action: this.performNavigation.bind(this) })
|
|
79
|
+
.compose({ action: this.handleBlockedRequestByContent.bind(this) })
|
|
80
|
+
.compose({ action: this.restoreRequestState.bind(this) }),
|
|
81
|
+
extendContext: extendContext,
|
|
82
|
+
});
|
|
97
83
|
this.launchContext = launchContext;
|
|
98
84
|
this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
|
|
99
|
-
this.requestHandlerTimeoutInnerMillis = requestHandlerTimeoutSecs * 1000;
|
|
100
85
|
this.proxyConfiguration = proxyConfiguration;
|
|
101
86
|
this.preNavigationHooks = preNavigationHooks;
|
|
102
87
|
this.postNavigationHooks = postNavigationHooks;
|
|
88
|
+
this.ignoreIframes = ignoreIframes;
|
|
89
|
+
this.ignoreShadowRoots = ignoreShadowRoots;
|
|
103
90
|
if (headless != null) {
|
|
104
91
|
this.launchContext.launchOptions ??= {};
|
|
105
92
|
this.launchContext.launchOptions.headless = headless;
|
|
106
93
|
}
|
|
107
|
-
|
|
108
|
-
this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
this.persistCookiesPerSession = false;
|
|
112
|
-
}
|
|
94
|
+
this.persistCookiesPerSession = persistCookiesPerSession;
|
|
113
95
|
if (launchContext?.userAgent) {
|
|
114
96
|
if (browserPoolOptions.useFingerprints)
|
|
115
97
|
this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
|
|
116
98
|
browserPoolOptions.useFingerprints = false;
|
|
117
99
|
}
|
|
118
|
-
const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
|
|
119
100
|
this.browserPool = new BrowserPool({
|
|
120
|
-
...
|
|
121
|
-
preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
|
|
122
|
-
postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
|
|
101
|
+
...browserPoolOptions,
|
|
123
102
|
});
|
|
124
103
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
104
|
+
buildContextPipeline() {
|
|
105
|
+
return ContextPipeline.create().compose({
|
|
106
|
+
action: this.preparePage.bind(this),
|
|
107
|
+
cleanup: async (context) => {
|
|
108
|
+
context.registerDeferredCleanup(async () => {
|
|
109
|
+
await context.page
|
|
110
|
+
.close()
|
|
111
|
+
.catch((error) => this.log.debug('Error while closing page', { error }));
|
|
112
|
+
});
|
|
113
|
+
},
|
|
114
|
+
});
|
|
131
115
|
}
|
|
132
116
|
async containsSelectors(page, selectors) {
|
|
133
117
|
const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
|
|
@@ -138,12 +122,6 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
138
122
|
}
|
|
139
123
|
async isRequestBlocked(crawlingContext) {
|
|
140
124
|
const { page, response } = crawlingContext;
|
|
141
|
-
const blockedStatusCodes =
|
|
142
|
-
// eslint-disable-next-line dot-notation
|
|
143
|
-
(this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
|
|
144
|
-
? // eslint-disable-next-line dot-notation
|
|
145
|
-
this.sessionPool['blockedStatusCodes']
|
|
146
|
-
: DEFAULT_BLOCKED_STATUS_CODES;
|
|
147
125
|
// Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
|
|
148
126
|
if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
|
|
149
127
|
await sleep(5000);
|
|
@@ -154,112 +132,64 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
154
132
|
return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
|
|
155
133
|
}
|
|
156
134
|
const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
|
|
157
|
-
const
|
|
135
|
+
const statusCode = response?.status() ?? 0;
|
|
158
136
|
if (foundSelectors)
|
|
159
137
|
return `Found selectors: ${foundSelectors.join(', ')}`;
|
|
160
|
-
if (
|
|
161
|
-
return `Received blocked status code: ${
|
|
138
|
+
if (this.blockedStatusCodes.has(statusCode))
|
|
139
|
+
return `Received blocked status code: ${statusCode}`;
|
|
162
140
|
return false;
|
|
163
141
|
}
|
|
164
|
-
|
|
165
|
-
* Wrapper around requestHandler that opens and closes pages etc.
|
|
166
|
-
*/
|
|
167
|
-
async _runRequestHandler(crawlingContext) {
|
|
142
|
+
async preparePage(crawlingContext) {
|
|
168
143
|
const newPageOptions = {
|
|
169
144
|
id: crawlingContext.id,
|
|
170
145
|
};
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
const { session } = crawlingContext;
|
|
174
|
-
const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, {
|
|
175
|
-
request: crawlingContext.request,
|
|
176
|
-
});
|
|
177
|
-
crawlingContext.proxyInfo = proxyInfo;
|
|
146
|
+
if (crawlingContext.session?.proxyInfo) {
|
|
147
|
+
const proxyInfo = crawlingContext.session.proxyInfo;
|
|
178
148
|
newPageOptions.proxyUrl = proxyInfo?.url;
|
|
179
149
|
newPageOptions.proxyTier = proxyInfo?.proxyTier;
|
|
180
|
-
|
|
181
|
-
/**
|
|
182
|
-
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
183
|
-
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
184
|
-
*/
|
|
185
|
-
newPageOptions.pageOptions = {
|
|
186
|
-
ignoreHTTPSErrors: true,
|
|
187
|
-
acceptInsecureCerts: true,
|
|
188
|
-
};
|
|
189
|
-
}
|
|
150
|
+
newPageOptions.ignoreTlsErrors = proxyInfo?.ignoreTlsErrors;
|
|
190
151
|
}
|
|
191
152
|
const page = (await this.browserPool.newPage(newPageOptions));
|
|
192
153
|
tryCancel();
|
|
193
|
-
this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
|
|
194
|
-
// DO NOT MOVE THIS LINE ABOVE!
|
|
195
|
-
// `enhanceCrawlingContextWithPageInfo` gives us a valid session.
|
|
196
|
-
// For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.
|
|
197
|
-
// So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.
|
|
198
|
-
const { request, session } = crawlingContext;
|
|
199
|
-
if (!request.skipNavigation) {
|
|
200
|
-
await this._handleNavigation(crawlingContext);
|
|
201
|
-
tryCancel();
|
|
202
|
-
await this._responseHandler(crawlingContext);
|
|
203
|
-
tryCancel();
|
|
204
|
-
// save cookies
|
|
205
|
-
// TODO: Should we save the cookies also after/only the handle page?
|
|
206
|
-
if (this.persistCookiesPerSession) {
|
|
207
|
-
const cookies = await crawlingContext.browserController.getCookies(page);
|
|
208
|
-
tryCancel();
|
|
209
|
-
session?.setCookies(cookies, request.loadedUrl);
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
if (!this.requestMatchesEnqueueStrategy(request)) {
|
|
213
|
-
this.log.debug(
|
|
214
|
-
// eslint-disable-next-line dot-notation
|
|
215
|
-
`Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`);
|
|
216
|
-
request.noRetry = true;
|
|
217
|
-
request.state = RequestState.SKIPPED;
|
|
218
|
-
return;
|
|
219
|
-
}
|
|
220
|
-
if (this.retryOnBlocked) {
|
|
221
|
-
const error = await this.isRequestBlocked(crawlingContext);
|
|
222
|
-
if (error)
|
|
223
|
-
throw new SessionError(error);
|
|
224
|
-
}
|
|
225
|
-
request.state = RequestState.REQUEST_HANDLER;
|
|
226
|
-
try {
|
|
227
|
-
await addTimeoutToPromise(async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)), this.requestHandlerTimeoutInnerMillis, `requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`);
|
|
228
|
-
request.state = RequestState.DONE;
|
|
229
|
-
}
|
|
230
|
-
catch (e) {
|
|
231
|
-
request.state = RequestState.ERROR;
|
|
232
|
-
throw e;
|
|
233
|
-
}
|
|
234
|
-
tryCancel();
|
|
235
|
-
}
|
|
236
|
-
_enhanceCrawlingContextWithPageInfo(crawlingContext, page, createNewSession) {
|
|
237
|
-
crawlingContext.page = page;
|
|
238
|
-
// This switch is because the crawlingContexts are created on per request basis.
|
|
239
|
-
// However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
|
|
240
|
-
// We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
|
|
241
|
-
// which configures proxy and session for every new request
|
|
242
154
|
const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
|
|
243
|
-
crawlingContext.
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
155
|
+
this.addSessionRetiredListener(crawlingContext.session, browserControllerInstance);
|
|
156
|
+
const contextEnqueueLinks = crawlingContext.enqueueLinks;
|
|
157
|
+
return {
|
|
158
|
+
page,
|
|
159
|
+
get response() {
|
|
160
|
+
throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
|
|
161
|
+
},
|
|
162
|
+
browserController: browserControllerInstance,
|
|
163
|
+
enqueueLinks: async (enqueueOptions = {}) => {
|
|
164
|
+
return (await browserCrawlerEnqueueLinks({
|
|
165
|
+
options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
|
|
166
|
+
page,
|
|
167
|
+
requestQueue: await this.getRequestQueue(),
|
|
168
|
+
robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
|
|
169
|
+
onSkippedRequest: this.handleSkippedRequest,
|
|
170
|
+
originalRequestUrl: crawlingContext.request.url,
|
|
171
|
+
finalRequestUrl: crawlingContext.request.loadedUrl,
|
|
172
|
+
enqueueLinks: contextEnqueueLinks,
|
|
173
|
+
})); // TODO make this type safe
|
|
174
|
+
},
|
|
260
175
|
};
|
|
261
176
|
}
|
|
262
|
-
async
|
|
177
|
+
async performNavigation(crawlingContext) {
|
|
178
|
+
if (crawlingContext.request.skipNavigation) {
|
|
179
|
+
return {
|
|
180
|
+
request: new Proxy(crawlingContext.request, {
|
|
181
|
+
get(target, propertyName, receiver) {
|
|
182
|
+
if (propertyName === 'loadedUrl') {
|
|
183
|
+
throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
184
|
+
}
|
|
185
|
+
return Reflect.get(target, propertyName, receiver);
|
|
186
|
+
},
|
|
187
|
+
}),
|
|
188
|
+
get response() {
|
|
189
|
+
throw new Error('The `response` property is not available - `skipNavigation` was used');
|
|
190
|
+
},
|
|
191
|
+
};
|
|
192
|
+
}
|
|
263
193
|
const gotoOptions = { timeout: this.navigationTimeoutMillis };
|
|
264
194
|
const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
|
|
265
195
|
crawlingContext.request.state = RequestState.BEFORE_NAV;
|
|
@@ -267,8 +197,9 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
267
197
|
tryCancel();
|
|
268
198
|
const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
|
|
269
199
|
await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
|
|
200
|
+
let response;
|
|
270
201
|
try {
|
|
271
|
-
|
|
202
|
+
response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
|
|
272
203
|
}
|
|
273
204
|
catch (error) {
|
|
274
205
|
await this._handleNavigationTimeout(crawlingContext, error);
|
|
@@ -279,6 +210,36 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
279
210
|
tryCancel();
|
|
280
211
|
crawlingContext.request.state = RequestState.AFTER_NAV;
|
|
281
212
|
await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
|
|
213
|
+
await this.processResponse(response, crawlingContext);
|
|
214
|
+
tryCancel();
|
|
215
|
+
// save cookies
|
|
216
|
+
// TODO: Should we save the cookies also after/only the handle page?
|
|
217
|
+
if (this.persistCookiesPerSession) {
|
|
218
|
+
const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
|
|
219
|
+
tryCancel();
|
|
220
|
+
crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl);
|
|
221
|
+
}
|
|
222
|
+
if (response !== undefined) {
|
|
223
|
+
return {
|
|
224
|
+
request: crawlingContext.request,
|
|
225
|
+
response,
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
return {
|
|
229
|
+
request: crawlingContext.request,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
async handleBlockedRequestByContent(crawlingContext) {
|
|
233
|
+
if (this.retryOnBlocked) {
|
|
234
|
+
const error = await this.isRequestBlocked(crawlingContext);
|
|
235
|
+
if (error)
|
|
236
|
+
throw new SessionError(error);
|
|
237
|
+
}
|
|
238
|
+
return {};
|
|
239
|
+
}
|
|
240
|
+
async restoreRequestState(crawlingContext) {
|
|
241
|
+
crawlingContext.request.state = RequestState.REQUEST_HANDLER;
|
|
242
|
+
return {};
|
|
282
243
|
}
|
|
283
244
|
async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
|
|
284
245
|
const sessionCookie = session?.getCookies(request.url) ?? [];
|
|
@@ -306,18 +267,21 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
306
267
|
throw new SessionError(this._getMessageFromError(error));
|
|
307
268
|
}
|
|
308
269
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
*/
|
|
312
|
-
async _responseHandler(crawlingContext) {
|
|
313
|
-
const { response, session, request, page } = crawlingContext;
|
|
270
|
+
async processResponse(response, crawlingContext) {
|
|
271
|
+
const { session, request, page } = crawlingContext;
|
|
314
272
|
if (typeof response === 'object' && typeof response.status === 'function') {
|
|
315
273
|
const status = response.status();
|
|
316
274
|
this.stats.registerStatusCode(status);
|
|
275
|
+
if (this.isErrorStatusCode(status)) {
|
|
276
|
+
if (this.additionalHttpErrorStatusCodes.has(status)) {
|
|
277
|
+
throw new Error(`${status} - Error status code was set by user.`);
|
|
278
|
+
}
|
|
279
|
+
throw new Error(`${status} - Internal Server Error`);
|
|
280
|
+
}
|
|
317
281
|
}
|
|
318
282
|
if (this.sessionPool && response && session) {
|
|
319
283
|
if (typeof response === 'object' && typeof response.status === 'function') {
|
|
320
|
-
this._throwOnBlockedRequest(
|
|
284
|
+
this._throwOnBlockedRequest(response.status());
|
|
321
285
|
}
|
|
322
286
|
else {
|
|
323
287
|
this.log.debug('Got a malformed Browser response.', { request, response });
|
|
@@ -325,42 +289,27 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
325
289
|
}
|
|
326
290
|
request.loadedUrl = await page.url();
|
|
327
291
|
}
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
if (this.sessionPool) {
|
|
331
|
-
|
|
332
|
-
}
|
|
333
|
-
if (this.proxyConfiguration && !launchContext.proxyUrl) {
|
|
334
|
-
const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id, {
|
|
335
|
-
proxyTier: launchContext.proxyTier ?? undefined,
|
|
336
|
-
});
|
|
337
|
-
launchContext.proxyUrl = proxyInfo?.url;
|
|
338
|
-
launchContextExtends.proxyInfo = proxyInfo;
|
|
339
|
-
// Disable SSL verification for MITM proxies
|
|
340
|
-
if (this.proxyConfiguration.isManInTheMiddle) {
|
|
341
|
-
/**
|
|
342
|
-
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
|
|
343
|
-
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
|
|
344
|
-
*/
|
|
345
|
-
launchContext.launchOptions.ignoreHTTPSErrors = true;
|
|
346
|
-
launchContext.launchOptions.acceptInsecureCerts = true;
|
|
347
|
-
}
|
|
292
|
+
browserSessionIds = new WeakMap();
|
|
293
|
+
addSessionRetiredListener(session, browserController) {
|
|
294
|
+
if (!this.sessionPool) {
|
|
295
|
+
return;
|
|
348
296
|
}
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
const listener = (session) => {
|
|
354
|
-
const { launchContext } = browserController;
|
|
355
|
-
if (session.id === launchContext.session.id) {
|
|
356
|
-
this.browserPool.retireBrowserController(browserController);
|
|
357
|
-
}
|
|
358
|
-
};
|
|
359
|
-
this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
|
|
360
|
-
browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
|
|
361
|
-
return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
|
|
362
|
-
});
|
|
297
|
+
let sessionIds = this.browserSessionIds.get(browserController);
|
|
298
|
+
if (sessionIds) {
|
|
299
|
+
sessionIds.add(session.id);
|
|
300
|
+
return;
|
|
363
301
|
}
|
|
302
|
+
sessionIds = new Set([session.id]);
|
|
303
|
+
this.browserSessionIds.set(browserController, sessionIds);
|
|
304
|
+
const listener = (retired) => {
|
|
305
|
+
if (this.browserSessionIds.get(browserController)?.has(retired.id)) {
|
|
306
|
+
this.browserPool.retireBrowserController(browserController);
|
|
307
|
+
}
|
|
308
|
+
};
|
|
309
|
+
this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
|
|
310
|
+
browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
|
|
311
|
+
return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
|
|
312
|
+
});
|
|
364
313
|
}
|
|
365
314
|
/**
|
|
366
315
|
* Function for cleaning up after all requests are processed.
|
|
@@ -372,21 +321,33 @@ export class BrowserCrawler extends BasicCrawler {
|
|
|
372
321
|
}
|
|
373
322
|
}
|
|
374
323
|
/** @internal */
|
|
375
|
-
|
|
324
|
+
function containsEnqueueLinks(options) {
|
|
325
|
+
return !!options.enqueueLinks;
|
|
326
|
+
}
|
|
327
|
+
/** @internal */
|
|
328
|
+
export async function browserCrawlerEnqueueLinks(options) {
|
|
329
|
+
const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
|
|
376
330
|
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
377
|
-
enqueueStrategy:
|
|
331
|
+
enqueueStrategy: enqueueLinksOptions?.strategy,
|
|
378
332
|
finalRequestUrl,
|
|
379
333
|
originalRequestUrl,
|
|
380
|
-
userProvidedBaseUrl:
|
|
334
|
+
userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
|
|
381
335
|
});
|
|
382
|
-
const urls = await extractUrlsFromPage(page,
|
|
336
|
+
const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
|
|
337
|
+
if (containsEnqueueLinks(options)) {
|
|
338
|
+
return options.enqueueLinks({
|
|
339
|
+
urls,
|
|
340
|
+
baseUrl,
|
|
341
|
+
...enqueueLinksOptions,
|
|
342
|
+
});
|
|
343
|
+
}
|
|
383
344
|
return enqueueLinks({
|
|
384
|
-
requestQueue,
|
|
385
|
-
robotsTxtFile,
|
|
386
|
-
onSkippedRequest,
|
|
345
|
+
requestQueue: options.requestQueue,
|
|
346
|
+
robotsTxtFile: options.robotsTxtFile,
|
|
347
|
+
onSkippedRequest: options.onSkippedRequest,
|
|
387
348
|
urls,
|
|
388
349
|
baseUrl,
|
|
389
|
-
...
|
|
350
|
+
...enqueueLinksOptions,
|
|
390
351
|
});
|
|
391
352
|
}
|
|
392
353
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-crawler.js","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAeA,OAAO,EACH,iCAAiC,EACjC,YAAY,EACZ,oBAAoB,IAAI,4BAA4B,EACpD,aAAa,EACb,yBAAyB,EACzB,YAAY,EACZ,qBAAqB,EACrB,oBAAoB,EACpB,YAAY,EACZ,sCAAsC,EACtC,YAAY,EACZ,cAAc,EACd,UAAU,GACb,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAA6B,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAG/E,OAAO,EAAE,8BAA8B,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5F,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAiLhE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,OAAgB,cAKpB,SAAQ,YAAqB;IA4CL;IA3CtB;;;OAGG;IACH,kBAAkB,CAAsB;IAExC;;OAEG;IACH,WAAW,CAA0C;IAErD,aAAa,CAA+C;IAElD,0BAA0B,CAAkC;IAC5D,uBAAuB,CAAS;IAChC,gCAAgC,CAAS;IACzC,kBAAkB,CAAyB;IAC3C,mBAAmB,CAAyB;IAC5C,wBAAwB,CAAU;IAElC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC;QAChD,kBAAkB,EAAE,EAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC;QAC9E,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACtC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;KACrC,CAAC;IAEF;;OAEG;IACH,YACI,UAA0C,EAAE,EAC1B,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,uBAAuB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,yBAAyB,GAAG,EAAE,EAC9B,wBAAwB,EACxB,kBAAkB,EAClB,aAAa,GAAG,EAAE,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,cAAc,EACd,oBAAoB,EACpB,QAAQ,EACR,iBAAiB,EACjB,aAAa,EACb,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,cAAc,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAI,IAAkB,CAAC;YAClF,yBAAyB,EACrB,qBAAqB,GAAG,yBAAyB,GAAG,iCAAiC;SAC5F,EACD,MAAM,CACT,CAAC;QA5BgB,WAAM,GAAN,MAAM,CAAkC;QA8B1D,YAAY;QACZ,IAAI,CAAC,0BAA0B,GAAI,cAAsB,IAAI,IAAI,CAAC,MAAM,CAAC;QACzE,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC,CAAC,6BAA6B;QAE/E,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,gCAAgC,GAAG,yBAAyB,GAAG,IAAI,CAAC;QACzE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,aAA4B,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzE,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;QAED,IAAI,aAAa,EAAE,SAAS,EAAE,CAAC;YAC3B,IAAI,kBAAkB,CAAC,eAAe;gBAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;YACpG,kBAAkB,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/C,CAAC;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAElF,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAA6B;YAC3D,GAAI,IAAY;YAChB,cAAc,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,CAAC;YACzE,eAAe,EAAE,CAAC,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,eAAe,CAAC;SACzF,CAAC,CAAC;IACP,CAAC;IAEkB,KAAK,CAAC,eAAe,CAAC,eAAwB;QAC7D,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAEjC,+BAA+B;QAC/B,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACtG,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,IAAgB,EAAE,SAAmB;QACjE,MAAM,cAAc,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAE,IAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC7F,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAErC,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,CAAC;IAEkB,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QAC9D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC;QAE3C,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,4BAA4B,CAAC;QAEvC,yGAAyG;QACzG,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC,IAAI,QAAQ,EAAE,MAAM,EAAE,KAAK,GAAG,EAAE,CAAC;YACrG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAElB,+IAA+I;YAC/I,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAE/E,IAAI,CAAC,cAAc;gBAAE,OAAO,KAAK,CAAC;YAClC,OAAO,iDAAiD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxF,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAC/E,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QAE1F,IAAI,cAAc;YAAE,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,IAAI,iBAAiB;YAAE,OAAO,iCAAiC,iBAAiB,EAAE,CAAC;QAEnF,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,cAAc,GAAe;YAC/B,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAEhE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;YAEpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,EAAE;gBACtE,OAAO,EAAE,eAAe,CAAC,OAAO;aACnC,CAAC,CAAC;YACH,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;YAEtC,cAAc,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACzC,cAAc,CAAC,SAAS,GAAG,SAAS,EAAE,SAAS,CAAC;YAEhD,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;gBAC3C;;;mBAGG;gBACH,cAAc,CAAC,WAAW,GAAG;oBACzB,iBAAiB,EAAE,IAAI;oBACvB,mBAAmB,EAAE,IAAI;iBAC5B,CAAC;YACN,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAe,CAAC;QAC5E,SAAS,EAAE,CAAC;QACZ,IAAI,CAAC,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAEnF,+BAA+B;QAC/B,iEAAiE;QACjE,6EAA6E;QAC7E,+GAA+G;QAC/G,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YAEZ,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC7C,SAAS,EAAE,CAAC;YAEZ,eAAe;YACf,oEAAoE;YACpE,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACzE,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,SAAU,CAAC,CAAC;YACrD,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,GAAG,CAAC,KAAK;YACV,wCAAwC;YACxC,oBAAoB,OAAO,CAAC,EAAE,mBAAmB,OAAO,CAAC,GAAG,mBAAmB,OAAO,CAAC,SAAS,qDAAqD,OAAO,CAAC,iBAAiB,CAAC,IAAI,CACtL,CAAC;YAEF,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC;YAErC,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,eAAe,CAAC;QAC7C,IAAI,CAAC;YACD,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,eAAyC,CAAC,CAAC,EACvG,IAAI,CAAC,gCAAgC,EACrC,kCAAkC,IAAI,CAAC,gCAAgC,GAAG,IAAI,WAAW,CAC5F,CAAC;YAEF,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC;QACtC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,CAAC;QACZ,CAAC;QACD,SAAS,EAAE,CAAC;IAChB,CAAC;IAES,mCAAmC,CACzC,eAAwB,EACxB,IAAgB,EAChB,gBAA0B;QAE1B,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAE5B,gFAAgF;QAChF,6HAA6H;QAC7H,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CACzE,IAAW,CACkB,CAAC;QAClC,eAAe,CAAC,iBAAiB,GAAG,yBAAyB,CAAC;QAE9D,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpB,eAAe,CAAC,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,OAAkB,CAAC;QACzF,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,CAAC;YAC7B,eAAe,CAAC,SAAS,GAAG,yBAAyB,CAAC,aAAa,CAAC,SAAsB,CAAC;QAC/F,CAAC;QAED,eAAe,CAAC,YAAY,GAAG,KAAK,EAAE,cAAc,EAAE,EAAE;YACpD,OAAO,0BAA0B,CAAC;gBAC9B,OAAO,EAAE,cAAc;gBACvB,IAAI;gBACJ,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;gBAC1C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;gBAC7E,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;gBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;aACrD,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAA4B,CAAC;QAExF,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE5F,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE7F,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEjG,IAAI,CAAC;YACD,eAAe,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC,IAAI,SAAS,CAAC;QAC1G,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAc,CAAC,CAAC;YAErE,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YAEnD,IAAI,CAAC,kBAAkB,CAAC,KAAc,CAAC,CAAC;YACxC,MAAM,KAAK,CAAC;QAChB,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IAES,KAAK,CAAC,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAW,EACtD,eAAuB,EACvB,gBAAwB;QAExB,MAAM,aAAa,GAAG,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,qBAAqB,GAAG,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtG,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QAExG,MAAM,iBAAiB,CAAC,UAAU,CAC9B,IAAI,EACJ,CAAC,GAAG,aAAa,EAAE,GAAG,qBAAqB,EAAE,GAAG,sBAAsB,CAAC;aAClE,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,OAAO,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,IAAI,CAAC;aACxE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CACvE,CAAC;IACN,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,wBAAwB,CAAC,eAAwB,EAAE,KAAY;QAC3E,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACrD,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,KAAY;QACrC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAW,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAOD;;OAEG;IACO,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QACrD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAE7D,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxE,MAAM,MAAM,GAAW,QAAQ,CAAC,MAAM,EAAE,CAAC;YAEzC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC1C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACxE,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5D,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC/E,CAAC;QACL,CAAC;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAES,KAAK,CAAC,oBAAoB,CAAC,OAAe,EAAE,aAA4B;QAC9E,MAAM,oBAAoB,GAAiD,EAAE,CAAC;QAE9E,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QACvE,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,EAAE,EAAE,EAAE;gBAC3F,SAAS,EAAG,aAAa,CAAC,SAAoB,IAAI,SAAS;aAC9D,CAAC,CAAC;YACH,aAAa,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACxC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;gBAC3C;;;mBAGG;gBACF,aAAa,CAAC,aAA4B,CAAC,iBAAiB,GAAG,IAAI,CAAC;gBACpE,aAAa,CAAC,aAA4B,CAAC,mBAAmB,GAAG,IAAI,CAAC;YAC3E,CAAC;QACL,CAAC;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAES,+BAA+B,CAAC,OAAe,EAAE,iBAA+C;QACtG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,EAAE;gBAClC,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAM,aAAa,CAAC,OAAmB,CAAC,EAAE,EAAE,CAAC;oBACvD,IAAI,CAAC,WAAW,CAAC,uBAAuB,CACpC,iBAEI,CACP,CAAC;gBACN,CAAC;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;YACrD,iBAAiB,CAAC,EAAE,iEAA2C,GAAG,EAAE;gBAChE,OAAO,IAAI,CAAC,WAAY,CAAC,cAAc,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AAcL,gBAAgB;AAChB,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,kBAAkB,EAClB,eAAe,GACW;IAC1B,MAAM,OAAO,GAAG,sCAAsC,CAAC;QACnD,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAClC,IAAW,EACX,OAAO,EAAE,QAAQ,IAAI,GAAG,EACxB,OAAO,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CAC5D,CAAC;IAEF,OAAO,YAAY,CAAC;QAChB,YAAY;QACZ,aAAa;QACb,gBAAgB;QAChB,IAAI;QACJ,OAAO;QACP,GAAI,OAA+B;KACtC,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;AACrC,sEAAsE;AACtE,IAA0B,EAC1B,QAAgB,EAChB,OAAe;IAEf,MAAM,IAAI,GACN,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAA0B,EAAE,EAAE,CACxD,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5E,CAAC,IAAI,EAAE,CAAC;IACb,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAsB,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI;SACN,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;QAClB,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,+CAA+C;gBACpE,+EAA+E,CACtF,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC"}
|
|
1
|
+
{"version":3,"file":"browser-crawler.js","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AAeA,OAAO,EACH,YAAY,EACZ,eAAe,EACf,yBAAyB,EACzB,YAAY,EACZ,qBAAqB,EACrB,oBAAoB,EACpB,YAAY,EACZ,sCAAsC,EACtC,YAAY,EACZ,cAAc,EACd,UAAU,GACb,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAA6B,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAG/E,OAAO,EAAE,8BAA8B,EAAE,mBAAmB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5F,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAwM3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,OAAgB,cAepB,SAAQ,YAAwD;IAC9D;;OAEG;IACH,WAAW,CAA0C;IAErD,aAAa,CAA+C;IAEzC,iBAAiB,CAAU;IAC3B,aAAa,CAAU;IAEhC,uBAAuB,CAAS;IAChC,kBAAkB,CAAyB;IAC3C,mBAAmB,CAAyB;IAC5C,wBAAwB,CAAU;IAElC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC;QAChD,kBAAkB,EAAE,EAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC;KACjF,CAAC;IAEF;;OAEG;IACH,YACI,OASC;QAED,EAAE,CAAC,OAAO,EAAE,uBAAuB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,wBAAwB,GAAG,IAAI,EAC/B,aAAa,GAAG,EAAE,EAClB,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,QAAQ,EACR,aAAa,GAAG,KAAK,EACrB,iBAAiB,GAAG,KAAK,EACzB,sBAAsB,EACtB,aAAa,EACb,kBAAkB,EAClB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,sBAAsB,EAAE,GAAG,EAAE,CACzB,sBAAsB,EAAE;iBACnB,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBACtD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBAClE,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,aAAa,EAAE,aAAkE;SACpF,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,aAAa,CAAC,aAAa,KAAK,EAAmB,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,aAA4B,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzE,CAAC;QAED,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,CAAC;QAEzD,IAAI,aAAa,EAAE,SAAS,EAAE,CAAC;YAC3B,IAAI,kBAAkB,CAAC,eAAe;gBAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;YACpG,kBAAkB,CAAC,eAAe,GAAG,KAAK,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAA6B;YAC3D,GAAI,kBAA0B;SACjC,CAAC,CAAC;IACP,CAAC;IAEkB,oBAAoB;QAInC,OAAO,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;YACrD,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YACnC,OAAO,EAAE,KAAK,EAAE,OAGf,EAAE,EAAE;gBACD,OAAO,CAAC,uBAAuB,CAAC,KAAK,IAAI,EAAE;oBACvC,MAAM,OAAO,CAAC,IAAI;yBACb,KAAK,EAAE;yBACP,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;gBACxF,CAAC,CAAC,CAAC;YACP,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,IAAgB,EAAE,SAAmB;QACjE,MAAM,cAAc,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAE,IAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;aAC7F,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;aACjD,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;QAErC,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7D,CAAC;IAES,KAAK,CAAC,gBAAgB,CAC5B,eAA2E;QAE3E,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC;QAE3C,yGAAyG;QACzG,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC,IAAI,QAAQ,EAAE,MAAM,EAAE,KAAK,GAAG,EAAE,CAAC;YACrG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAElB,+IAA+I;YAC/I,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;YAE/E,IAAI,CAAC,cAAc;gBAAE,OAAO,KAAK,CAAC;YAClC,OAAO,iDAAiD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxF,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QAC/E,MAAM,UAAU,GAAG,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3C,IAAI,cAAc;YAAE,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,OAAO,iCAAiC,UAAU,EAAE,CAAC;QAElG,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CACrB,eAAgC;QAIhC,MAAM,cAAc,GAAe;YAC/B,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,IAAI,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC;YACrC,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC;YAEpD,cAAc,CAAC,QAAQ,GAAG,SAAS,EAAE,GAAG,CAAC;YACzC,cAAc,CAAC,SAAS,GAAG,SAAS,EAAE,SAAS,CAAC;YAChD,cAAc,CAAC,eAAe,GAAG,SAAS,EAAE,eAAe,CAAC;QAChE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAS,CAAC;QACtE,SAAS,EAAE,CAAC;QAEZ,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CACzE,IAAW,CACQ,CAAC;QAExB,IAAI,CAAC,yBAAyB,CAAC,eAAe,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAAC;QAEnF,MAAM,mBAAmB,GAAG,eAAe,CAAC,YAAY,CAAC;QAEzD,OAAO;YACH,IAAI;YACJ,IAAI,QAAQ;gBACR,MAAM,IAAI,KAAK,CACX,gMAAgM,CACnM,CAAC;YACN,CAAC;YACD,iBAAiB,EAAE,yBAAyB;YAC5C,YAAY,EAAE,KAAK,EAAE,iBAAsC,EAAE,EAAE,EAAE;gBAC7D,OAAO,CAAC,MAAM,0BAA0B,CAAC;oBACrC,OAAO,EAAE,EAAE,GAAG,cAAc,EAAE,KAAK,EAAE,IAAI,CAAC,6BAA6B,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE;oBAChG,IAAI;oBACJ,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;oBAC1C,aAAa,EAAE,MAAM,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;oBAC7E,gBAAgB,EAAE,IAAI,CAAC,oBAAoB;oBAC3C,kBAAkB,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG;oBAC/C,eAAe,EAAE,eAAe,CAAC,OAAO,CAAC,SAAS;oBAClD,YAAY,EAAE,mBAAmB;iBACpC,CAAC,CAA2B,CAAC,CAAC,2BAA2B;YAC9D,CAAC;SACJ,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QAIpD,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE;oBACxC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAA2B;gBAC5B,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAA4B,CAAC;QAExF,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE5F,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE7F,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEjG,IAAI,QAA8B,CAAC;QAEnC,IAAI,CAAC;YACD,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC,IAAI,SAAS,CAAC;QAC1F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAc,CAAC,CAAC;YAErE,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YAEnD,IAAI,CAAC,kBAAkB,CAAC,KAAc,CAAC,CAAC;YACxC,MAAM,KAAK,CAAC;QAChB,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAEjF,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QACtD,SAAS,EAAE,CAAC;QAEZ,eAAe;QACf,oEAAoE;QACpE,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YACzF,SAAS,EAAE,CAAC;YACZ,eAAe,CAAC,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,SAAU,CAAC,CAAC;QACrF,CAAC;QAED,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,eAAe,CAAC,OAAiC;gBAC1D,QAAQ;aACX,CAAC;QACN,CAAC;QAED,OAAO;YACH,OAAO,EAAE,eAAe,CAAC,OAAiC;SAC7D,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CACvC,eAA2E;QAE3E,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,EAAE,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,eAAgC;QAC9D,eAAe,CAAC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,eAAe,CAAC;QAC7D,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,aAAa,CACzB,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAA0B,EACrE,eAAuB,EACvB,gBAAwB;QAExB,MAAM,aAAa,GAAG,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,qBAAqB,GAAG,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtG,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC,CAAC;QAExG,MAAM,iBAAiB,CAAC,UAAU,CAC9B,IAAI,EACJ,CAAC,GAAG,aAAa,EAAE,GAAG,qBAAqB,EAAE,GAAG,sBAAsB,CAAC;aAClE,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,OAAO,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,IAAI,CAAC;aACxE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CACvE,CAAC;IACN,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,wBAAwB,CAAC,eAAuC,EAAE,KAAY;QAC1F,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACrD,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,KAAY;QACrC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAW,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAOO,KAAK,CAAC,eAAe,CACzB,QAA8B,EAC9B,eAAuC;QAEvC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAEnD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxE,MAAM,MAAM,GAAW,QAAQ,CAAC,MAAM,EAAE,CAAC;YAEzC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAEtC,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,IAAI,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBAClD,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;gBACtE,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,0BAA0B,CAAC,CAAC;YACzD,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC1C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACxE,IAAI,CAAC,sBAAsB,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC/E,CAAC;QACL,CAAC;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAEO,iBAAiB,GAAG,IAAI,OAAO,EAA6C,CAAC;IAE7E,yBAAyB,CAAC,OAAgB,EAAE,iBAA+C;QAC/F,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,OAAO;QACX,CAAC;QAED,IAAI,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAE/D,IAAI,UAAU,EAAE,CAAC;YACb,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC3B,OAAO;QACX,CAAC;QAED,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QAE1D,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,EAAE;YAClC,IAAI,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;gBACjE,IAAI,CAAC,WAAW,CAAC,uBAAuB,CACpC,iBAEI,CACP,CAAC;YACN,CAAC;QACL,CAAC,CAAC;QAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;QACrD,iBAAiB,CAAC,EAAE,iEAA2C,GAAG,EAAE;YAChE,OAAO,IAAI,CAAC,WAAY,CAAC,cAAc,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AAuBL,gBAAgB;AAChB,SAAS,oBAAoB,CACzB,OAAuE;IAEvE,OAAO,CAAC,CAAE,OAA4C,CAAC,YAAY,CAAC;AACxE,CAAC;AAED,gBAAgB;AAChB,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC5C,OAAuE;IAEvE,MAAM,EAAE,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,kBAAkB,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;IAE5F,MAAM,OAAO,GAAG,sCAAsC,CAAC;QACnD,eAAe,EAAE,mBAAmB,EAAE,QAAQ;QAC9C,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,mBAAmB,EAAE,OAAO;KACpD,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAClC,IAAW,EACX,mBAAmB,EAAE,QAAQ,IAAI,GAAG,EACpC,mBAAmB,EAAE,OAAO,IAAI,eAAe,IAAI,kBAAkB,CACxE,CAAC;IAEF,IAAI,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,YAAY,CAAC;YACxB,IAAI;YACJ,OAAO;YACP,GAAG,mBAAmB;SACzB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,YAAY,CAAC;QAChB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,IAAI;QACJ,OAAO;QACP,GAAI,mBAA2C;KAClD,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;AACrC,sEAAsE;AACtE,IAA0B,EAC1B,QAAgB,EAChB,OAAe;IAEf,MAAM,IAAI,GACN,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAA0B,EAAE,EAAE,CACxD,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC5E,CAAC,IAAI,EAAE,CAAC;IACb,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAsB,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,IAAI;SACN,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;QAClB,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,+CAA+C;gBACpE,+EAA+E,CACtF,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,IAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC"}
|
|
@@ -44,6 +44,11 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
|
|
|
44
44
|
* to reduce the chance of detection of the crawler.
|
|
45
45
|
*/
|
|
46
46
|
userAgent?: string;
|
|
47
|
+
/**
|
|
48
|
+
* If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
|
|
49
|
+
* This is useful when using HTTPS proxies with self-signed certificates.
|
|
50
|
+
*/
|
|
51
|
+
ignoreProxyCertificate?: boolean;
|
|
47
52
|
/**
|
|
48
53
|
* The type of browser to be launched.
|
|
49
54
|
* By default, `chromium` is used. Other browsers like `webkit` or `firefox` can be used.
|
|
@@ -81,6 +86,8 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
|
|
|
81
86
|
useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
82
87
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
83
88
|
browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
89
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
90
|
+
ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
84
91
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
85
92
|
userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
86
93
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AACjF,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAU9D,MAAM,WAAW,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAE,SAAQ,oBAAoB,CAAC,QAAQ,CAAC;IAC5F;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACvB;AAED;;;GAGG;AACH,8BAAsB,eAAe,CACjC,MAAM,SAAS,aAAa,EAC5B,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,EAC5B,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,EACnD,aAAa,SAAS,UAAU,CAAC,GAAG,CAAC,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAC5F,YAAY,SAAS,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AACjF,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAU9D,MAAM,WAAW,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAE,SAAQ,oBAAoB,CAAC,QAAQ,CAAC;IAC5F;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAEjC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACvB;AAED;;;GAGG;AACH,8BAAsB,eAAe,CACjC,MAAM,SAAS,aAAa,EAC5B,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,EAC5B,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,EACnD,aAAa,SAAS,UAAU,CAAC,GAAG,CAAC,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAC5F,YAAY,SAAS,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IA6C5E,QAAQ,CAAC,MAAM;IA3CnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,uBAAuB,EAAE,UAAU,CAAC;IAEpC,MAAM,EAAG,CAAC,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;MAS3B;IAEF,MAAM,CAAC,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,CAAC;IAkB7E;;OAEG;gBAEC,aAAa,EAAE,oBAAoB,CAAC,aAAa,EAAE,QAAQ,CAAC,EACnD,MAAM,gBAAkC;IAsBrD;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAQ7B;;;OAGG;IACH,MAAM,IAAI,YAAY;IAOtB,mBAAmB,IAAI,UAAU;IA0BjC,SAAS,CAAC,yBAAyB,IAAI,OAAO;IAI9C,SAAS,CAAC,wBAAwB,IAAI,MAAM;IAI5C;;OAEG;IACH,SAAS,CAAC,+BAA+B,IAAI,MAAM;IA6BnD,SAAS,CAAC,yBAAyB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI;CAe/D"}
|
|
@@ -27,6 +27,7 @@ export class BrowserLauncher {
|
|
|
27
27
|
useChrome: ow.optional.boolean,
|
|
28
28
|
useIncognitoPages: ow.optional.boolean,
|
|
29
29
|
browserPerProxy: ow.optional.boolean,
|
|
30
|
+
ignoreProxyCertificate: ow.optional.boolean,
|
|
30
31
|
userDataDir: ow.optional.string,
|
|
31
32
|
launchOptions: ow.optional.object,
|
|
32
33
|
userAgent: ow.optional.string,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-launcher.js","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,MAAM,gBAAgB,GAAG;IACrB,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,GAAG;CACd,CAAC;AAEF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"browser-launcher.js","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,MAAM,gBAAgB,GAAG;IACrB,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,GAAG;CACd,CAAC;AAEF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAwE/C;;;GAGG;AACH,MAAM,OAAgB,eAAe;IAkDpB;IA3Cb,QAAQ,CAAW;IACnB,QAAQ,CAAU;IAClB,SAAS,CAAW;IACpB,aAAa,CAAa;IAC1B,uBAAuB,CAAa;IACpC,mCAAmC;IACnC,MAAM,CAAK;IACX,SAAS,CAAU;IAET,MAAM,CAAC,YAAY,GAAG;QAC5B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;QAChC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,iBAAiB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACtC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,sBAAsB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAC3C,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC;IAEF,MAAM,CAAC,sBAAsB,CAAI,QAAgB,EAAE,cAAsB;QACrE,IAAI,CAAC;YACD,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,sBAAsB;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,GAA+B,CAAC;YAC1C,IAAI,CAAC,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBAChC,MAAM,GAAG,GACL,uBAAuB,QAAQ,+BAA+B,QAAQ,cAAc;oBACpF,uBAAuB,QAAQ,mFAAmF,CAAC;gBACvH,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;oBAC/B,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,6BAA6B,QAAQ,+BAA+B,cAAc,gBAAgB,CAAC;gBACzH,CAAC;YACL,CAAC;YAED,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;OAEG;IACH,YACI,aAA4D,EACnD,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,MAAM,EACF,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,SAAS,EACT,aAAa,GAAG,EAAE,EAClB,GAAG,uBAAuB,EAC7B,GAAG,aAAa,CAAC;QAElB,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAEzC,6DAA6D;QAC7D,IAAI,CAAC,QAAQ,GAAG,QAAS,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,uBAAqC,CAAC;IACzE,CAAC;IAED;;OAEG;IACH,mBAAmB;QACf,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,mBAAmB,EAAE;YACzC,GAAG,IAAI,CAAC,uBAAuB;SAClC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAE7C,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAiB,CAAC;IAClD,CAAC;IAED,mBAAmB;QACf,MAAM,aAAa,GAAoC;YACnD,IAAI,EAAE,EAAE;YACR,eAAe,EAAE,gBAAgB;YACjC,GAAG,IAAI,CAAC,aAAa;SACxB,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,EAAE,CAAC;YAC3C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,aAAa,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YACjC,aAAa,CAAC,QAAQ,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAC9D,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,CAAC;YAClD,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC,wBAAwB,EAAE,CAAC;QACnE,CAAC;QAED,OAAO,aAAa,CAAC;IACzB,CAAC;IAES,yBAAyB;QAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3E,CAAC;IAES,wBAAwB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED;;OAEG;IACO,+BAA+B;QACrC;;;;;WAKG;QACH,MAAM,YAAY,GAAG,GAAG,EAAE;YACtB,IAAI,oBAAoB,GAAG,4DAA4D,CAAC;YACxF,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,2CAA2C,CAAC;YACtF,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,2CAA2C,CAAC;YAE9F,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,oBAAoB,GAAG,MAAM,CAAC;YAClC,CAAC;iBAAM,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/B,oBAAoB,GAAG,MAAM,CAAC;YAClC,CAAC;YACD,OAAO,oBAAoB,CAAC;QAChC,CAAC,CAAC;QACF,QAAQ,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;YACpB,KAAK,QAAQ;gBACT,OAAO,8DAA8D,CAAC;YAC1E,KAAK,OAAO;gBACR,OAAO,YAAY,EAAE,CAAC;YAC1B;gBACI,OAAO,wBAAwB,CAAC;QACxC,CAAC;IACL,CAAC;IAES,yBAAyB,CAAC,QAAiB;QACjD,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEtB,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,6CAA6C,QAAQ,GAAG,CAAC,CAAC;QAC9E,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE9B,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;YACxG,CAAC;QACL,CAAC;IACL,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.41",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.3.2",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.41",
|
|
52
|
+
"@crawlee/browser-pool": "4.0.0-beta.41",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.41",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.41",
|
|
55
55
|
"ow": "^2.0.0",
|
|
56
56
|
"tslib": "^2.8.1",
|
|
57
57
|
"type-fest": "^4.41.0"
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "0f3d1a433e1a27647d99ad5dc81a605e82505d1a"
|
|
79
79
|
}
|