@crawlee/http 3.13.6-beta.0 → 4.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,14 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.HttpCrawler = void 0;
4
- exports.createHttpRouter = createHttpRouter;
5
- const tslib_1 = require("tslib");
6
- const node_path_1 = require("node:path");
7
- const node_util_1 = tslib_1.__importDefault(require("node:util"));
8
- const basic_1 = require("@crawlee/basic");
9
- const utils_1 = require("@crawlee/utils");
10
- const cheerio = tslib_1.__importStar(require("cheerio"));
11
- const content_type_1 = tslib_1.__importDefault(require("content-type"));
12
- const iconv_lite_1 = tslib_1.__importDefault(require("iconv-lite"));
13
- const mime_types_1 = tslib_1.__importDefault(require("mime-types"));
14
- const ow_1 = tslib_1.__importStar(require("ow"));
15
- const timeout_1 = require("@apify/timeout");
16
- const utilities_1 = require("@apify/utilities");
1
+ import { extname } from 'node:path';
2
+ import util from 'node:util';
3
+ import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, Configuration, CrawlerExtension, mergeCookies, processHttpRequestOptions, RequestState, Router, SessionError, validators, } from '@crawlee/basic';
4
+ import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
5
+ import * as cheerio from 'cheerio';
6
+ import contentTypeParser from 'content-type';
7
+ import iconv from 'iconv-lite';
8
+ import mime from 'mime-types';
9
+ import ow, { ObjectPredicate } from 'ow';
10
+ import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
11
+ import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';
17
12
  let TimeoutError;
18
13
  /**
19
14
  * Default mime types, which HttpScraper supports.
@@ -97,15 +92,44 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
97
92
  * ```
98
93
  * @category Crawlers
99
94
  */
100
- class HttpCrawler extends basic_1.BasicCrawler {
95
+ export class HttpCrawler extends BasicCrawler {
96
+ config;
97
+ /**
98
+ * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
99
+ * Only available if used by the crawler.
100
+ */
101
+ proxyConfiguration;
102
+ userRequestHandlerTimeoutMillis;
103
+ preNavigationHooks;
104
+ postNavigationHooks;
105
+ persistCookiesPerSession;
106
+ navigationTimeoutMillis;
107
+ ignoreSslErrors;
108
+ suggestResponseEncoding;
109
+ forceResponseEncoding;
110
+ additionalHttpErrorStatusCodes;
111
+ ignoreHttpErrorStatusCodes;
112
+ supportedMimeTypes;
113
+ static optionsShape = {
114
+ ...BasicCrawler.optionsShape,
115
+ navigationTimeoutSecs: ow.optional.number,
116
+ ignoreSslErrors: ow.optional.boolean,
117
+ additionalMimeTypes: ow.optional.array.ofType(ow.string),
118
+ suggestResponseEncoding: ow.optional.string,
119
+ forceResponseEncoding: ow.optional.string,
120
+ proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
121
+ persistCookiesPerSession: ow.optional.boolean,
122
+ additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
123
+ ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
124
+ preNavigationHooks: ow.optional.array,
125
+ postNavigationHooks: ow.optional.array,
126
+ };
101
127
  /**
102
128
  * All `HttpCrawlerOptions` parameters are passed via an options object.
103
129
  */
104
- constructor(options = {}, config = basic_1.Configuration.getGlobalConfig()) {
105
- (0, ow_1.default)(options, 'HttpCrawlerOptions', ow_1.default.object.exactShape(HttpCrawler.optionsShape));
106
- const { requestHandler, handlePageFunction, requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [],
107
- // Ignored
108
- handleRequestFunction,
130
+ constructor(options = {}, config = Configuration.getGlobalConfig()) {
131
+ ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
132
+ const { requestHandler, requestHandlerTimeoutSecs = 60, navigationTimeoutSecs = 30, ignoreSslErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, proxyConfiguration, persistCookiesPerSession, preNavigationHooks = [], postNavigationHooks = [], additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [],
109
133
  // BasicCrawler
110
134
  autoscaledPoolOptions = HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS, ...basicCrawlerOptions } = options;
111
135
  super({
@@ -114,125 +138,11 @@ class HttpCrawler extends basic_1.BasicCrawler {
114
138
  autoscaledPoolOptions,
115
139
  // We need to add some time for internal functions to finish,
116
140
  // but not too much so that we would stall the crawler.
117
- requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + basic_1.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
141
+ requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
118
142
  }, config);
119
- Object.defineProperty(this, "config", {
120
- enumerable: true,
121
- configurable: true,
122
- writable: true,
123
- value: config
124
- });
125
- /**
126
- * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
127
- * Only available if used by the crawler.
128
- */
129
- Object.defineProperty(this, "proxyConfiguration", {
130
- enumerable: true,
131
- configurable: true,
132
- writable: true,
133
- value: void 0
134
- });
135
- Object.defineProperty(this, "userRequestHandlerTimeoutMillis", {
136
- enumerable: true,
137
- configurable: true,
138
- writable: true,
139
- value: void 0
140
- });
141
- Object.defineProperty(this, "preNavigationHooks", {
142
- enumerable: true,
143
- configurable: true,
144
- writable: true,
145
- value: void 0
146
- });
147
- Object.defineProperty(this, "postNavigationHooks", {
148
- enumerable: true,
149
- configurable: true,
150
- writable: true,
151
- value: void 0
152
- });
153
- Object.defineProperty(this, "persistCookiesPerSession", {
154
- enumerable: true,
155
- configurable: true,
156
- writable: true,
157
- value: void 0
158
- });
159
- Object.defineProperty(this, "navigationTimeoutMillis", {
160
- enumerable: true,
161
- configurable: true,
162
- writable: true,
163
- value: void 0
164
- });
165
- Object.defineProperty(this, "ignoreSslErrors", {
166
- enumerable: true,
167
- configurable: true,
168
- writable: true,
169
- value: void 0
170
- });
171
- Object.defineProperty(this, "suggestResponseEncoding", {
172
- enumerable: true,
173
- configurable: true,
174
- writable: true,
175
- value: void 0
176
- });
177
- Object.defineProperty(this, "forceResponseEncoding", {
178
- enumerable: true,
179
- configurable: true,
180
- writable: true,
181
- value: void 0
182
- });
183
- Object.defineProperty(this, "additionalHttpErrorStatusCodes", {
184
- enumerable: true,
185
- configurable: true,
186
- writable: true,
187
- value: void 0
188
- });
189
- Object.defineProperty(this, "ignoreHttpErrorStatusCodes", {
190
- enumerable: true,
191
- configurable: true,
192
- writable: true,
193
- value: void 0
194
- });
195
- Object.defineProperty(this, "supportedMimeTypes", {
196
- enumerable: true,
197
- configurable: true,
198
- writable: true,
199
- value: void 0
200
- });
201
- /**
202
- * @internal wraps public utility for mocking purposes
203
- */
204
- Object.defineProperty(this, "_requestAsBrowser", {
205
- enumerable: true,
206
- configurable: true,
207
- writable: true,
208
- value: async (options, session) => {
209
- const response = await this.httpClient.stream((0, basic_1.processHttpRequestOptions)({
210
- ...options,
211
- cookieJar: options.cookieJar, // HACK - the type of ToughCookieJar in got is wrong
212
- responseType: 'text',
213
- }), (redirectResponse, updatedRequest) => {
214
- if (this.persistCookiesPerSession) {
215
- session.setCookiesFromResponse(redirectResponse);
216
- const cookieString = session.getCookieString(updatedRequest.url.toString());
217
- if (cookieString !== '') {
218
- updatedRequest.headers.Cookie = cookieString;
219
- }
220
- }
221
- });
222
- return addResponsePropertiesToStream(response.stream, response);
223
- }
224
- });
225
- this._handlePropertyNameChange({
226
- newName: 'requestHandler',
227
- oldName: 'handlePageFunction',
228
- propertyKey: 'requestHandler',
229
- newProperty: requestHandler,
230
- oldProperty: handlePageFunction,
231
- allowUndefined: true,
232
- });
233
- if (!this.requestHandler) {
234
- this.requestHandler = this.router;
235
- }
143
+ this.config = config;
144
+ // FIXME any
145
+ this.requestHandler = requestHandler ?? this.router;
236
146
  // Cookies should be persisted per session only if session pool is used
237
147
  if (!this.useSessionPool && persistCookiesPerSession) {
238
148
  throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
@@ -269,7 +179,7 @@ class HttpCrawler extends basic_1.BasicCrawler {
269
179
  * @param extension Crawler extension that overrides the crawler configuration.
270
180
  */
271
181
  use(extension) {
272
- (0, ow_1.default)(extension, ow_1.default.object.instanceOf(basic_1.CrawlerExtension));
182
+ ow(extension, ow.object.instanceOf(CrawlerExtension));
273
183
  const className = this.constructor.name;
274
184
  const extensionOptions = extension.getCrawlerOptions();
275
185
  for (const [key, value] of Object.entries(extensionOptions)) {
@@ -301,25 +211,25 @@ class HttpCrawler extends basic_1.BasicCrawler {
301
211
  }
302
212
  if (!request.skipNavigation) {
303
213
  await this._handleNavigation(crawlingContext);
304
- (0, timeout_1.tryCancel)();
214
+ tryCancel();
305
215
  const parsed = await this._parseResponse(request, crawlingContext.response, crawlingContext);
306
216
  const response = parsed.response;
307
217
  const contentType = parsed.contentType;
308
- (0, timeout_1.tryCancel)();
218
+ tryCancel();
309
219
  // `??=` because descendant classes may already set optimized version
310
- crawlingContext.waitForSelector ?? (crawlingContext.waitForSelector = async (selector, _timeoutMs) => {
220
+ crawlingContext.waitForSelector ??= async (selector, _timeoutMs) => {
311
221
  const $ = cheerio.load(parsed.body.toString());
312
222
  if ($(selector).get().length === 0) {
313
223
  throw new Error(`Selector '${selector}' not found.`);
314
224
  }
315
- });
316
- crawlingContext.parseWithCheerio ?? (crawlingContext.parseWithCheerio = async (selector, timeoutMs) => {
225
+ };
226
+ crawlingContext.parseWithCheerio ??= async (selector, timeoutMs) => {
317
227
  const $ = cheerio.load(parsed.body.toString());
318
228
  if (selector) {
319
229
  await crawlingContext.waitForSelector(selector, timeoutMs);
320
230
  }
321
231
  return $;
322
- });
232
+ };
323
233
  if (this.useSessionPool) {
324
234
  this._throwOnBlockedRequest(crawlingContext.session, response.statusCode);
325
235
  }
@@ -332,7 +242,7 @@ class HttpCrawler extends basic_1.BasicCrawler {
332
242
  // eslint-disable-next-line dot-notation
333
243
  `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`);
334
244
  request.noRetry = true;
335
- request.state = basic_1.RequestState.SKIPPED;
245
+ request.state = RequestState.SKIPPED;
336
246
  return;
337
247
  }
338
248
  Object.assign(crawlingContext, parsed);
@@ -348,22 +258,22 @@ class HttpCrawler extends basic_1.BasicCrawler {
348
258
  if (this.retryOnBlocked) {
349
259
  const error = await this.isRequestBlocked(crawlingContext);
350
260
  if (error)
351
- throw new basic_1.SessionError(error);
261
+ throw new SessionError(error);
352
262
  }
353
- request.state = basic_1.RequestState.REQUEST_HANDLER;
263
+ request.state = RequestState.REQUEST_HANDLER;
354
264
  try {
355
- await (0, timeout_1.addTimeoutToPromise)(async () => Promise.resolve(this.requestHandler(crawlingContext)), this.userRequestHandlerTimeoutMillis, `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`);
356
- request.state = basic_1.RequestState.DONE;
265
+ await addTimeoutToPromise(async () => Promise.resolve(this.requestHandler(crawlingContext)), this.userRequestHandlerTimeoutMillis, `requestHandler timed out after ${this.userRequestHandlerTimeoutMillis / 1000} seconds.`);
266
+ request.state = RequestState.DONE;
357
267
  }
358
268
  catch (e) {
359
- request.state = basic_1.RequestState.ERROR;
269
+ request.state = RequestState.ERROR;
360
270
  throw e;
361
271
  }
362
272
  }
363
273
  async isRequestBlocked(crawlingContext) {
364
274
  if (HTML_AND_XML_MIME_TYPES.includes(crawlingContext.contentType.type)) {
365
275
  const $ = await crawlingContext.parseWithCheerio();
366
- const foundSelectors = utils_1.RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
276
+ const foundSelectors = RETRY_CSS_SELECTORS.filter((selector) => $(selector).length > 0);
367
277
  if (foundSelectors.length > 0) {
368
278
  return `Found selectors: ${foundSelectors.join(', ')}`;
369
279
  }
@@ -374,19 +284,19 @@ class HttpCrawler extends basic_1.BasicCrawler {
374
284
  const gotOptions = {};
375
285
  const { request, session } = crawlingContext;
376
286
  const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
377
- request.state = basic_1.RequestState.BEFORE_NAV;
287
+ request.state = RequestState.BEFORE_NAV;
378
288
  // Execute pre navigation hooks before applying session pool cookies,
379
289
  // as they may also set cookies in the session
380
290
  await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
381
- (0, timeout_1.tryCancel)();
291
+ tryCancel();
382
292
  const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
383
293
  this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
384
294
  const proxyUrl = crawlingContext.proxyInfo?.url;
385
- crawlingContext.response = await (0, timeout_1.addTimeoutToPromise)(async () => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
386
- (0, timeout_1.tryCancel)();
387
- request.state = basic_1.RequestState.AFTER_NAV;
295
+ crawlingContext.response = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
296
+ tryCancel();
297
+ request.state = RequestState.AFTER_NAV;
388
298
  await this._executeHooks(this.postNavigationHooks, crawlingContext, gotOptions);
389
- (0, timeout_1.tryCancel)();
299
+ tryCancel();
390
300
  }
391
301
  /**
392
302
  * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
@@ -410,7 +320,7 @@ class HttpCrawler extends basic_1.BasicCrawler {
410
320
  else {
411
321
  sourceCookies.push(upperCaseHeader);
412
322
  }
413
- alteredGotOptionsCookies = (0, basic_1.mergeCookies)(request.url, sourceCookies);
323
+ alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);
414
324
  }
415
325
  const sourceCookies = [sessionCookie, preHookCookies];
416
326
  if (Array.isArray(alteredGotOptionsCookies)) {
@@ -420,8 +330,8 @@ class HttpCrawler extends basic_1.BasicCrawler {
420
330
  sourceCookies.push(alteredGotOptionsCookies);
421
331
  }
422
332
  sourceCookies.push(postHookCookies);
423
- const mergedCookie = (0, basic_1.mergeCookies)(request.url, sourceCookies);
424
- gotOptions.headers ?? (gotOptions.headers = {});
333
+ const mergedCookie = mergeCookies(request.url, sourceCookies);
334
+ gotOptions.headers ??= {};
425
335
  Reflect.deleteProperty(gotOptions.headers, 'Cookie');
426
336
  Reflect.deleteProperty(gotOptions.headers, 'cookie');
427
337
  if (mergedCookie !== '') {
@@ -448,7 +358,7 @@ class HttpCrawler extends basic_1.BasicCrawler {
448
358
  return undefined;
449
359
  }
450
360
  if (this.isProxyError(e)) {
451
- throw new basic_1.SessionError(this._getMessageFromError(e));
361
+ throw new SessionError(this._getMessageFromError(e));
452
362
  }
453
363
  else {
454
364
  throw e;
@@ -469,14 +379,14 @@ class HttpCrawler extends basic_1.BasicCrawler {
469
379
  const excludeError = this.ignoreHttpErrorStatusCodes.has(statusCode);
470
380
  const includeError = this.additionalHttpErrorStatusCodes.has(statusCode);
471
381
  if ((statusCode >= 500 && !excludeError) || includeError) {
472
- const body = await (0, utilities_1.readStreamToString)(response, encoding);
382
+ const body = await readStreamToString(response, encoding);
473
383
  // Errors are often sent as JSON, so attempt to parse them,
474
384
  // despite Accept header being set to text/html.
475
385
  if (type === APPLICATION_JSON_MIME_TYPE) {
476
386
  const errorResponse = JSON.parse(body);
477
387
  let { message } = errorResponse;
478
388
  if (!message)
479
- message = node_util_1.default.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
389
+ message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
480
390
  throw new Error(`${statusCode} - ${message}`);
481
391
  }
482
392
  if (includeError) {
@@ -491,7 +401,7 @@ class HttpCrawler extends basic_1.BasicCrawler {
491
401
  return { ...parsed, isXml, response, contentType };
492
402
  }
493
403
  else {
494
- const body = await (0, utilities_1.concatStreamToBuffer)(response);
404
+ const body = await concatStreamToBuffer(response);
495
405
  return {
496
406
  body,
497
407
  response,
@@ -502,7 +412,7 @@ class HttpCrawler extends basic_1.BasicCrawler {
502
412
  }
503
413
  async _parseHTML(response, _isXml, _crawlingContext) {
504
414
  return {
505
- body: await (0, utilities_1.concatStreamToBuffer)(response),
415
+ body: await concatStreamToBuffer(response),
506
416
  };
507
417
  }
508
418
  /**
@@ -555,9 +465,9 @@ class HttpCrawler extends basic_1.BasicCrawler {
555
465
  if (Buffer.isEncoding(encoding))
556
466
  return { response, encoding };
557
467
  // Try to re-encode a variety of unsupported encodings to utf-8
558
- if (iconv_lite_1.default.encodingExists(encoding)) {
559
- const encodeStream = iconv_lite_1.default.encodeStream(utf8);
560
- const decodeStream = iconv_lite_1.default.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
468
+ if (iconv.encodingExists(encoding)) {
469
+ const encodeStream = iconv.encodeStream(utf8);
470
+ const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
561
471
  response.on('error', (err) => decodeStream.emit('error', err));
562
472
  const encodedResponse = response.pipe(decodeStream).pipe(encodeStream);
563
473
  encodedResponse.statusCode = response.statusCode;
@@ -580,7 +490,7 @@ class HttpCrawler extends basic_1.BasicCrawler {
580
490
  continue;
581
491
  }
582
492
  try {
583
- const parsedType = content_type_1.default.parse(mimeType);
493
+ const parsedType = contentTypeParser.parse(mimeType);
584
494
  this.supportedMimeTypes.add(parsedType.type);
585
495
  }
586
496
  catch (err) {
@@ -608,28 +518,26 @@ class HttpCrawler extends basic_1.BasicCrawler {
608
518
  `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
609
519
  }
610
520
  }
521
+ /**
522
+ * @internal wraps public utility for mocking purposes
523
+ */
524
+ _requestAsBrowser = async (options, session) => {
525
+ const response = await this.httpClient.stream(processHttpRequestOptions({
526
+ ...options,
527
+ cookieJar: options.cookieJar, // HACK - the type of ToughCookieJar in got is wrong
528
+ responseType: 'text',
529
+ }), (redirectResponse, updatedRequest) => {
530
+ if (this.persistCookiesPerSession) {
531
+ session.setCookiesFromResponse(redirectResponse);
532
+ const cookieString = session.getCookieString(updatedRequest.url.toString());
533
+ if (cookieString !== '') {
534
+ updatedRequest.headers.Cookie = cookieString;
535
+ }
536
+ }
537
+ });
538
+ return addResponsePropertiesToStream(response.stream, response);
539
+ };
611
540
  }
612
- exports.HttpCrawler = HttpCrawler;
613
- Object.defineProperty(HttpCrawler, "optionsShape", {
614
- enumerable: true,
615
- configurable: true,
616
- writable: true,
617
- value: {
618
- ...basic_1.BasicCrawler.optionsShape,
619
- handlePageFunction: ow_1.default.optional.function,
620
- navigationTimeoutSecs: ow_1.default.optional.number,
621
- ignoreSslErrors: ow_1.default.optional.boolean,
622
- additionalMimeTypes: ow_1.default.optional.array.ofType(ow_1.default.string),
623
- suggestResponseEncoding: ow_1.default.optional.string,
624
- forceResponseEncoding: ow_1.default.optional.string,
625
- proxyConfiguration: ow_1.default.optional.object.validate(basic_1.validators.proxyConfiguration),
626
- persistCookiesPerSession: ow_1.default.optional.boolean,
627
- additionalHttpErrorStatusCodes: ow_1.default.optional.array.ofType(ow_1.default.number),
628
- ignoreHttpErrorStatusCodes: ow_1.default.optional.array.ofType(ow_1.default.number),
629
- preNavigationHooks: ow_1.default.optional.array,
630
- postNavigationHooks: ow_1.default.optional.array,
631
- }
632
- });
633
541
  /**
634
542
  * The stream object returned from got does not have the below properties.
635
543
  * At the same time, you can't read data directly from the response stream,
@@ -673,15 +581,15 @@ function addResponsePropertiesToStream(stream, response) {
673
581
  * @param response HTTP response object
674
582
  */
675
583
  function parseContentTypeFromResponse(response) {
676
- (0, ow_1.default)(response, ow_1.default.object.partialShape({
677
- url: ow_1.default.string.url,
678
- headers: new ow_1.ObjectPredicate(),
584
+ ow(response, ow.object.partialShape({
585
+ url: ow.string.url,
586
+ headers: new ObjectPredicate(),
679
587
  }));
680
588
  const { url, headers } = response;
681
589
  let parsedContentType;
682
590
  if (headers['content-type']) {
683
591
  try {
684
- parsedContentType = content_type_1.default.parse(headers['content-type']);
592
+ parsedContentType = contentTypeParser.parse(headers['content-type']);
685
593
  }
686
594
  catch {
687
595
  // Can not parse content type from Content-Type header. Try to parse it from file extension.
@@ -690,8 +598,8 @@ function parseContentTypeFromResponse(response) {
690
598
  // Parse content type from file extension as fallback
691
599
  if (!parsedContentType) {
692
600
  const parsedUrl = new URL(url);
693
- const contentTypeFromExtname = mime_types_1.default.contentType((0, node_path_1.extname)(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
694
- parsedContentType = content_type_1.default.parse(contentTypeFromExtname);
601
+ const contentTypeFromExtname = mime.contentType(extname(parsedUrl.pathname)) || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
602
+ parsedContentType = contentTypeParser.parse(contentTypeFromExtname);
695
603
  }
696
604
  return {
697
605
  type: parsedContentType.type,
@@ -722,7 +630,7 @@ function parseContentTypeFromResponse(response) {
722
630
  * await crawler.run();
723
631
  * ```
724
632
  */
725
- function createHttpRouter(routes) {
726
- return basic_1.Router.create(routes);
633
+ export function createHttpRouter(routes) {
634
+ return Router.create(routes);
727
635
  }
728
636
  //# sourceMappingURL=http-crawler.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":";;;AA2hCA,4CAKC;;AA/hCD,yCAAoC;AAEpC,kEAA6B;AAe7B,0CAWwB;AAGxB,0CAAuE;AACvE,yDAAmC;AAEnC,wEAA6C;AAG7C,oEAA+B;AAC/B,oEAA8B;AAC9B,iDAAyC;AAGzC,4CAAgE;AAChE,gDAA4E;AAE5E,IAAI,YAAsC,CAAC;AAW3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA8LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAa,WAEX,SAAQ,oBAAqB;IAsC3B;;OAEG;IACH,YACI,UAAuC,EAAE,EACvB,SAAS,qBAAa,CAAC,eAAe,EAAE;QAE1D,IAAA,YAAE,EAAC,OAAO,EAAE,oBAAoB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,cAAc,EACd,kBAAkB,EAElB,yBAAyB,GAAG,EAAE,EAC9B,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,UAAU;QACV,qBAAqB;QAErB,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,cAAc;YACd,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,yBAAyB,EACrB,qBAAqB,GAAG,yBAAyB,GAAG,yCAAiC;SAC5F,EACD,MAAM,CACT,CAAC;QAxCF;;;;mBAAkB,MAAM;WAAkC;QA1C9D;;;WAGG;QACH;;;;;WAAwC;QAE9B;;;;;WAAwC;QACxC;;;;;WAAgD;QAChD;;;;;WAAiD;QACjD;;;;;WAAkC;QAClC;;;;;WAAgC;QAChC;;;;;WAAyB;QACzB;;;;;WAAiC;QACjC;;;;;WAA+B;QAC/B;;;;;WAA4C;QAC5C;;;;;WAAwC;QAC/B;;;;;WAAgC;QAojBnD;;WAEG;QACK;;;;mBAAoB,KAAK,EAC7B,OAA4D,EAC5D,OAAiB,EACnB,EAAE;gBACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,IAAA,iCAAyB,EAAC;oBACtB,GAAG,OAAO;oBACV,SAAS,EAAE,OAAO,CAAC,SAAgB,EAAE,oDAAoD;oBACzF,YAAY,EAAE,MAAM;iBACvB,CAAC,EACF,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;oBACjC,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;wBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;wBAElD,MAAM,YAAY,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;wBAC9E,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;4BACtB,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;wBACjD,CAAC;oBACL,CAAC;gBACL,CAAC,CACJ,CAAC;gBAEF,OAAO,6BAA6B,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACpE,CAAC;WAAC;QA1gBE,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,oBAAoB;YAC7B,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,kBAAkB;YAC/B,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,CAAC;QAED,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,+BAA+B,GAAG,yBAAyB,GAAG,IAAI,CAAC;QACxE,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAA2B;QAC3B,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAExC,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QAEvD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1D,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAiB,CAAC,CAAC;YACpD,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAiB,CAAC,IAAI,IAAI,CAAC;YAE/C,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,wDAAwD;gBACxD,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,iCAAiC,SAAS,YAAY,CACxG,CAAC;YACN,CAAC;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,CAAC;gBACxB,0DAA0D;gBAC1D,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,OAAO,SAAS,IAAI,GAAG,KAAK,YAAY,IAAI,CAC1H,CAAC;YACN,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,mBAAmB,SAAS,IAAI,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE1G,IAAI,CAAC,GAAiB,CAAC,GAAG,KAAyB,CAAC;QACxD,CAAC;IACL,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,IAAA,mBAAS,GAAE,CAAC;YAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAS,EAAE,eAAe,CAAC,CAAC;YAC9F,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;YACxC,IAAA,mBAAS,GAAE,CAAC;YAEZ,qEAAqE;YACrE,eAAe,CAAC,eAAe,KAA/B,eAAe,CAAC,eAAe,GAAK,KAAK,EAAE,QAAiB,EAAE,UAAmB,EAAE,EAAE;gBACjF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;gBACzD,CAAC;YACL,CAAC,EAAC;YACF,eAAe,CAAC,gBAAgB,KAAhC,eAAe,CAAC,gBAAgB,GAAK,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;gBACjF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAEhD,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,eAAe,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAC/D,CAAC;gBAED,OAAO,CAAC,CAAC;YACb,CAAC,EAAC;YAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,UAAW,CAAC,CAAC;YAChF,CAAC;YAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;YAC9D,CAAC;YAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;YAEjC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK;gBACV,wCAAwC;gBACxC,oBAAoB,OAAO,CAAC,EAAE,mBAAmB,OAAO,CAAC,GAAG,mBAAmB,OAAO,CAAC,SAAS,qDAAqD,OAAO,CAAC,iBAAiB,CAAC,IAAI,CACtL,CAAC;gBAEF,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;gBACvB,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,OAAO,CAAC;gBAErC,OAAO;YACX,CAAC;YAED,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YAEvC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;gBAC3C,GAAG;oBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;wBAAE,OAAO,IAAI,CAAC;oBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;aACJ,CAAC,CAAC;QACP,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,oBAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,eAAe,CAAC;QAC7C,IAAI,CAAC;YACD,MAAM,IAAA,6BAAmB,EACrB,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,eAAyC,CAAC,CAAC,EAC3F,IAAI,CAAC,+BAA+B,EACpC,kCAAkC,IAAI,CAAC,+BAA+B,GAAG,IAAI,WAAW,CAC3F,CAAC;YACF,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,IAAI,CAAC;QACtC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAEkB,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QAC9D,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,2BAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAA,6BAAmB,EAChD,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAC7E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,IAAA,mBAAS,GAAE,CAAC;QAEZ,OAAO,CAAC,KAAK,GAAG,oBAAY,CAAC,SAAS,CAAC;QACvC,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;IAChB,CAAC;IAED;;OAEG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,UAAuB,EACvB,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;QAE9F,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC3D,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YAEhF,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAC/I,CAAC;YAEF,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,wBAAwB,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QAEtD,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC1C,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAlB,UAAU,CAAC,OAAO,GAAK,EAAE,EAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACW;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,SAAqC,CAAC;YACjD,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,oBAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,cAA+B,EAAE,eAAwB;QACtG,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAW,IAAI,GAAG,IAAI,UAAW,IAAI,GAAG,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,UAAW,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QACtE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QAE1E,IAAI,CAAC,UAAW,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,MAAM,IAAA,8BAAkB,EAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,mBAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,uCAAuC,CAAC,CAAC;YAC1E,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC;YACvE,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;QACvD,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC,CAAC;YAClD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;gBACX,YAAY,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;aAChG,CAAC;QACN,CAAC;IACL,CAAC;IAES,KAAK,CAAC,UAAU,CACtB,QAAyB,EACzB,MAAe,EACf,gBAAyB;QAEzB,OAAO;YACH,IAAI,EAAE,MAAM,IAAA,gCAAoB,EAAC,QAAQ,CAAC;SACzB,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAA0E;YAC1F,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,2EAA2E;QAC3E,sEAAsE;QACtE,2EAA2E;QAC3E,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;YACtE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAgB,EAChB,QAAyB,EACzB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,oBAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,oBAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACtE,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAIpE,CAAC;YACF,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAsB;gBAChC,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,sBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,2BAA2B,GAAG,IAAI,WAAW,CAAC,CAAC;IACnG,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAyB;QACpE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,UAAW,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,UAAW,CAAC,CAAC;QAE9F,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;;AArkBL,kCAkmBC;AA7kB6B;;;;WAAe;QACrC,GAAG,oBAAY,CAAC,YAAY;QAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAExC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAU,CAAC,kBAAkB,CAAC;QAC9E,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;KACzC;EAjBqC,CAiBpC;AAqkBN;;;;;;;GAOG;AACH,SAAS,6BAA6B,CAAC,MAAgB,EAAE,QAA+B;IACpF,MAAM,UAAU,GAA4B;QACxC,YAAY;QACZ,eAAe;QACf,SAAS;QACT,UAAU;QACV,aAAa;QACb,YAAY;QACZ,aAAa;QACb,UAAU;QACV,KAAK;QACL,SAAS;KACZ,CAAC;IAEF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAClB,mBAAmB;QACnB,IAAI,MAAM,CAAC,WAAW;YAAE,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,mCAAmC;QAEtG,mBAAmB;QACnB,IAAI,MAAM,CAAC,QAAQ;YAAE,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAEzD,mBAAmB;QACnB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;YACnB,MAAc,CAAC,IAAI,CAAC,GAAI,QAAgB,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;IAED,OAAO,MAAkC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,SAAS,4BAA4B,CAAC,QAAiB;IACnD,IAAA,YAAE,EACE,QAAQ,EACR,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QACnB,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,IAAI,oBAAe,EAA2B;KAC1D,CAAC,CACL,CAAC;IAEF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAC1B,IAAI,CAAC;YACD,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAW,CAAC,CAAC;QACnF,CAAC;QAAC,MAAM,CAAC;YACL,4FAA4F;QAChG,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GACxB,oBAAI,CAAC,WAAW,CAAC,IAAA,mBAAO,EAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,yCAAyC,CAAC,CAAC,0FAA0F;QAC1L,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACxE,CAAC;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,cAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
1
+ {"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,IAAI,MAAM,WAAW,CAAC;AAe7B,OAAO,EACH,iCAAiC,EACjC,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,yBAAyB,EACzB,YAAY,EACZ,MAAM,EACN,YAAY,EACZ,UAAU,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,IAAI,MAAM,YAAY,CAAC;AAC9B,OAAO,EAAE,EAAE,EAAE,eAAe,EAAE,MAAM,IAAI,CAAC;AAGzC,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE5E,IAAI,YAAsC,CAAC;AAW3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AAuLF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAEX,SAAQ,YAAqB;IA0CL;IAzCtB;;;OAGG;IACH,kBAAkB,CAAsB;IAE9B,+BAA+B,CAAS;IACxC,kBAAkB,CAA8B;IAChD,mBAAmB,CAA8B;IACjD,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,kBAAkB,CAAC;QAC9E,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UAAuC,EAAE,EACvB,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,cAAc,EACd,yBAAyB,GAAG,EAAE,EAC9B,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,EAClB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,cAAc;YACd,qBAAqB;YACrB,6DAA6D;YAC7D,uDAAuD;YACvD,yBAAyB,EACrB,qBAAqB,GAAG,yBAAyB,GAAG,iCAAiC;SAC5F,EACD,MAAM,CACT,CAAC;QAnCgB,WAAM,GAAN,MAAM,CAAkC;QAqC1D,YAAY;QACZ,IAAI,CAAC,cAAc,GAAI,cAAsB,IAAI,IAAI,CAAC,MAAM,CAAC;QAE7D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,+BAA+B,GAAG,yBAAyB,GAAG,IAAI,CAAC;QACxE,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,SAA2B;QAC3B,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAExC,MAAM,gBAAgB,GAAG,SAAS,CAAC,iBAAiB,EAAE,CAAC;QAEvD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1D,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAChD,MAAM,YAAY,GAAG,OAAO,IAAI,CAAC,GAAiB,CAAC,CAAC;YACpD,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,CAAC,0DAA0D;YAC9F,MAAM,UAAU,GAAG,YAAY,KAAK,aAAa,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,+BAA+B;YACnG,MAAM,MAAM,GAAG,IAAI,CAAC,GAAiB,CAAC,IAAI,IAAI,CAAC;YAE/C,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,wDAAwD;gBACxD,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,2BAA2B,GAAG,iCAAiC,SAAS,YAAY,CACxG,CAAC;YACN,CAAC;YAED,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,CAAC;gBACxB,0DAA0D;gBAC1D,MAAM,IAAI,KAAK,CACX,GAAG,SAAS,CAAC,IAAI,6CAA6C,aAAa,OAAO,SAAS,IAAI,GAAG,KAAK,YAAY,IAAI,CAC1H,CAAC;YACN,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,IAAI,mBAAmB,SAAS,IAAI,GAAG,KAAK,YAAY,UAAU,KAAK,GAAG,CAAC,CAAC;YAE1G,IAAI,CAAC,GAAiB,CAAC,GAAG,KAAyB,CAAC;QACxD,CAAC;IACL,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,eAAe,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,eAAe,CAAC,QAAS,EAAE,eAAe,CAAC,CAAC;YAC9F,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;YACxC,SAAS,EAAE,CAAC;YAEZ,qEAAqE;YACrE,eAAe,CAAC,eAAe,KAAK,KAAK,EAAE,QAAiB,EAAE,UAAmB,EAAE,EAAE;gBACjF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;gBACzD,CAAC;YACL,CAAC,CAAC;YACF,eAAe,CAAC,gBAAgB,KAAK,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;gBACjF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAEhD,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,eAAe,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAC/D,CAAC;gBAED,OAAO,CAAC,CAAC;YACb,CAAC,CAAC;YAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,UAAW,CAAC,CAAC;YAChF,CAAC;YAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;YAC9D,CAAC;YAED,OAAO,CAAC,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC;YAEjC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK;gBACV,wCAAwC;gBACxC,oBAAoB,OAAO,CAAC,EAAE,mBAAmB,OAAO,CAAC,GAAG,mBAAmB,OAAO,CAAC,SAAS,qDAAqD,OAAO,CAAC,iBAAiB,CAAC,IAAI,CACtL,CAAC;gBAEF,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;gBACvB,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC;gBAErC,OAAO;YACX,CAAC;YAED,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;YAEvC,MAAM,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,EAAE;gBAC3C,GAAG;oBACC,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;wBAAE,OAAO,IAAI,CAAC;oBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;oBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAClC,CAAC;aACJ,CAAC,CAAC;QACP,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,eAAe,CAAC;QAC7C,IAAI,CAAC;YACD,MAAM,mBAAmB,CACrB,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,eAAyC,CAAC,CAAC,EAC3F,IAAI,CAAC,+BAA+B,EACpC,kCAAkC,IAAI,CAAC,+BAA+B,GAAG,IAAI,WAAW,CAC3F,CAAC;YACF,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC;QACtC,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAEkB,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QAC9D,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAC7C,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,eAAe,CAAC,QAAQ,GAAG,MAAM,mBAAmB,CAChD,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAC7E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QACvC,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAChF,SAAS,EAAE,CAAC;IAChB,CAAC;IAED;;OAEG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,UAAuB,EACvB,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;QAE9F,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC3D,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YAEhF,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAC/I,CAAC;YAEF,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,wBAAwB,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QAEtD,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC1C,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAK,EAAE,CAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACW;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,SAAqC,CAAC;YACjD,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,cAA+B,EAAE,eAAwB;QACtG,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAW,IAAI,GAAG,IAAI,UAAW,IAAI,GAAG,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,UAAW,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QACtE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QAE1E,IAAI,CAAC,UAAW,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,uCAAuC,CAAC,CAAC;YAC1E,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC;YACvE,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;QACvD,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YAClD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;gBACX,YAAY,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;aAChG,CAAC;QACN,CAAC;IACL,CAAC;IAES,KAAK,CAAC,UAAU,CACtB,QAAyB,EACzB,MAAe,EACf,gBAAyB;QAEzB,OAAO;YACH,IAAI,EAAE,MAAM,oBAAoB,CAAC,QAAQ,CAAC;SACzB,CAAC;IAC1B,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAA0E;YAC1F,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,2EAA2E;QAC3E,sEAAsE;QACtE,2EAA2E;QAC3E,4CAA4C;QAC5C,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,CAAC;YACtE,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAgB,EAChB,QAAyB,EACzB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACtE,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAIpE,CAAC;YACF,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAsB;gBAChC,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,2BAA2B,GAAG,IAAI,WAAW,CAAC,CAAC;IACnG,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAyB;QACpE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,UAAW,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,UAAW,CAAC,CAAC;QAE9F,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAC7B,OAA4D,EAC5D,OAAiB,EACnB,EAAE;QACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,yBAAyB,CAAC;YACtB,GAAG,OAAO;YACV,SAAS,EAAE,OAAO,CAAC,SAAgB,EAAE,oDAAoD;YACzF,YAAY,EAAE,MAAM;SACvB,CAAC,EACF,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;gBAElD,MAAM,YAAY,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC9E,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;oBACtB,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;gBACjD,CAAC;YACL,CAAC;QACL,CAAC,CACJ,CAAC;QAEF,OAAO,6BAA6B,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC,CAAC;;AAUN;;;;;;;GAOG;AACH,SAAS,6BAA6B,CAAC,MAAgB,EAAE,QAA+B;IACpF,MAAM,UAAU,GAA4B;QACxC,YAAY;QACZ,eAAe;QACf,SAAS;QACT,UAAU;QACV,aAAa;QACb,YAAY;QACZ,aAAa;QACb,UAAU;QACV,KAAK;QACL,SAAS;KACZ,CAAC;IAEF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAClB,mBAAmB;QACnB,IAAI,MAAM,CAAC,WAAW;YAAE,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,mCAAmC;QAEtG,mBAAmB;QACnB,IAAI,MAAM,CAAC,QAAQ;YAAE,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAEzD,mBAAmB;QACnB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;YACnB,MAAc,CAAC,IAAI,CAAC,GAAI,QAAgB,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;IAED,OAAO,MAAkC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,SAAS,4BAA4B,CAAC,QAAiB;IACnD,EAAE,CACE,QAAQ,EACR,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QACnB,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,IAAI,eAAe,EAA2B;KAC1D,CAAC,CACL,CAAC;IAEF,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAC1B,IAAI,CAAC;YACD,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAW,CAAC,CAAC;QACnF,CAAC;QAAC,MAAM,CAAC;YACL,4FAA4F;QAChG,CAAC;IACL,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GACxB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,yCAAyC,CAAC,CAAC,0FAA0F;QAC1L,iBAAiB,GAAG,iBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACxE,CAAC;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
package/package.json CHANGED
@@ -1,19 +1,13 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "3.13.6-beta.0",
3
+ "version": "4.0.0-beta.0",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
- "node": ">=16.0.0"
6
+ "node": ">=22.0.0"
7
7
  },
8
- "main": "./index.js",
9
- "module": "./index.mjs",
10
- "types": "./index.d.ts",
8
+ "type": "module",
11
9
  "exports": {
12
- ".": {
13
- "import": "./index.mjs",
14
- "require": "./index.js",
15
- "types": "./index.d.ts"
16
- },
10
+ ".": "./index.js",
17
11
  "./package.json": "./package.json"
18
12
  },
19
13
  "keywords": [
@@ -46,27 +40,27 @@
46
40
  "scripts": {
47
41
  "build": "yarn clean && yarn compile && yarn copy",
48
42
  "clean": "rimraf ./dist",
49
- "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
43
+ "compile": "tsc -p tsconfig.build.json",
50
44
  "copy": "tsx ../../scripts/copy.ts"
51
45
  },
52
46
  "publishConfig": {
53
47
  "access": "public"
54
48
  },
55
49
  "dependencies": {
56
- "@apify/timeout": "^0.3.0",
57
- "@apify/utilities": "^2.7.10",
58
- "@crawlee/basic": "^3.13.6-beta.0",
59
- "@crawlee/types": "^3.13.6-beta.0",
60
- "@crawlee/utils": "^3.13.6-beta.0",
61
- "@types/content-type": "^1.1.5",
62
- "cheerio": "1.0.0-rc.12",
63
- "content-type": "^1.0.4",
64
- "got-scraping": "^4.0.0",
50
+ "@apify/timeout": "^0.3.2",
51
+ "@apify/utilities": "^2.15.5",
52
+ "@crawlee/basic": "4.0.0-beta.0",
53
+ "@crawlee/types": "4.0.0-beta.0",
54
+ "@crawlee/utils": "4.0.0-beta.0",
55
+ "@types/content-type": "^1.1.8",
56
+ "cheerio": "^1.0.0",
57
+ "content-type": "^1.0.5",
58
+ "got-scraping": "^4.1.1",
65
59
  "iconv-lite": "^0.6.3",
66
- "mime-types": "^2.1.35",
67
- "ow": "^0.28.1",
68
- "tslib": "^2.4.0",
69
- "type-fest": "^4.0.0"
60
+ "mime-types": "^3.0.1",
61
+ "ow": "^2.0.0",
62
+ "tslib": "^2.8.1",
63
+ "type-fest": "^4.41.0"
70
64
  },
71
65
  "lerna": {
72
66
  "command": {
@@ -75,5 +69,5 @@
75
69
  }
76
70
  }
77
71
  },
78
- "gitHead": "595148da5825258b9c92e49071ae6c76bc824c86"
72
+ "gitHead": "927bdafa403ff347327158b01d20b817378168a7"
79
73
  }