@crawlee/basic 3.0.0-beta.8 → 3.0.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -133
- package/index.mjs +69 -0
- package/internals/basic-crawler.d.ts +164 -138
- package/internals/basic-crawler.d.ts.map +1 -1
- package/internals/basic-crawler.js +168 -74
- package/internals/basic-crawler.js.map +1 -1
- package/internals/constants.d.ts +3 -2
- package/internals/constants.d.ts.map +1 -1
- package/internals/constants.js +3 -2
- package/internals/constants.js.map +1 -1
- package/package.json +22 -22
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.BasicCrawler = void 0;
|
|
3
|
+
exports.createBasicRouter = exports.BasicCrawler = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
6
6
|
const timeout_1 = require("@apify/timeout");
|
|
@@ -12,7 +12,7 @@ const ow_1 = tslib_1.__importStar(require("ow"));
|
|
|
12
12
|
/**
|
|
13
13
|
* Since there's no set number of seconds before the container is terminated after
|
|
14
14
|
* a migration event, we need some reasonable number to use for RequestList persistence.
|
|
15
|
-
* Once a migration event is received, the
|
|
15
|
+
* Once a migration event is received, the crawler will be paused, and it will wait for
|
|
16
16
|
* this long before persisting the RequestList state. This should allow most healthy
|
|
17
17
|
* requests to finish and be marked as handled, thus lowering the amount of duplicate
|
|
18
18
|
* results after migration.
|
|
@@ -26,62 +26,60 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;
|
|
|
26
26
|
*
|
|
27
27
|
* `BasicCrawler` is a low-level tool that requires the user to implement the page
|
|
28
28
|
* download and data extraction functionality themselves.
|
|
29
|
-
* If
|
|
30
|
-
*
|
|
29
|
+
* If we want a crawler that already facilitates this functionality,
|
|
30
|
+
* we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
|
|
31
31
|
*
|
|
32
|
-
* `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler}
|
|
32
|
+
* `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
|
|
33
33
|
* for each {@link Request} object, which represents a single URL to crawl.
|
|
34
|
-
* The {@link Request} objects are fed from the {@link RequestList} or
|
|
35
|
-
* instances provided by the {@link BasicCrawlerOptions.requestList} or {@link BasicCrawlerOptions.requestQueue}
|
|
36
|
-
* constructor options, respectively.
|
|
34
|
+
* The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
|
|
35
|
+
* instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
|
|
36
|
+
* constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
|
|
37
|
+
* the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
|
|
38
|
+
* or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
|
|
37
39
|
*
|
|
38
|
-
* If both {@link BasicCrawlerOptions.requestList} and {@link BasicCrawlerOptions.requestQueue} options are used,
|
|
40
|
+
* If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
|
|
39
41
|
* the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
|
|
40
|
-
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
42
|
+
* to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
|
|
41
43
|
*
|
|
42
44
|
* The crawler finishes if there are no more {@link Request} objects to crawl.
|
|
43
45
|
*
|
|
44
46
|
* New requests are only dispatched when there is enough free CPU and memory available,
|
|
45
47
|
* using the functionality provided by the {@link AutoscaledPool} class.
|
|
46
|
-
* All {@link AutoscaledPool} configuration options can be passed to the
|
|
47
|
-
* parameter of the `BasicCrawler` constructor.
|
|
48
|
-
*
|
|
48
|
+
* All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
|
|
49
|
+
* parameter of the `BasicCrawler` constructor.
|
|
50
|
+
* For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
|
|
51
|
+
* {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
|
|
52
|
+
* underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
|
|
49
53
|
*
|
|
50
54
|
* **Example usage:**
|
|
51
55
|
*
|
|
52
56
|
* ```javascript
|
|
53
|
-
*
|
|
57
|
+
* import { BasicCrawler, Dataset } from 'crawlee';
|
|
54
58
|
*
|
|
55
|
-
* //
|
|
56
|
-
* const requestList = new RequestList({
|
|
57
|
-
* sources: [
|
|
58
|
-
* { url: 'http://www.example.com/page-1' },
|
|
59
|
-
* { url: 'http://www.example.com/page-2' },
|
|
60
|
-
* ],
|
|
61
|
-
* });
|
|
62
|
-
* await requestList.initialize();
|
|
63
|
-
*
|
|
64
|
-
* // Crawl the URLs
|
|
59
|
+
* // Create a crawler instance
|
|
65
60
|
* const crawler = new BasicCrawler({
|
|
66
|
-
*
|
|
67
|
-
* async requestHandler({ request }) {
|
|
61
|
+
* async requestHandler({ request, sendRequest }) {
|
|
68
62
|
* // 'request' contains an instance of the Request class
|
|
69
63
|
* // Here we simply fetch the HTML of the page and store it to a dataset
|
|
70
|
-
* const { body } = await
|
|
64
|
+
* const { body } = await sendRequest({
|
|
71
65
|
* url: request.url,
|
|
72
66
|
* method: request.method,
|
|
73
67
|
* body: request.payload,
|
|
74
68
|
* headers: request.headers,
|
|
75
69
|
* });
|
|
76
70
|
*
|
|
77
|
-
* await
|
|
71
|
+
* await Dataset.pushData({
|
|
78
72
|
* url: request.url,
|
|
79
73
|
* html: body,
|
|
80
74
|
* })
|
|
81
75
|
* },
|
|
82
76
|
* });
|
|
83
77
|
*
|
|
84
|
-
*
|
|
78
|
+
* // Enqueue the initial requests and run the crawler
|
|
79
|
+
* await crawler.run([
|
|
80
|
+
* 'http://www.example.com/page-1',
|
|
81
|
+
* 'http://www.example.com/page-2',
|
|
82
|
+
* ]);
|
|
85
83
|
* ```
|
|
86
84
|
* @category Crawlers
|
|
87
85
|
*/
|
|
@@ -89,7 +87,7 @@ class BasicCrawler {
|
|
|
89
87
|
/**
|
|
90
88
|
* All `BasicCrawler` parameters are passed via an options object.
|
|
91
89
|
*/
|
|
92
|
-
constructor(options, config = core_1.Configuration.getGlobalConfig()) {
|
|
90
|
+
constructor(options = {}, config = core_1.Configuration.getGlobalConfig()) {
|
|
93
91
|
Object.defineProperty(this, "config", {
|
|
94
92
|
enumerable: true,
|
|
95
93
|
configurable: true,
|
|
@@ -97,7 +95,7 @@ class BasicCrawler {
|
|
|
97
95
|
value: config
|
|
98
96
|
});
|
|
99
97
|
/**
|
|
100
|
-
*
|
|
98
|
+
* A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
|
|
101
99
|
*/
|
|
102
100
|
Object.defineProperty(this, "stats", {
|
|
103
101
|
enumerable: true,
|
|
@@ -106,8 +104,7 @@ class BasicCrawler {
|
|
|
106
104
|
value: void 0
|
|
107
105
|
});
|
|
108
106
|
/**
|
|
109
|
-
* A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}
|
|
110
|
-
* Either `requestList` or `requestQueue` option must be provided (or both).
|
|
107
|
+
* A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
|
|
111
108
|
* Only available if used by the crawler.
|
|
112
109
|
*/
|
|
113
110
|
Object.defineProperty(this, "requestList", {
|
|
@@ -118,8 +115,7 @@ class BasicCrawler {
|
|
|
118
115
|
});
|
|
119
116
|
/**
|
|
120
117
|
* Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
|
|
121
|
-
* A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}
|
|
122
|
-
* Either `requestList` or `requestQueue` option must be provided (or both).
|
|
118
|
+
* A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
|
|
123
119
|
* Only available if used by the crawler.
|
|
124
120
|
*/
|
|
125
121
|
Object.defineProperty(this, "requestQueue", {
|
|
@@ -129,7 +125,7 @@ class BasicCrawler {
|
|
|
129
125
|
value: void 0
|
|
130
126
|
});
|
|
131
127
|
/**
|
|
132
|
-
* A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}
|
|
128
|
+
* A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
|
|
133
129
|
* Only available if used by the crawler.
|
|
134
130
|
*/
|
|
135
131
|
Object.defineProperty(this, "sessionPool", {
|
|
@@ -140,10 +136,10 @@ class BasicCrawler {
|
|
|
140
136
|
});
|
|
141
137
|
/**
|
|
142
138
|
* A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
* to pause the crawler by calling {@link AutoscaledPool.pause}
|
|
146
|
-
* or to abort it by calling {@link AutoscaledPool.abort}.
|
|
139
|
+
* > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
|
|
140
|
+
* We can use it to change the concurrency settings on the fly,
|
|
141
|
+
* to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
|
|
142
|
+
* or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
|
|
147
143
|
*/
|
|
148
144
|
Object.defineProperty(this, "autoscaledPool", {
|
|
149
145
|
enumerable: true,
|
|
@@ -151,6 +147,16 @@ class BasicCrawler {
|
|
|
151
147
|
writable: true,
|
|
152
148
|
value: void 0
|
|
153
149
|
});
|
|
150
|
+
/**
|
|
151
|
+
* Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
|
|
152
|
+
* See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
|
|
153
|
+
*/
|
|
154
|
+
Object.defineProperty(this, "router", {
|
|
155
|
+
enumerable: true,
|
|
156
|
+
configurable: true,
|
|
157
|
+
writable: true,
|
|
158
|
+
value: core_1.Router.create()
|
|
159
|
+
});
|
|
154
160
|
Object.defineProperty(this, "log", {
|
|
155
161
|
enumerable: true,
|
|
156
162
|
configurable: true,
|
|
@@ -163,6 +169,12 @@ class BasicCrawler {
|
|
|
163
169
|
writable: true,
|
|
164
170
|
value: void 0
|
|
165
171
|
});
|
|
172
|
+
Object.defineProperty(this, "errorHandler", {
|
|
173
|
+
enumerable: true,
|
|
174
|
+
configurable: true,
|
|
175
|
+
writable: true,
|
|
176
|
+
value: void 0
|
|
177
|
+
});
|
|
166
178
|
Object.defineProperty(this, "failedRequestHandler", {
|
|
167
179
|
enumerable: true,
|
|
168
180
|
configurable: true,
|
|
@@ -223,14 +235,20 @@ class BasicCrawler {
|
|
|
223
235
|
writable: true,
|
|
224
236
|
value: void 0
|
|
225
237
|
});
|
|
238
|
+
Object.defineProperty(this, "_closeEvents", {
|
|
239
|
+
enumerable: true,
|
|
240
|
+
configurable: true,
|
|
241
|
+
writable: true,
|
|
242
|
+
value: void 0
|
|
243
|
+
});
|
|
226
244
|
(0, ow_1.default)(options, 'BasicCrawlerOptions', ow_1.default.object.exactShape(BasicCrawler.optionsShape));
|
|
227
245
|
const { requestList, requestQueue, maxRequestRetries = 3, maxRequestsPerCrawl, autoscaledPoolOptions = {}, sessionPoolOptions = {}, useSessionPool = true,
|
|
228
246
|
// AutoscaledPool shorthands
|
|
229
|
-
minConcurrency, maxConcurrency,
|
|
247
|
+
minConcurrency, maxConcurrency, maxRequestsPerMinute,
|
|
230
248
|
// internal
|
|
231
249
|
log = log_1.default.child({ prefix: this.constructor.name }),
|
|
232
250
|
// Old and new request handler methods
|
|
233
|
-
handleRequestFunction, requestHandler, handleRequestTimeoutSecs, requestHandlerTimeoutSecs, handleFailedRequestFunction, failedRequestHandler, } = options;
|
|
251
|
+
handleRequestFunction, requestHandler, handleRequestTimeoutSecs, requestHandlerTimeoutSecs, errorHandler, handleFailedRequestFunction, failedRequestHandler, } = options;
|
|
234
252
|
this.requestList = requestList;
|
|
235
253
|
this.requestQueue = requestQueue;
|
|
236
254
|
this.log = log;
|
|
@@ -241,7 +259,12 @@ class BasicCrawler {
|
|
|
241
259
|
propertyKey: 'requestHandler',
|
|
242
260
|
newProperty: requestHandler,
|
|
243
261
|
oldProperty: handleRequestFunction,
|
|
262
|
+
allowUndefined: true, // fallback to the default router
|
|
244
263
|
});
|
|
264
|
+
if (!this.requestHandler) {
|
|
265
|
+
this.requestHandler = this.router;
|
|
266
|
+
}
|
|
267
|
+
this.errorHandler = errorHandler;
|
|
245
268
|
this._handlePropertyNameChange({
|
|
246
269
|
newName: 'failedRequestHandler',
|
|
247
270
|
oldName: 'handleFailedRequestFunction',
|
|
@@ -297,6 +320,7 @@ class BasicCrawler {
|
|
|
297
320
|
const basicCrawlerAutoscaledPoolConfiguration = {
|
|
298
321
|
minConcurrency,
|
|
299
322
|
maxConcurrency,
|
|
323
|
+
maxTasksPerMinute: maxRequestsPerMinute ?? autoscaledPoolOptions?.maxTasksPerMinute,
|
|
300
324
|
runTaskFunction: this._runTaskFunction.bind(this),
|
|
301
325
|
isTaskReadyFunction: async () => {
|
|
302
326
|
if (isMaxPagesExceeded()) {
|
|
@@ -336,8 +360,17 @@ class BasicCrawler {
|
|
|
336
360
|
}
|
|
337
361
|
/**
|
|
338
362
|
* Runs the crawler. Returns a promise that gets resolved once all the requests are processed.
|
|
363
|
+
* We can use the `requests` parameter to enqueue the initial requests - it is a shortcut for
|
|
364
|
+
* running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before the {@link BasicCrawler.run|`crawler.run()`}.
|
|
365
|
+
*
|
|
366
|
+
* @param [requests] The requests to add
|
|
367
|
+
* @param [options] Options for the request queue
|
|
339
368
|
*/
|
|
340
|
-
async run() {
|
|
369
|
+
async run(requests, options) {
|
|
370
|
+
await (0, core_1.purgeDefaultStorages)();
|
|
371
|
+
if (requests) {
|
|
372
|
+
await this.addRequests(requests, options);
|
|
373
|
+
}
|
|
341
374
|
await this._init();
|
|
342
375
|
await this.stats.startCapturing();
|
|
343
376
|
try {
|
|
@@ -361,6 +394,10 @@ class BasicCrawler {
|
|
|
361
394
|
this.requestQueue ?? (this.requestQueue = await core_1.RequestQueue.open());
|
|
362
395
|
return this.requestQueue;
|
|
363
396
|
}
|
|
397
|
+
async useState(defaultValue = {}) {
|
|
398
|
+
const kvs = await core_1.KeyValueStore.open(null, { config: this.config });
|
|
399
|
+
return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
|
|
400
|
+
}
|
|
364
401
|
/**
|
|
365
402
|
* Adds requests to be processed by the crawler
|
|
366
403
|
* @param requests The requests to add
|
|
@@ -417,6 +454,10 @@ class BasicCrawler {
|
|
|
417
454
|
};
|
|
418
455
|
}
|
|
419
456
|
async _init() {
|
|
457
|
+
if (!this.events.isInitialized()) {
|
|
458
|
+
await this.events.init();
|
|
459
|
+
this._closeEvents = true;
|
|
460
|
+
}
|
|
420
461
|
// Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),
|
|
421
462
|
// so that the caller can get a reference to it before awaiting the promise returned from run()
|
|
422
463
|
// (otherwise there would be no way)
|
|
@@ -527,7 +568,9 @@ class BasicCrawler {
|
|
|
527
568
|
const statisticsId = request.id || request.uniqueKey;
|
|
528
569
|
this.stats.startJob(statisticsId);
|
|
529
570
|
// Shared crawling context
|
|
530
|
-
// @ts-expect-error
|
|
571
|
+
// @ts-expect-error
|
|
572
|
+
// All missing properties properties (that extend CrawlingContext) are set dynamically,
|
|
573
|
+
// but TS does not know that, so otherwise it would throw when compiling.
|
|
531
574
|
const crawlingContext = {
|
|
532
575
|
id: (0, utilities_1.cryptoRandomObjectId)(10),
|
|
533
576
|
crawler: this,
|
|
@@ -540,22 +583,24 @@ class BasicCrawler {
|
|
|
540
583
|
requestQueue: await this.getRequestQueue(),
|
|
541
584
|
});
|
|
542
585
|
},
|
|
543
|
-
sendRequest: async (
|
|
544
|
-
req ?? (req = request);
|
|
586
|
+
sendRequest: async (overrideOptions) => {
|
|
545
587
|
return (0, got_scraping_1.gotScraping)({
|
|
546
|
-
url:
|
|
547
|
-
method:
|
|
548
|
-
body:
|
|
549
|
-
headers:
|
|
550
|
-
retry: {
|
|
551
|
-
limit: 0,
|
|
552
|
-
},
|
|
588
|
+
url: request.url,
|
|
589
|
+
method: request.method,
|
|
590
|
+
body: request.payload,
|
|
591
|
+
headers: request.headers,
|
|
553
592
|
proxyUrl: crawlingContext.proxyInfo?.url,
|
|
554
593
|
sessionToken: session,
|
|
555
594
|
responseType: 'text',
|
|
595
|
+
...overrideOptions,
|
|
596
|
+
retry: {
|
|
597
|
+
limit: 0,
|
|
598
|
+
...overrideOptions?.retry,
|
|
599
|
+
},
|
|
556
600
|
cookieJar: {
|
|
557
601
|
getCookieString: (url) => session.getCookieString(url),
|
|
558
602
|
setCookie: (rawCookie, url) => session.setCookie(rawCookie, url),
|
|
603
|
+
...overrideOptions?.cookieJar,
|
|
559
604
|
},
|
|
560
605
|
});
|
|
561
606
|
},
|
|
@@ -571,14 +616,15 @@ class BasicCrawler {
|
|
|
571
616
|
}
|
|
572
617
|
catch (err) {
|
|
573
618
|
try {
|
|
574
|
-
await
|
|
619
|
+
await (0, timeout_1.addTimeoutToPromise)(() => this._requestFunctionErrorHandler(err, crawlingContext, source), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
|
|
575
620
|
}
|
|
576
621
|
catch (secondaryError) {
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
622
|
+
if (!secondaryError.triggeredFromUserHandler) {
|
|
623
|
+
const apifySpecific = process.env.APIFY_IS_AT_HOME
|
|
624
|
+
? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.` : '';
|
|
625
|
+
this.log.exception(secondaryError, 'An exception occurred during handling of failed request. '
|
|
626
|
+
+ `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
|
|
627
|
+
}
|
|
582
628
|
throw secondaryError;
|
|
583
629
|
}
|
|
584
630
|
}
|
|
@@ -631,13 +677,18 @@ class BasicCrawler {
|
|
|
631
677
|
async _requestFunctionErrorHandler(error, crawlingContext, source) {
|
|
632
678
|
const { request } = crawlingContext;
|
|
633
679
|
request.pushErrorMessage(error);
|
|
634
|
-
|
|
680
|
+
if (error instanceof core_1.CriticalError) {
|
|
681
|
+
throw error;
|
|
682
|
+
}
|
|
683
|
+
const shouldRetryRequest = !request.noRetry && request.retryCount < this.maxRequestRetries && !(error instanceof core_1.NonRetryableError);
|
|
635
684
|
if (shouldRetryRequest) {
|
|
636
685
|
request.retryCount++;
|
|
686
|
+
await this._tagUserHandlerError(() => this.errorHandler?.(crawlingContext, error));
|
|
637
687
|
const { url, retryCount, id } = request;
|
|
638
688
|
// We don't want to see the stack trace in the logs by default, when we are going to retry the request.
|
|
639
689
|
// Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
|
|
640
|
-
|
|
690
|
+
const message = process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error;
|
|
691
|
+
this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, { id, url, retryCount });
|
|
641
692
|
await source.reclaimRequest(request);
|
|
642
693
|
}
|
|
643
694
|
else {
|
|
@@ -647,23 +698,27 @@ class BasicCrawler {
|
|
|
647
698
|
this.handledRequestsCount++;
|
|
648
699
|
await source.markRequestHandled(request);
|
|
649
700
|
this.stats.failJob(request.id || request.uniqueKey);
|
|
650
|
-
|
|
651
|
-
const castedErrorContext = crawlingContext;
|
|
652
|
-
castedErrorContext.error = error;
|
|
653
|
-
await this._handleFailedRequestHandler(castedErrorContext); // This function prints an error message.
|
|
701
|
+
await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
|
|
654
702
|
}
|
|
655
703
|
}
|
|
656
|
-
async
|
|
657
|
-
|
|
658
|
-
await
|
|
704
|
+
async _tagUserHandlerError(cb) {
|
|
705
|
+
try {
|
|
706
|
+
return await cb();
|
|
659
707
|
}
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
? crawlingContext.error.message
|
|
664
|
-
: crawlingContext.error.stack}`, { id, url, method, uniqueKey });
|
|
708
|
+
catch (e) {
|
|
709
|
+
Object.defineProperty(e, 'triggeredFromUserHandler', { value: true });
|
|
710
|
+
throw e;
|
|
665
711
|
}
|
|
666
712
|
}
|
|
713
|
+
async _handleFailedRequestHandler(crawlingContext, error) {
|
|
714
|
+
if (this.failedRequestHandler) {
|
|
715
|
+
await this._tagUserHandlerError(() => this.failedRequestHandler?.(crawlingContext, error));
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
const { id, url, method, uniqueKey } = crawlingContext.request;
|
|
719
|
+
const message = error instanceof timeout_1.TimeoutError && !process.env.CRAWLEE_VERBOSE_LOG ? error.message : error.stack;
|
|
720
|
+
this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
|
|
721
|
+
}
|
|
667
722
|
/**
|
|
668
723
|
* Updates handledRequestsCount from possibly stored counts,
|
|
669
724
|
* usually after worker migration. Since one of the stores
|
|
@@ -692,9 +747,13 @@ class BasicCrawler {
|
|
|
692
747
|
* @ignore
|
|
693
748
|
*/
|
|
694
749
|
async teardown() {
|
|
750
|
+
this.events.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
|
|
695
751
|
if (this.useSessionPool) {
|
|
696
752
|
await this.sessionPool.teardown();
|
|
697
753
|
}
|
|
754
|
+
if (this._closeEvents) {
|
|
755
|
+
await this.events.close();
|
|
756
|
+
}
|
|
698
757
|
}
|
|
699
758
|
_handlePropertyNameChange({ newProperty, newName, oldProperty, oldName, propertyKey, allowUndefined = false, }) {
|
|
700
759
|
if (newProperty && oldProperty) {
|
|
@@ -727,6 +786,12 @@ class BasicCrawler {
|
|
|
727
786
|
}
|
|
728
787
|
}
|
|
729
788
|
exports.BasicCrawler = BasicCrawler;
|
|
789
|
+
Object.defineProperty(BasicCrawler, "CRAWLEE_STATE_KEY", {
|
|
790
|
+
enumerable: true,
|
|
791
|
+
configurable: true,
|
|
792
|
+
writable: true,
|
|
793
|
+
value: 'CRAWLEE_STATE'
|
|
794
|
+
});
|
|
730
795
|
Object.defineProperty(BasicCrawler, "optionsShape", {
|
|
731
796
|
enumerable: true,
|
|
732
797
|
configurable: true,
|
|
@@ -737,13 +802,13 @@ Object.defineProperty(BasicCrawler, "optionsShape", {
|
|
|
737
802
|
// Subclasses override this function instead of passing it
|
|
738
803
|
// in constructor, so this validation needs to apply only
|
|
739
804
|
// if the user creates an instance of BasicCrawler directly.
|
|
740
|
-
// TODO: remove .optional from requestHandler once migration period is over
|
|
741
805
|
requestHandler: ow_1.default.optional.function,
|
|
742
806
|
// TODO: remove in a future release
|
|
743
807
|
handleRequestFunction: ow_1.default.optional.function,
|
|
744
808
|
requestHandlerTimeoutSecs: ow_1.default.optional.number,
|
|
745
809
|
// TODO: remove in a future release
|
|
746
810
|
handleRequestTimeoutSecs: ow_1.default.optional.number,
|
|
811
|
+
errorHandler: ow_1.default.optional.function,
|
|
747
812
|
failedRequestHandler: ow_1.default.optional.function,
|
|
748
813
|
// TODO: remove in a future release
|
|
749
814
|
handleFailedRequestFunction: ow_1.default.optional.function,
|
|
@@ -755,8 +820,37 @@ Object.defineProperty(BasicCrawler, "optionsShape", {
|
|
|
755
820
|
// AutoscaledPool shorthands
|
|
756
821
|
minConcurrency: ow_1.default.optional.number,
|
|
757
822
|
maxConcurrency: ow_1.default.optional.number,
|
|
823
|
+
maxRequestsPerMinute: ow_1.default.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
|
|
758
824
|
// internal
|
|
759
825
|
log: ow_1.default.optional.object,
|
|
760
826
|
}
|
|
761
827
|
});
|
|
828
|
+
/**
|
|
829
|
+
* Creates new {@link Router} instance that works based on request labels.
|
|
830
|
+
* This instance can then serve as a {@link BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@link BasicCrawler}.
|
|
831
|
+
* Defaults to the {@link BasicCrawlingContext}.
|
|
832
|
+
*
|
|
833
|
+
* > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
|
|
834
|
+
*
|
|
835
|
+
* ```ts
|
|
836
|
+
* import { BasicCrawler, createBasicRouter } from 'crawlee';
|
|
837
|
+
*
|
|
838
|
+
* const router = createBasicRouter();
|
|
839
|
+
* router.addHandler('label-a', async (ctx) => {
|
|
840
|
+
* ctx.log.info('...');
|
|
841
|
+
* });
|
|
842
|
+
* router.addDefaultHandler(async (ctx) => {
|
|
843
|
+
* ctx.log.info('...');
|
|
844
|
+
* });
|
|
845
|
+
*
|
|
846
|
+
* const crawler = new BasicCrawler({
|
|
847
|
+
* requestHandler: router,
|
|
848
|
+
* });
|
|
849
|
+
* await crawler.run();
|
|
850
|
+
* ```
|
|
851
|
+
*/
|
|
852
|
+
function createBasicRouter() {
|
|
853
|
+
return core_1.Router.create();
|
|
854
|
+
}
|
|
855
|
+
exports.createBasicRouter = createBasicRouter;
|
|
762
856
|
//# sourceMappingURL=basic-crawler.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"basic-crawler.js","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":";;;;AAAA,6DAA6C;AAC7C,4CAA8E;AAC9E,gDAAwD;AACxD,wCAwBuB;AACvB,+CAA+F;AAE/F,0CAAyD;AACzD,iDAAuC;AAevC;;;;;;;;GAQG;AACH,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAsLzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiEG;AACH,MAAa,YAAY;IAkFrB;;OAEG;IACH,YAAY,OAAmD,EAAW,SAAS,oBAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAjF1E;;WAEG;QACH;;;;;WAA2B;QAE3B;;;;WAIG;QACH;;;;;WAA0B;QAE1B;;;;;WAKG;QACH;;;;;WAA4B;QAE5B;;;WAGG;QACH;;;;;WAA0B;QAE1B;;;;;;WAMG;QACH;;;;;WAAgC;QAEhC;;;;;WAAmB;QACnB;;;;;WAAmD;QACnD;;;;;WAAoE;QACpE;;;;;WAA+C;QAC/C;;;;;WAAwC;QACxC;;;;;WAAoC;QACpC;;;;;WAAuC;QACvC;;;;;WAAiD;QACjD;;;;;WAAkC;QAClC;;;;mBAA6B,IAAI,GAAG,EAAmB;WAAC;QACxD;;;;;WAAuD;QACvD;;;;;WAA+B;QAoC3B,IAAA,YAAE,EAAC,OAAO,EAAE,qBAAqB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;QAEpF,MAAM,EACF,WAAW,EACX,YAAY,EACZ,iBAAiB,GAAG,CAAC,EACrB,mBAAmB,EACnB,qBAAqB,GAAG,EAAE,EAC1B,kBAAkB,GAAG,EAAE,EACvB,cAAc,GAAG,IAAI;QAErB,4BAA4B;QAC5B,cAAc,EACd,cAAc;QAEd,WAAW;QACX,GAAG,GAAG,aAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QAEzD,sCAAsC;QACtC,qBAAqB,EACrB,cAAc,EAEd,wBAAwB,EACxB,yBAAyB,EAEzB,2BAA2B,EAC3B,oBAAoB,GACvB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,uBAAuB;YAChC,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,qBAAqB;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,sBAAsB;YAC/B,OAAO,EAAE,6BAA6B;YACtC,WAAW,EAAE,sBAAsB;YACnC,WAAW,EAAE,oBAAoB;YACjC,WAAW,EAAE,2BAA2B;YACxC,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,wBAA4C,CAAC;QAEjD,IAAI,CAAC,wBAAwB,EAAE;YAC3B,IAAI,CAAC,yBAAyB,EAAE;gBAC5B,wBAAwB,GAAG,KAAM,CAAC;aACrC;iBAAM;gBACH,wBAAwB,GAAG,yBAAyB,GAAG,IAAI,CAAC;aAC/D;SACJ;aAAM,IAAI,yBAAyB,EAAE;YAClC,wBAAwB,GAAG,yBAAyB,GAAG,IAAI,CAAC;SAC/D;QAED,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,2BAA2B;YACpC,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,6BAA6B;YAC1C,WAAW,EAAE,wBAAwB;YACrC,WAAW,EAAE,wBAAwB,CAAC,CAAC,CAAC,wBAAwB,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS;SACtF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,CAAC,GAAY,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7D,4CAA4C;QAC5C,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACnI,kGAAkG;QAClG,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,YAAY,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC;SACxE;QAED,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAU,CAAC,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB,EAAE,MAAM,EAAE,CAAC,CAAC;QACtG,IAAI,CAAC,kBAAkB,GAAG;YACtB,GAAG,kBAAkB;YACrB,GAAG;SACN,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;QAElC,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,2BAA2B,GAAG,gBAAgB,EAAE;YACrD,GAAG,CAAC,OAAO,CAAC,+BAA+B,IAAI,CAAC,2BAA2B,EAAE;kBACvE,+DAA+D,gBAAgB,EAAE,CAAC,CAAC;YAEzF,IAAI,CAAC,2BAA2B,GAAG,gBAAgB,CAAC;SACvD;QAED,IAAI,yBAAyB,GAAG,IAAI,CAAC;QACrC,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,mBAAmB,IAAI,mBAAmB,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAEzG,MAAM,EAAE,kBAAkB,EAAE,GAAG,qBAAqB,CAAC;QAErD,MAAM,uCAAuC,GAAG;YAC5C,cAAc;YACd,cAAc;YACd,eAAe,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,mBAAmB,EAAE,KAAK,IAAI,EAAE;gBAC5B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,IAAI,yBAAyB,EAAE;wBAC3B,GAAG,CAAC,IAAI,CAAC,mDAAmD;8BACtD,GAAG,mBAAmB,6FAA6F,CAAC,CAAC;wBAC3H,yBAAyB,GAAG,KAAK,CAAC;qBACrC;oBACD,OAAO,KAAK,CAAC;iBAChB;gBAED,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvC,CAAC;YACD,kBAAkB,EAAE,KAAK,IAAI,EAAE;gBAC3B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,GAAG,CAAC,IAAI,CAAC,iEAAiE,mBAAmB,YAAY;0BACnG,yEAAyE;0BACzE,mCAAmC,IAAI,CAAC,oBAAoB,+BAA+B,CAAC,CAAC;oBACnG,OAAO,IAAI,CAAC;iBACf;gBAED,MAAM,UAAU,GAAG,kBAAkB;oBACjC,CAAC,CAAC,MAAM,kBAAkB,EAAE;oBAC5B,CAAC,CAAC,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAE9C,IAAI,UAAU,EAAE;oBACZ,MAAM,MAAM,GAAG,kBAAkB;wBAC7B,CAAC,CAAC,mFAAmF;wBACrF,CAAC,CAAC,0GAA0G,CAAC;oBACjH,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACpB;gBAED,OAAO,UAAU,CAAC;YACtB,CAAC;YACD,GAAG;SACN,CAAC;QAEF,IAAI,CAAC,qBAAqB,GAAG,EAAE,GAAG,qBAAqB,EAAE,GAAG,uCAAuC,EAAE,CAAC;QAEtG,wEAAwE;QACxE,IAAI,CAAC,MAAM,CAAC,EAAE,wCAAsB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,MAAM,CAAC,EAAE,sCAAqB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACL,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAElC,IAAI;YACA,MAAM,IAAI,CAAC,cAAe,CAAC,GAAG,EAAE,CAAC;SACpC;gBAAS;YACN,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;SACpC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,gBAAgB;YACnD,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc;YAC/C,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB;YAChD,GAAG,UAAU;SAChB,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;QAElD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,eAAe;QACjB,IAAI,CAAC,YAAY,KAAjB,IAAI,CAAC,YAAY,GAAK,MAAM,mBAAY,CAAC,IAAI,EAAE,EAAC;QAEhD,OAAO,IAAI,CAAC,YAAa,CAAC;IAC9B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,QAA+C,EAAE,UAAqC,EAAE;QACtG,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAClE,GAAG,EAAE,YAAE,CAAC,MAAM;YACd,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC,CAAC,CAAC;QACN,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACnD,CAAC,CAAC,CAAC;QAEJ,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAClD,MAAM,aAAa,GAAG,IAAA,qBAAc,EAAC,QAAQ,CAAC,CAAC;QAE/C,MAAM,uCAAuC,GAAG,KAAK,EAAE,gBAA2B,EAAE,EAAE;YAClF,MAAM,eAAe,GAAuB,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;YACrG,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAErD,IAAI,SAAS,CAAC,mBAAmB,CAAC,MAAM,EAAE;gBACtC,MAAM,IAAA,aAAK,EAAC,IAAI,CAAC,CAAC;gBAElB,eAAe,CAAC,IAAI,CAAC,GAAG,MAAM,uCAAuC,CACjE,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,CAC1G,CAAC,CAAC;aACN;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAEnD,uDAAuD;QACvD,MAAM,aAAa,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC;QAElF,mDAAmD;QACnD,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;YACvB,OAAO;gBACH,aAAa;gBACb,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;SACL;QAED,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAqB,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9D,MAAM,MAAM,GAAG,IAAA,aAAK,EAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC1C,MAAM,kBAAkB,GAAuB,EAAE,CAAC;YAElD,KAAK,MAAM,YAAY,IAAI,MAAM,EAAE;gBAC/B,kBAAkB,CAAC,IAAI,CAAC,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC,CAAC;gBAExF,MAAM,IAAA,aAAK,EAAC,IAAI,CAAC,CAAC;aACrB;YAED,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,0GAA0G;QAC1G,IAAI,OAAO,CAAC,2BAA2B,EAAE;YACrC,aAAa,CAAC,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC;SACxC;QAED,OAAO;YACH,aAAa;YACb,2BAA2B,EAAE,OAAO;SACvC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,KAAK;QACjB,wEAAwE;QACxE,+FAA+F;QAC/F,oCAAoC;QACpC,IAAI,CAAC,cAAc,GAAG,IAAI,qBAAc,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAElF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,WAAW,GAAG,MAAM,kBAAW,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACnE,gEAAgE;YAChE,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACxC;QAED,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;IAC1C,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QACvD,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,OAAgB,EAAE,UAAkB;QACjE,MAAM,SAAS,GAAG,OAAO,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAEjE,IAAI,SAAS,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,eAAe,CAAC,CAAC;SAC5E;IACL,CAAC;IAES,KAAK,CAAC,iBAAiB;QAC7B,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,+CAA+C;YAC/C,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,0BAA0B,CAAC;iBACtD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAC,EAAE;oBACtD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2DAA2D;0BACpE,uFAAuF,CAAC,CAAC;iBAClG;qBAAM;oBACH,MAAM,GAAG,CAAC;iBACb;YACL,CAAC,CAAC,CAAC;SACV;QAED,MAAM,yBAAyB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;oBAAE,OAAO;gBAChD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;qBAChC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE;wBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0FAA0F;8BACnG,2GAA2G;8BAC3G,8FAA8F,CAAC,CAAC;qBACzG;yBAAM;wBACH,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,gDAAgD;8BAClE,iDAAiD,CAAC,CAAC;qBAC5D;gBACL,CAAC,CAAC,CAAC;aACV;QACL,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,yBAAyB;YACzB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;SAC5B,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,YAAa,CAAC,gBAAgB,EAAE,CAAC;QACpE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC;QACvC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI;YACA,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,GAAG,EAAE;YACV,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yGAAyG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACvI,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;SACf;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;IAChD,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAErF,IAAI,OAAmC,CAAC;QACxC,IAAI,OAA4B,CAAC;QAEjC,MAAM,IAAI,CAAC,gBAAgB,CACvB,KAAK,IAAI,EAAE;YACP,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,CAAC,EACD,IAAI,CAAC,qBAAqB,EAC1B,yCAAyC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CACvF,CAAC;QAEF,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,gBAAgB,CACvB,KAAK,IAAI,EAAE;gBACP,OAAO,GAAG,MAAM,IAAI,CAAC,WAAY,CAAC,UAAU,EAAE,CAAC;YACnD,CAAC,EACD,IAAI,CAAC,qBAAqB,EAC1B,oCAAoC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAClF,CAAC;SACL;QAED,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,gEAAgE;QAChE,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,MAAM,YAAY,GAAG,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElC,0BAA0B;QAC1B,8DAA8D;QAC9D,MAAM,eAAe,GAAY;YAC7B,EAAE,EAAE,IAAA,gCAAoB,EAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,OAAO;YACP,OAAO;YACP,YAAY,EAAE,KAAK,EAAE,cAA+C,EAAE,EAAE;gBACpE,OAAO,IAAA,mBAAY,EAAC;oBAChB,GAAG,cAAc;oBACjB,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;iBAC7C,CAAC,CAAC;YACP,CAAC;YACD,WAAW,EAAE,KAAK,EAAE,GAAa,EAAE,EAAE;gBACjC,GAAG,KAAH,GAAG,GAAK,OAAQ,EAAC;gBACjB,OAAO,IAAA,0BAAW,EAAC;oBACf,GAAG,EAAE,GAAG,CAAC,GAAG;oBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,IAAI,EAAE,GAAG,CAAC,OAAO;oBACjB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,KAAK,EAAE;wBACH,KAAK,EAAE,CAAC;qBACX;oBACD,QAAQ,EAAE,eAAe,CAAC,SAAS,EAAE,GAAG;oBACxC,YAAY,EAAE,OAAO;oBACrB,YAAY,EAAE,MAAM;oBACpB,SAAS,EAAE;wBACP,eAAe,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAQ,CAAC,eAAe,CAAC,GAAG,CAAC;wBAC/D,SAAS,EAAE,CAAC,SAAiB,EAAE,GAAW,EAAE,EAAE,CAAC,OAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC;qBACpF;iBACyB,CAAC,CAAC;YACpC,CAAC;SACJ,CAAC;QAEF,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAE/D,IAAI;YACA,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,EAC9C,IAAI,CAAC,2BAA2B,EAChC,kCAAkC,IAAI,CAAC,2BAA2B,GAAG,IAAI,aAAa,OAAO,CAAC,EAAE,IAAI,CACvG,CAAC;YAEF,MAAM,IAAI,CAAC,gBAAgB,CACvB,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAQ,CAAC,EACzC,IAAI,CAAC,qBAAqB,EAC1B,mBAAmB,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAC3H,CAAC;YAEF,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,mDAAmD;YACnD,OAAO,EAAE,QAAQ,EAAE,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACV,IAAI;gBACA,MAAM,IAAI,CAAC,gBAAgB,CACvB,GAAG,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAY,EAAE,eAAe,EAAE,MAAM,CAAC,EAC9E,IAAI,CAAC,qBAAqB,EAC1B,+BAA+B,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,qBAAqB,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAC5H,CAAC;aACL;YAAC,OAAO,cAAc,EAAE;gBACrB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,cAAuB,EAAE,oDAAoD;sBAC1F,6GAA6G;sBAC7G,qGAAqG;sBACrG,4GAA4G;sBAC5G,sGAAsG,CAAC,CAAC;gBAC9G,MAAM,cAAc,CAAC;aACxB;SACJ;gBAAS;YACN,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACpD;IACL,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,gBAAgB,CAAC,OAA+B,EAAE,OAAe,EAAE,KAAqB,EAAE,UAAU,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC;QACjI,IAAI;YACA,MAAM,IAAA,6BAAmB,EAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;SACtD;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,OAAO,IAAI,UAAU,EAAE,EAAE,0CAA0C;gBACnE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAI,CAAW,CAAC,OAAO,cAAc,OAAO,IAAI,UAAU,GAAG,CAAC,CAAC;gBAChF,OAAO,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;aAClF;YAED,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,sDAAsD;QACtD,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxF,+EAA+E;QAC/E,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QACrC,6EAA6E;QAC7E,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC5E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,0BAA0B;QACtC,MAAM,CACF,qBAAqB,EACrB,sBAAsB,EACzB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;YACvD,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;SAC5D,CAAC,CAAC;QACH,6DAA6D;QAC7D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;IAC3D,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,4BAA4B,CACxC,KAAY,EACZ,eAAwB,EACxB,MAAkC;QAElC,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QACpC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEhC,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAC3F,IAAI,kBAAkB,EAAE;YACpB,OAAO,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;YACxC,uGAAuG;YACvG,yGAAyG;YACzG,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,wDAAwD,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,EAChH,EAAE,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,CAC1B,CAAC;YACF,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;SACxC;aAAM;YACH,sDAAsD;YACtD,wEAAwE;YACxE,+CAA+C;YAC/C,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC5B,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;YAEpD,8DAA8D;YAC9D,MAAM,kBAAkB,GAAG,eAA+B,CAAC;YAC3D,kBAAkB,CAAC,KAAK,GAAG,KAAK,CAAC;YAEjC,MAAM,IAAI,CAAC,2BAA2B,CAAC,kBAAkB,CAAC,CAAC,CAAC,yCAAyC;SACxG;IACL,CAAC;IAES,KAAK,CAAC,2BAA2B,CAAC,eAA6B;QACrE,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,MAAM,IAAI,CAAC,oBAAoB,CAAC,eAAe,CAAC,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC;YAC/D,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,+CAA+C,eAAe,CAAC,KAAK,YAAY,sBAAY,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB;gBAC5H,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,OAAO;gBAC/B,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,KAC5B,EAAE,EACF,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CACjC,CAAC;SACL;IACL,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,wBAAwB;QACpC,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;SACtE;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE;YACzB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;SAC/D;IACL,CAAC;IAES,KAAK,CAAC,aAAa,CAAuD,KAAiB,EAAE,GAAG,IAA0B;QAChI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE;YACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;aACvB;SACJ;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,WAAY,CAAC,QAAQ,EAAE,CAAC;SACtC;IACL,CAAC;IAES,yBAAyB,CAAW,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAc,GAAG,KAAK,GACe;QACrC,IAAI,WAAW,IAAI,WAAW,EAAE;YAC5B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBACb,SAAS,OAAO,UAAU,OAAO,yCAAyC;gBAC1E,IAAI,OAAO,0BAA0B,OAAO,6CAA6C;gBACzF,aAAa,OAAO,yBAAyB;aAChD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAEd,6DAA6D;YAC7D,IAAI,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC;SACnC;aAAM,IAAI,WAAW,EAAE;YACpB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBACb,IAAI,OAAO,0BAA0B,OAAO,6CAA6C;gBACzF,2DAA2D,OAAO,SAAS,OAAO,4BAA4B;aACjH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAEd,6DAA6D;YAC7D,IAAI,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC;SACnC;aAAM,IAAI,WAAW,EAAE;YACpB,6DAA6D;YAC7D,IAAI,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC;SACnC;aAAM,IAAI,CAAC,cAAc,EAAE;YACxB,MAAM,IAAI,kBAAa,CAAC,IAAI,OAAO,2CAA2C,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SACrG;IACL,CAAC;IAES,2BAA2B,CAAC,OAAgB;QAClD,OAAO,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;IACpE,CAAC;;AAtsBL,oCAusBC;AAnpBG;;;;WAAgC;QAC5B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAU,CAAC,WAAW,CAAC;QAChE,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAU,CAAC,YAAY,CAAC;QAClE,0DAA0D;QAC1D,yDAAyD;QACzD,4DAA4D;QAC5D,2EAA2E;QAC3E,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACpC,mCAAmC;QACnC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC3C,yBAAyB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC7C,mCAAmC;QACnC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5C,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC1C,mCAAmC;QACnC,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACjD,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAEnC,4BAA4B;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAElC,WAAW;QACX,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC1B;GAAC"}
|
|
1
|
+
{"version":3,"file":"basic-crawler.js","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":";;;;AACA,6DAAoC;AACpC,4CAA8E;AAC9E,gDAAwD;AAexD,wCAgBuB;AAEvB,+CAA2C;AAE3C,0CAA8C;AAC9C,iDAAuC;AAWvC;;;;;;;;GAQG;AACH,MAAM,0BAA0B,GAAG,KAAK,CAAC;AA0KzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,MAAa,YAAY;IAwFrB;;OAEG;IACH,YAAY,UAAwC,EAAE,EAAW,SAAS,oBAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAxFjE;;WAEG;QACH;;;;;WAA2B;QAE3B;;;WAGG;QACH;;;;;WAA0B;QAE1B;;;;WAIG;QACH;;;;;WAA4B;QAE5B;;;WAGG;QACH;;;;;WAA0B;QAE1B;;;;;;WAMG;QACH;;;;;WAAgC;QAEhC;;;WAGG;QACH;;;;mBAA0C,aAAM,CAAC,MAAM,EAAW;WAAC;QAEnE;;;;;WAAmB;QACnB;;;;;WAAmD;QACnD;;;;;WAAuD;QACvD;;;;;WAA+D;QAC/D;;;;;WAA+C;QAC/C;;;;;WAAwC;QACxC;;;;;WAAoC;QACpC;;;;;WAAuC;QACvC;;;;;WAAiD;QACjD;;;;;WAAkC;QAClC;;;;mBAA6B,IAAI,GAAG,EAAmB;WAAC;QACxD;;;;;WAAuD;QACvD;;;;;WAA+B;QAC/B;;;;;WAA+B;QAqC3B,IAAA,YAAE,EAAC,OAAO,EAAE,qBAAqB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;QAEpF,MAAM,EACF,WAAW,EACX,YAAY,EACZ,iBAAiB,GAAG,CAAC,EACrB,mBAAmB,EACnB,qBAAqB,GAAG,EAAE,EAC1B,kBAAkB,GAAG,EAAE,EACvB,cAAc,GAAG,IAAI;QAErB,4BAA4B;QAC5B,cAAc,EACd,cAAc,EACd,oBAAoB;QAEpB,WAAW;QACX,GAAG,GAAG,aAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QAEzD,sCAAsC;QACtC,qBAAqB,EACrB,cAAc,EAEd,wBAAwB,EACxB,yBAAyB,EAEzB,YAAY,EAEZ,2BAA2B,EAC3B,oBAAoB,GACvB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,uBAAuB;YAChC,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,cAAc;YAC3B,WAAW,EAAE,qBAAqB;YAClC,cAAc,EAAE,IAAI,EAAE,iCAAiC;SAC1D,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACtB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;SACrC;QAED,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QAEjC,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,sBAAsB;YAC/B,OAAO,EAAE,6BAA6B;YACtC,WAAW,EAAE,sBAAsB;YACnC,WAAW,EAAE,oBAAoB;YACjC,WAAW,EAAE,2BAA2B;YACxC,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,wBAA4C,CAAC;QAEjD,IAAI,CAAC,wBAAwB,EAAE;YAC3B,IAAI,CAAC,yBAAyB,EAAE;gBAC5B,wBAAwB,GAAG,KAAM,CAAC;aACrC;iBAAM;gBACH,wBAAwB,GAAG,yBAAyB,GAAG,IAAI,CAAC;aAC/D;SACJ;aAAM,IAAI,yBAAyB,EAAE;YAClC,wBAAwB,GAAG,yBAAyB,GAAG,IAAI,CAAC;SAC/D;QAED,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,2BAA2B;YACpC,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,6BAA6B;YAC1C,WAAW,EAAE,wBAAwB;YACrC,WAAW,EAAE,wBAAwB,CAAC,CAAC,CAAC,wBAAwB,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS;SACtF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,CAAC,GAAY,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7D,4CAA4C;QAC5C,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACnI,kGAAkG;QAClG,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,YAAY,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC;SACxE;QAED,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAU,CAAC,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,sBAAsB,EAAE,MAAM,EAAE,CAAC,CAAC;QACtG,IAAI,CAAC,kBAAkB,GAAG;YACtB,GAAG,kBAAkB;YACrB,GAAG;SACN,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,EAAE,CAAC;QAElC,MAAM,gBAAgB,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,2BAA2B,GAAG,gBAAgB,EAAE;YACrD,GAAG,CAAC,OAAO,CAAC,+BAA+B,IAAI,CAAC,2BAA2B,EAAE;kBACvE,+DAA+D,gBAAgB,EAAE,CAAC,CAAC;YAEzF,IAAI,CAAC,2BAA2B,GAAG,gBAAgB,CAAC;SACvD;QAED,IAAI,yBAAyB,GAAG,IAAI,CAAC;QACrC,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,mBAAmB,IAAI,mBAAmB,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAEzG,MAAM,EAAE,kBAAkB,EAAE,GAAG,qBAAqB,CAAC;QAErD,MAAM,uCAAuC,GAAmC;YAC5E,cAAc;YACd,cAAc;YACd,iBAAiB,EAAE,oBAAoB,IAAI,qBAAqB,EAAE,iBAAiB;YACnF,eAAe,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,mBAAmB,EAAE,KAAK,IAAI,EAAE;gBAC5B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,IAAI,yBAAyB,EAAE;wBAC3B,GAAG,CAAC,IAAI,CAAC,mDAAmD;8BACtD,GAAG,mBAAmB,6FAA6F,CAAC,CAAC;wBAC3H,yBAAyB,GAAG,KAAK,CAAC;qBACrC;oBACD,OAAO,KAAK,CAAC;iBAChB;gBAED,OAAO,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACvC,CAAC;YACD,kBAAkB,EAAE,KAAK,IAAI,EAAE;gBAC3B,IAAI,kBAAkB,EAAE,EAAE;oBACtB,GAAG,CAAC,IAAI,CAAC,iEAAiE,mBAAmB,YAAY;0BACnG,yEAAyE;0BACzE,mCAAmC,IAAI,CAAC,oBAAoB,+BAA+B,CAAC,CAAC;oBACnG,OAAO,IAAI,CAAC;iBACf;gBAED,MAAM,UAAU,GAAG,kBAAkB;oBACjC,CAAC,CAAC,MAAM,kBAAkB,EAAE;oBAC5B,CAAC,CAAC,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC;gBAE9C,IAAI,UAAU,EAAE;oBACZ,MAAM,MAAM,GAAG,kBAAkB;wBAC7B,CAAC,CAAC,mFAAmF;wBACrF,CAAC,CAAC,0GAA0G,CAAC;oBACjH,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBACpB;gBAED,OAAO,UAAU,CAAC;YACtB,CAAC;YACD,GAAG;SACN,CAAC;QAEF,IAAI,CAAC,qBAAqB,GAAG,EAAE,GAAG,qBAAqB,EAAE,GAAG,uCAAuC,EAAE,CAAC;QAEtG,wEAAwE;QACxE,IAAI,CAAC,MAAM,CAAC,EAAE,wCAAsB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,MAAM,CAAC,EAAE,sCAAqB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,GAAG,CAAC,QAAgD,EAAE,OAAmC;QAC3F,MAAM,IAAA,2BAAoB,GAAE,CAAC;QAE7B,IAAI,QAAQ,EAAE;YACV,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;SAC7C;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAElC,IAAI;YACA,MAAM,IAAI,CAAC,cAAe,CAAC,GAAG,EAAE,CAAC;SACpC;gBAAS;YACN,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;SACpC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG;YACV,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,gBAAgB;YACnD,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc;YAC/C,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,qBAAqB;YAChD,GAAG,UAAU;SAChB,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;QAElD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,eAAe;QACjB,IAAI,CAAC,YAAY,KAAjB,IAAI,CAAC,YAAY,GAAK,MAAM,mBAAY,CAAC,IAAI,EAAE,EAAC;QAEhD,OAAO,IAAI,CAAC,YAAa,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAwC,eAAe,EAAW;QAC5E,MAAM,GAAG,GAAG,MAAM,oBAAa,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,GAAG,CAAC,iBAAiB,CAAQ,YAAY,CAAC,iBAAiB,EAAE,YAAY,CAAC,CAAC;IACtF,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,QAA+C,EAAE,UAAqC,EAAE;QACtG,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAClE,GAAG,EAAE,YAAE,CAAC,MAAM;YACd,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC,CAAC,CAAC;QACN,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACnD,CAAC,CAAC,CAAC;QAEJ,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAClD,MAAM,aAAa,GAAG,IAAA,qBAAc,EAAC,QAAQ,CAAC,CAAC;QAE/C,MAAM,uCAAuC,GAAG,KAAK,EAAE,gBAA2B,EAAE,EAAE;YAClF,MAAM,eAAe,GAAuB,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;YACrG,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAErD,IAAI,SAAS,CAAC,mBAAmB,CAAC,MAAM,EAAE;gBACtC,MAAM,IAAA,aAAK,EAAC,IAAI,CAAC,CAAC;gBAElB,eAAe,CAAC,IAAI,CAAC,GAAG,MAAM,uCAAuC,CACjE,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,CAC1G,CAAC,CAAC;aACN;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAEnD,uDAAuD;QACvD,MAAM,aAAa,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC;QAElF,mDAAmD;QACnD,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;YACvB,OAAO;gBACH,aAAa;gBACb,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;SACL;QAED,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAqB,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9D,MAAM,MAAM,GAAG,IAAA,aAAK,EAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC1C,MAAM,kBAAkB,GAAuB,EAAE,CAAC;YAElD,KAAK,MAAM,YAAY,IAAI,MAAM,EAAE;gBAC/B,kBAAkB,CAAC,IAAI,CAAC,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC,CAAC;gBAExF,MAAM,IAAA,aAAK,EAAC,IAAI,CAAC,CAAC;aACrB;YAED,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,0GAA0G;QAC1G,IAAI,OAAO,CAAC,2BAA2B,EAAE;YACrC,aAAa,CAAC,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC;SACxC;QAED,OAAO;YACH,aAAa;YACb,2BAA2B,EAAE,OAAO;SACvC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,KAAK;QACjB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE;YAC9B,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;SAC5B;QAED,wEAAwE;QACxE,+FAA+F;QAC/F,oCAAoC;QACpC,IAAI,CAAC,cAAc,GAAG,IAAI,qBAAc,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAElF,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,WAAW,GAAG,MAAM,kBAAW,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACnE,gEAAgE;YAChE,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACxC;QAED,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;IAC1C,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QACvD,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,OAAgB,EAAE,UAAkB;QACjE,MAAM,SAAS,GAAG,OAAO,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAEjE,IAAI,SAAS,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,eAAe,CAAC,CAAC;SAC5E;IACL,CAAC;IAES,KAAK,CAAC,iBAAiB;QAC7B,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,+CAA+C;YAC/C,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,0BAA0B,CAAC;iBACtD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAC,EAAE;oBACtD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2DAA2D;0BACpE,uFAAuF,CAAC,CAAC;iBAClG;qBAAM;oBACH,MAAM,GAAG,CAAC;iBACb;YACL,CAAC,CAAC,CAAC;SACV;QAED,MAAM,yBAAyB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1C,IAAI,IAAI,CAAC,WAAW,EAAE;gBAClB,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;oBAAE,OAAO;gBAChD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE;qBAChC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACX,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE;wBAC/C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0FAA0F;8BACnG,2GAA2G;8BAC3G,8FAA8F,CAAC,CAAC;qBACzG;yBAAM;wBACH,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,gDAAgD;8BAClE,iDAAiD,CAAC,CAAC;qBAC5D;gBACL,CAAC,CAAC,CAAC;aACV;QACL,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,OAAO,CAAC,GAAG,CAAC;YACd,yBAAyB;YACzB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE;SAC5B,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,YAAa,CAAC,gBAAgB,EAAE,CAAC;QACpE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,OAAO,CAAC;QACvC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI;YACA,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACpE;QAAC,OAAO,GAAG,EAAE;YACV,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yGAAyG,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YACvI,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC;SACf;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC;IAChD,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,IAAI,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAErF,IAAI,OAAmC,CAAC;QACxC,IAAI,OAA4B,CAAC;QAEjC,MAAM,IAAI,CAAC,gBAAgB,CACvB,KAAK,IAAI,EAAE;YACP,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,CAAC,EACD,IAAI,CAAC,qBAAqB,EAC1B,yCAAyC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CACvF,CAAC;QAEF,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,gBAAgB,CACvB,KAAK,IAAI,EAAE;gBACP,OAAO,GAAG,MAAM,IAAI,CAAC,WAAY,CAAC,UAAU,EAAE,CAAC;YACnD,CAAC,EACD,IAAI,CAAC,qBAAqB,EAC1B,oCAAoC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAClF,CAAC;SACL;QAED,IAAA,mBAAS,GAAE,CAAC;QAEZ,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,gEAAgE;QAChE,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,MAAM,YAAY,GAAG,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAElC,0BAA0B;QAC1B,mBAAmB;QACnB,uFAAuF;QACvF,yEAAyE;QACzE,MAAM,eAAe,GAAY;YAC7B,EAAE,EAAE,IAAA,gCAAoB,EAAC,EAAE,CAAC;YAC5B,OAAO,EAAE,IAAI;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,OAAO;YACP,OAAO;YACP,YAAY,EAAE,KAAK,EAAE,cAA+C,EAAE,EAAE;gBACpE,OAAO,IAAA,mBAAY,EAAC;oBAChB,GAAG,cAAc;oBACjB,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;iBAC7C,CAAC,CAAC;YACP,CAAC;YACD,WAAW,EAAE,KAAK,EAAE,eAAgC,EAAE,EAAE;gBACpD,OAAO,IAAA,0BAAW,EAAC;oBACf,GAAG,EAAE,OAAQ,CAAC,GAAG;oBACjB,MAAM,EAAE,OAAQ,CAAC,MAAM;oBACvB,IAAI,EAAE,OAAQ,CAAC,OAAO;oBACtB,OAAO,EAAE,OAAQ,CAAC,OAAO;oBACzB,QAAQ,EAAE,eAAe,CAAC,SAAS,EAAE,GAAG;oBACxC,YAAY,EAAE,OAAO;oBACrB,YAAY,EAAE,MAAM;oBACpB,GAAG,eAAe;oBAClB,KAAK,EAAE;wBACH,KAAK,EAAE,CAAC;wBACR,GAAG,eAAe,EAAE,KAAK;qBAC5B;oBACD,SAAS,EAAE;wBACP,eAAe,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAQ,CAAC,eAAe,CAAC,GAAG,CAAC;wBAC/D,SAAS,EAAE,CAAC,SAAiB,EAAE,GAAW,EAAE,EAAE,CAAC,OAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC;wBACjF,GAAG,eAAe,EAAE,SAAS;qBAChC;iBACyB,CAAC,CAAC;YACpC,CAAC;SACJ,CAAC;QAEF,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;QAE/D,IAAI;YACA,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC,EAC9C,IAAI,CAAC,2BAA2B,EAChC,kCAAkC,IAAI,CAAC,2BAA2B,GAAG,IAAI,aAAa,OAAO,CAAC,EAAE,IAAI,CACvG,CAAC;YAEF,MAAM,IAAI,CAAC,gBAAgB,CACvB,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAQ,CAAC,EACzC,IAAI,CAAC,qBAAqB,EAC1B,mBAAmB,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAC3H,CAAC;YAEF,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,mDAAmD;YACnD,OAAO,EAAE,QAAQ,EAAE,CAAC;SACvB;QAAC,OAAO,GAAG,EAAE;YACV,IAAI;gBACA,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAY,EAAE,eAAe,EAAE,MAAM,CAAC,EAC9E,IAAI,CAAC,qBAAqB,EAC1B,+BAA+B,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,qBAAqB,IAAI,CAAC,qBAAqB,GAAG,GAAG,WAAW,CAC5H,CAAC;aACL;YAAC,OAAO,cAAmB,EAAE;gBAC1B,IAAI,CAAC,cAAc,CAAC,wBAAwB,EAAE;oBAC1C,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB;wBAC9C,CAAC,CAAC,mGAAmG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/G,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,cAAuB,EAAE,2DAA2D;0BACjG,8GAA8G,aAAa,EAAE,CAAC,CAAC;iBACxI;gBACD,MAAM,cAAc,CAAC;aACxB;SACJ;gBAAS;YACN,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;SACpD;IACL,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,gBAAgB,CAAC,OAA+B,EAAE,OAAe,EAAE,KAAqB,EAAE,UAAU,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC;QACjI,IAAI;YACA,MAAM,IAAA,6BAAmB,EAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;SACtD;QAAC,OAAO,CAAC,EAAE;YACR,IAAI,OAAO,IAAI,UAAU,EAAE,EAAE,0CAA0C;gBACnE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAI,CAAW,CAAC,OAAO,cAAc,OAAO,IAAI,UAAU,GAAG,CAAC,CAAC;gBAChF,OAAO,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;aAClF;YAED,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,sDAAsD;QACtD,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxF,+EAA+E;QAC/E,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QACrC,6EAA6E;QAC7E,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC5E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,0BAA0B;QACtC,MAAM,CACF,qBAAqB,EACrB,sBAAsB,EACzB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClB,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;YACvD,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI;SAC5D,CAAC,CAAC;QACH,6DAA6D;QAC7D,OAAO,qBAAqB,IAAI,sBAAsB,CAAC;IAC3D,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,4BAA4B,CACxC,KAAY,EACZ,eAAwB,EACxB,MAAkC;QAElC,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QACpC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,KAAK,YAAY,oBAAa,EAAE;YAChC,MAAM,KAAK,CAAC;SACf;QAED,MAAM,kBAAkB,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC,KAAK,YAAY,wBAAiB,CAAC,CAAC;QACpI,IAAI,kBAAkB,EAAE;YACpB,OAAO,CAAC,UAAU,EAAE,CAAC;YAErB,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC;YAEnF,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;YACxC,uGAAuG;YACvG,yGAAyG;YACzG,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YACtE,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,wDAAwD,OAAO,EAAE,EACjE,EAAE,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,CAC1B,CAAC;YAEF,MAAM,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;SACxC;aAAM;YACH,sDAAsD;YACtD,wEAAwE;YACxE,+CAA+C;YAC/C,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC5B,MAAM,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC;YAEpD,MAAM,IAAI,CAAC,2BAA2B,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,yCAAyC;SAC5G;IACL,CAAC;IAES,KAAK,CAAC,oBAAoB,CAAI,EAAiB;QACrD,IAAI;YACA,OAAO,MAAM,EAAE,EAAO,CAAC;SAC1B;QAAC,OAAO,CAAM,EAAE;YACb,MAAM,CAAC,cAAc,CAAC,CAAC,EAAE,0BAA0B,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAES,KAAK,CAAC,2BAA2B,CAAC,eAAwB,EAAE,KAAY;QAC9E,IAAI,IAAI,CAAC,oBAAoB,EAAE;YAC3B,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3F,OAAO;SACV;QAED,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC;QAC/D,MAAM,OAAO,GAAG,KAAK,YAAY,sBAAY,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;QAChH,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,+CAA+C,OAAO,EAAE,EACxD,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CACjC,CAAC;IACN,CAAC;IAED;;;;;;;OAOG;IACO,KAAK,CAAC,wBAAwB;QACpC,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;SACtE;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE;YACzB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;SAC/D;IACL,CAAC;IAES,KAAK,CAAC,aAAa,CAAuD,KAAiB,EAAE,GAAG,IAA0B;QAChI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE;YACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACtB,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;aACvB;SACJ;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAElE,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,WAAY,CAAC,QAAQ,EAAE,CAAC;SACtC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;SAC7B;IACL,CAAC;IAES,yBAAyB,CAAW,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAc,GAAG,KAAK,GACe;QACrC,IAAI,WAAW,IAAI,WAAW,EAAE;YAC5B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBACb,SAAS,OAAO,UAAU,OAAO,yCAAyC;gBAC1E,IAAI,OAAO,0BAA0B,OAAO,6CAA6C;gBACzF,aAAa,OAAO,yBAAyB;aAChD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAEd,6DAA6D;YAC7D,IAAI,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC;SACnC;aAAM,IAAI,WAAW,EAAE;YACpB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBACb,IAAI,OAAO,0BAA0B,OAAO,6CAA6C;gBACzF,2DAA2D,OAAO,SAAS,OAAO,4BAA4B;aACjH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAEd,6DAA6D;YAC7D,IAAI,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC;SACnC;aAAM,IAAI,WAAW,EAAE;YACpB,6DAA6D;YAC7D,IAAI,CAAC,WAAW,CAAC,GAAG,WAAW,CAAC;SACnC;aAAM,IAAI,CAAC,cAAc,EAAE;YACxB,MAAM,IAAI,kBAAa,CAAC,IAAI,OAAO,2CAA2C,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SACrG;IACL,CAAC;IAES,2BAA2B,CAAC,OAAgB;QAClD,OAAO,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;IACpE,CAAC;;AApwBL,oCAqwBC;AApwBG;;;;WAA4C,eAAe;GAAC;AAwD5D;;;;WAAgC;QAC5B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAU,CAAC,WAAW,CAAC;QAChE,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAU,CAAC,YAAY,CAAC;QAClE,0DAA0D;QAC1D,yDAAyD;QACzD,4DAA4D;QAC5D,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACpC,mCAAmC;QACnC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC3C,yBAAyB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC7C,mCAAmC;QACnC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5C,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAClC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC1C,mCAAmC;QACnC,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACjD,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACvC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAEnC,4BAA4B;QAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAClC,oBAAoB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAEzF,WAAW;QACX,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC1B;GAAC;AA6tBN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,iBAAiB;IAC7B,OAAO,aAAM,CAAC,MAAM,EAAW,CAAC;AACpC,CAAC;AAFD,8CAEC"}
|
package/internals/constants.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Additional number of seconds used in CheerioCrawler and BrowserCrawler to set a reasonable
|
|
3
|
-
*
|
|
2
|
+
* Additional number of seconds used in {@link CheerioCrawler} and {@link BrowserCrawler} to set a reasonable
|
|
3
|
+
* {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`} for {@link BasicCrawler}
|
|
4
|
+
* that would not impare functionality (not timeout before crawlers).
|
|
4
5
|
*/
|
|
5
6
|
export declare const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = 10;
|
|
6
7
|
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/internals/constants.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/internals/constants.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,eAAO,MAAM,iCAAiC,KAAK,CAAC"}
|
package/internals/constants.js
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = void 0;
|
|
4
4
|
/**
|
|
5
|
-
* Additional number of seconds used in CheerioCrawler and BrowserCrawler to set a reasonable
|
|
6
|
-
*
|
|
5
|
+
* Additional number of seconds used in {@link CheerioCrawler} and {@link BrowserCrawler} to set a reasonable
|
|
6
|
+
* {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`} for {@link BasicCrawler}
|
|
7
|
+
* that would not impare functionality (not timeout before crawlers).
|
|
7
8
|
*/
|
|
8
9
|
exports.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = 10;
|
|
9
10
|
//# sourceMappingURL=constants.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/internals/constants.ts"],"names":[],"mappings":";;;AAAA
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/internals/constants.ts"],"names":[],"mappings":";;;AAAA;;;;GAIG;AACU,QAAA,iCAAiC,GAAG,EAAE,CAAC"}
|