@crawlee/basic 3.0.0-beta.9 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,168 +1,151 @@
1
- import { Log } from '@apify/log';
2
- import { AutoscaledPool, AutoscaledPoolOptions, EnqueueLinksOptions, CrawlerHandleFailedRequestInput, ProxyInfo, QueueOperationInfo, Request, RequestList, RequestQueue, Session, SessionPool, SessionPoolOptions, Statistics, type CrawlingContext, RequestOptions, RequestQueueOperationOptions, Configuration, EventManager, FinalStatistics } from '@crawlee/core';
3
- import { Response as GotResponse } from 'got-scraping';
4
- import { ProcessedRequest } from '@crawlee/types';
5
- import { Awaitable } from '@crawlee/utils';
6
- export interface BasicCrawlerCrawlingContext extends CrawlingContext {
7
- crawler: BasicCrawler;
8
- enqueueLinks: (options: BasicCrawlerEnqueueLinksOptions) => Promise<QueueOperationInfo[]>;
9
- sendRequest: (request?: Request) => Promise<GotResponse<string>>;
10
- }
11
- export interface BasicCrawlerHandleFailedRequestInput extends CrawlerHandleFailedRequestInput {
1
+ import type { Log } from '@apify/log';
2
+ import type { AutoscaledPoolOptions, EnqueueLinksOptions, EventManager, FinalStatistics, ProxyInfo, Request, RequestList, RequestOptions, RequestQueueOperationOptions, RouterHandler, Session, SessionPoolOptions } from '@crawlee/core';
3
+ import { AutoscaledPool, Configuration, type CrawlingContext, RequestQueue, SessionPool, Statistics } from '@crawlee/core';
4
+ import type { GotOptionsInit, Response as GotResponse } from 'got-scraping';
5
+ import type { ProcessedRequest, Dictionary, Awaitable, BatchAddRequestsResult } from '@crawlee/types';
6
+ export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
12
7
  crawler: BasicCrawler;
8
+ enqueueLinks: (options: BasicCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
9
+ sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;
13
10
  }
14
11
  /** @internal */
15
12
  export declare type BasicCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'requestQueue'>;
16
- export declare type RequestHandler<Context extends CrawlingContext = BasicCrawlerCrawlingContext> = (inputs: Context) => Awaitable<void>;
17
- export declare type FailedRequestHandler<Inputs extends CrawlerHandleFailedRequestInput = BasicCrawlerHandleFailedRequestInput> = (inputs: Inputs) => Awaitable<void>;
18
- export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlerCrawlingContext, ErrorContext extends CrawlerHandleFailedRequestInput = BasicCrawlerHandleFailedRequestInput> {
13
+ export declare type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context) => Awaitable<void>;
14
+ export declare type FailedRequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;
15
+ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlingContext> {
19
16
  /**
20
17
  * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
21
18
  *
22
- * The function receives the following object as an argument:
23
- * ```
24
- * {
25
- * request: Request,
26
- * session: Session,
27
- * crawler: BasicCrawler,
28
- * }
29
- * ```
30
- * where the {@link Request} instance represents the URL to crawl.
19
+ * The function receives the {@link BasicCrawlingContext} as an argument,
20
+ * where the {@link BasicCrawlingContext.request|`request`} represents the URL to crawl.
31
21
  *
32
22
  * The function must return a promise, which is then awaited by the crawler.
33
23
  *
34
24
  * If the function throws an exception, the crawler will try to re-crawl the
35
- * request later, up to `option.maxRequestRetries` times.
25
+ * request later, up to the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
36
26
  * If all the retries fail, the crawler calls the function
37
- * provided to the `handleFailedRequestFunction` parameter.
38
- * To make this work, you should **always**
39
- * let your function throw exceptions rather than catch them.
27
+ * provided to the {@link BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
28
+ * To make this work, we should **always**
29
+ * let our function throw exceptions rather than catch them.
40
30
  * The exceptions are logged to the request using the
41
- * {@link Request.pushErrorMessage} function.
31
+ * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
42
32
  */
43
- requestHandler: RequestHandler<Context>;
33
+ requestHandler?: RequestHandler<Context>;
44
34
  /**
45
35
  * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
46
36
  *
47
- * The function receives the following object as an argument:
48
- * ```
49
- * {
50
- * request: Request,
51
- * session: Session,
52
- * crawler: BasicCrawler,
53
- * }
54
- * ```
55
- * where the {@link Request} instance represents the URL to crawl.
37
+ * The function receives the {@link BasicCrawlingContext} as an argument,
38
+ * where the {@link BasicCrawlingContext.request|`request`} represents the URL to crawl.
56
39
  *
57
40
  * The function must return a promise, which is then awaited by the crawler.
58
41
  *
59
42
  * If the function throws an exception, the crawler will try to re-crawl the
60
- * request later, up to `option.maxRequestRetries` times.
43
+ * request later, up to the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
61
44
  * If all the retries fail, the crawler calls the function
62
- * provided to the `handleFailedRequestFunction` parameter.
63
- * To make this work, you should **always**
64
- * let your function throw exceptions rather than catch them.
45
+ * provided to the {@link BasicCrawlerOptions.failedRequestHandler|`failedRequestHandler`} parameter.
46
+ * To make this work, we should **always**
47
+ * let our function throw exceptions rather than catch them.
65
48
  * The exceptions are logged to the request using the
66
- * {@link Request.pushErrorMessage} function.
49
+ * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
67
50
  *
68
51
  * @deprecated `handleRequestFunction` has been renamed to `requestHandler` and will be removed in a future version.
69
52
  */
70
53
  handleRequestFunction?: RequestHandler<Context>;
71
54
  /**
72
55
  * Static list of URLs to be processed.
73
- * Either `requestList` or `requestQueue` option must be provided (or both).
56
+ * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
57
+ * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
58
+ * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
74
59
  */
75
60
  requestList?: RequestList;
76
61
  /**
77
62
  * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
78
- * Either `requestList` or `requestQueue` option must be provided (or both).
63
+ * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
64
+ * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
65
+ * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
79
66
  */
80
67
  requestQueue?: RequestQueue;
81
68
  /**
82
- * Timeout in which the function passed as `requestHandler` needs to finish, in seconds.
69
+ * Timeout in which the function passed as {@link BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
83
70
  * @default 60
84
71
  */
85
72
  requestHandlerTimeoutSecs?: number;
86
73
  /**
87
- * Timeout in which the function passed as `requestHandler` needs to finish, in seconds.
74
+ * Timeout in which the function passed as {@link BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
88
75
  * @default 60
89
76
  * @deprecated `handleRequestTimeoutSecs` has been renamed to `requestHandlerTimeoutSecs` and will be removed in a future version.
90
77
  */
91
78
  handleRequestTimeoutSecs?: number;
92
79
  /**
93
- * A function to handle requests that failed more than `option.maxRequestRetries` times.
80
+ * User-provided function that allows modifying the request object before it gets retried by the crawler.
81
+ * It's executed before each retry for the requests that failed less than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
94
82
  *
95
- * The function receives the following object as an argument:
96
- * ```
97
- * {
98
- * request: Request,
99
- * error: Error,
100
- * session: Session,
101
- * crawler: BasicCrawler,
102
- * }
103
- * ```
104
- * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
83
+ * The function receives the {@link BasicCrawlingContext} as the first argument,
84
+ * where the {@link BasicCrawlingContext.request|`request`} corresponds to the request to be retried.
85
+ * Second argument is the `Error` instance that
105
86
  * represents the last error thrown during processing of the request.
106
- *
107
- * See
108
- * [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/basic_crawler.js#L11)
109
- * for the default implementation of this function.
110
87
  */
111
- failedRequestHandler?: FailedRequestHandler<ErrorContext>;
88
+ errorHandler?: FailedRequestHandler<Context>;
112
89
  /**
113
- * A function to handle requests that failed more than `option.maxRequestRetries` times.
90
+ * A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
114
91
  *
115
- * The function receives the following object as an argument:
116
- * ```
117
- * {
118
- * request: Request,
119
- * error: Error,
120
- * session: Session,
121
- * crawler: BasicCrawler,
122
- * }
123
- * ```
124
- * where the {@link Request} instance corresponds to the failed request, and the `Error` instance
92
+ * The function receives the {@link BasicCrawlingContext} as the first argument,
93
+ * where the {@link BasicCrawlingContext.request|`request`} corresponds to the failed request.
94
+ * Second argument is the `Error` instance that
125
95
  * represents the last error thrown during processing of the request.
96
+ */
97
+ failedRequestHandler?: FailedRequestHandler<Context>;
98
+ /**
99
+ * A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
126
100
  *
127
- * See
128
- * [source code](https://github.com/apify/apify-js/blob/master/src/crawlers/basic_crawler.js#L11)
129
- * for the default implementation of this function.
101
+ * The function receives the {@link BasicCrawlingContext} as the first argument,
102
+ * where the {@link BasicCrawlingContext.request|`request`} corresponds to the failed request.
103
+ * Second argument is the `Error` instance that
104
+ * represents the last error thrown during processing of the request.
130
105
  *
131
106
  * @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
132
107
  */
133
- handleFailedRequestFunction?: FailedRequestHandler<ErrorContext>;
108
+ handleFailedRequestFunction?: FailedRequestHandler<Context>;
134
109
  /**
135
- * Indicates how many times the request is retried if {@link requestHandler} or {@link handlePageFunction} fails.
110
+ * Indicates how many times the request is retried if {@link BasicCrawlerOptions.requestHandler|`requestHandler`} fails.
136
111
  * @default 3
137
112
  */
138
113
  maxRequestRetries?: number;
139
114
  /**
140
115
  * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
141
- * Always set this value in order to prevent infinite loops in misconfigured crawlers.
142
- * Note that in cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
116
+ * This value should always be set in order to prevent infinite loops in misconfigured crawlers.
117
+ * > *NOTE:* In cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
143
118
  */
144
119
  maxRequestsPerCrawl?: number;
145
120
  /**
146
121
  * Custom options passed to the underlying {@link AutoscaledPool} constructor.
147
- * Note that the `runTaskFunction` and `isTaskReadyFunction` options
122
+ * > *NOTE:* The {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
123
+ * and {@link AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`} options
148
124
  * are provided by the crawler and cannot be overridden.
149
- * However, you can provide a custom implementation of `isFinishedFunction`.
125
+ * However, we can provide a custom implementation of {@link AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}.
150
126
  */
151
127
  autoscaledPoolOptions?: AutoscaledPoolOptions;
152
128
  /**
153
- * Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
154
- *
155
- * *WARNING:* If you set this value too high with respect to the available system memory and CPU, your crawler will run extremely slow or crash.
156
- * If you're not sure, just keep the default value and the concurrency will scale up automatically.
129
+ * Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
130
+ * AutoscaledPool {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} option.
131
+ * > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
132
+ * If not sure, it's better to keep the default value and the concurrency will scale up automatically.
157
133
  */
158
134
  minConcurrency?: number;
159
135
  /**
160
- * Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding {@link AutoscaledPool} option.
136
+ * Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
137
+ * AutoscaledPool {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} option.
161
138
  */
162
139
  maxConcurrency?: number;
163
140
  /**
164
- * Basic crawler will initialize the {@link SessionPool} with the corresponding `sessionPoolOptions`.
165
- * The session instance will be than available in the `requestHandler`.
141
+ * The maximum number of requests per minute the crawler should run.
142
+ * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
143
+ * Shortcut for the AutoscaledPool {@link AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`} option.
144
+ */
145
+ maxRequestsPerMinute?: number;
146
+ /**
147
+ * Basic crawler will initialize the {@link SessionPool} with the corresponding {@link SessionPoolOptions|`sessionPoolOptions`}.
148
+ * The session instance will be than available in the {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
166
149
  */
167
150
  useSessionPool?: boolean;
168
151
  /**
@@ -179,100 +162,103 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
179
162
  *
180
163
  * `BasicCrawler` is a low-level tool that requires the user to implement the page
181
164
  * download and data extraction functionality themselves.
182
- * If you want a crawler that already facilitates this functionality,
183
- * please consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
165
+ * If we want a crawler that already facilitates this functionality,
166
+ * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
184
167
  *
185
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler}
168
+ * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
186
169
  * for each {@link Request} object, which represents a single URL to crawl.
187
- * The {@link Request} objects are fed from the {@link RequestList} or the {@link RequestQueue}
188
- * instances provided by the {@link BasicCrawlerOptions.requestList} or {@link BasicCrawlerOptions.requestQueue}
189
- * constructor options, respectively.
170
+ * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
171
+ * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
172
+ * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
173
+ * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
174
+ * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
190
175
  *
191
- * If both {@link BasicCrawlerOptions.requestList} and {@link BasicCrawlerOptions.requestQueue} options are used,
176
+ * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
192
177
  * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
193
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
178
+ * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
194
179
  *
195
180
  * The crawler finishes if there are no more {@link Request} objects to crawl.
196
181
  *
197
182
  * New requests are only dispatched when there is enough free CPU and memory available,
198
183
  * using the functionality provided by the {@link AutoscaledPool} class.
199
- * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
200
- * parameter of the `BasicCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
201
- * {@link AutoscaledPool} options are available directly in the `BasicCrawler` constructor.
184
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
185
+ * parameter of the `BasicCrawler` constructor.
186
+ * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
187
+ * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
188
+ * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
202
189
  *
203
190
  * **Example usage:**
204
191
  *
205
192
  * ```javascript
206
- * const { gotScraping } = require('got-scraping');
193
+ * import { BasicCrawler, Dataset } from 'crawlee';
207
194
  *
208
- * // Prepare a list of URLs to crawl
209
- * const requestList = new RequestList({
210
- * sources: [
211
- * { url: 'http://www.example.com/page-1' },
212
- * { url: 'http://www.example.com/page-2' },
213
- * ],
214
- * });
215
- * await requestList.initialize();
216
- *
217
- * // Crawl the URLs
195
+ * // Create a crawler instance
218
196
  * const crawler = new BasicCrawler({
219
- * requestList,
220
- * async requestHandler({ request }) {
197
+ * async requestHandler({ request, sendRequest }) {
221
198
  * // 'request' contains an instance of the Request class
222
199
  * // Here we simply fetch the HTML of the page and store it to a dataset
223
- * const { body } = await gotScraping({
200
+ * const { body } = await sendRequest({
224
201
  * url: request.url,
225
202
  * method: request.method,
226
203
  * body: request.payload,
227
204
  * headers: request.headers,
228
205
  * });
229
206
  *
230
- * await Actor.pushData({
207
+ * await Dataset.pushData({
231
208
  * url: request.url,
232
209
  * html: body,
233
210
  * })
234
211
  * },
235
212
  * });
236
213
  *
237
- * await crawler.run();
214
+ * // Enqueue the initial requests and run the crawler
215
+ * await crawler.run([
216
+ * 'http://www.example.com/page-1',
217
+ * 'http://www.example.com/page-2',
218
+ * ]);
238
219
  * ```
239
220
  * @category Crawlers
240
221
  */
241
- export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlerCrawlingContext, ErrorContext extends CrawlerHandleFailedRequestInput = BasicCrawlerHandleFailedRequestInput> {
222
+ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext> {
242
223
  readonly config: Configuration;
224
+ private static readonly CRAWLEE_STATE_KEY;
243
225
  /**
244
- * Static list of URLs to be processed.
226
+ * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
245
227
  */
246
228
  readonly stats: Statistics;
247
229
  /**
248
- * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
249
- * Either `requestList` or `requestQueue` option must be provided (or both).
230
+ * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
250
231
  * Only available if used by the crawler.
251
232
  */
252
233
  requestList?: RequestList;
253
234
  /**
254
235
  * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
255
- * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
256
- * Either `requestList` or `requestQueue` option must be provided (or both).
236
+ * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
257
237
  * Only available if used by the crawler.
258
238
  */
259
239
  requestQueue?: RequestQueue;
260
240
  /**
261
- * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
241
+ * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
262
242
  * Only available if used by the crawler.
263
243
  */
264
244
  sessionPool?: SessionPool;
265
245
  /**
266
246
  * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
267
- * Note that this property is only initialized after calling the {@link BasicCrawler.run} function.
268
- * You can use it to change the concurrency settings on the fly,
269
- * to pause the crawler by calling {@link AutoscaledPool.pause}
270
- * or to abort it by calling {@link AutoscaledPool.abort}.
247
+ * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
248
+ * We can use it to change the concurrency settings on the fly,
249
+ * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
250
+ * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
271
251
  */
272
252
  autoscaledPool?: AutoscaledPool;
253
+ /**
254
+ * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
255
+ * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
256
+ */
257
+ readonly router: RouterHandler<Context>;
273
258
  protected log: Log;
274
259
  protected requestHandler: RequestHandler<Context>;
275
- protected failedRequestHandler?: FailedRequestHandler<ErrorContext>;
260
+ protected errorHandler?: FailedRequestHandler<Context>;
261
+ protected failedRequestHandler?: FailedRequestHandler<Context>;
276
262
  protected requestHandlerTimeoutMillis: number;
277
263
  protected internalTimeoutMillis: number;
278
264
  protected maxRequestRetries: number;
@@ -282,6 +268,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
282
268
  protected crawlingContexts: Map<string, Context>;
283
269
  protected autoscaledPoolOptions: AutoscaledPoolOptions;
284
270
  protected events: EventManager;
271
+ private _closeEvents?;
285
272
  protected static optionsShape: {
286
273
  requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
287
274
  requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
@@ -289,6 +276,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
289
276
  handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
290
277
  requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
291
278
  handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
279
+ errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
292
280
  failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
293
281
  handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
294
282
  maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -298,17 +286,24 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
298
286
  useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
299
287
  minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
300
288
  maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
289
+ maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
301
290
  log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
302
291
  };
303
292
  /**
304
293
  * All `BasicCrawler` parameters are passed via an options object.
305
294
  */
306
- constructor(options: BasicCrawlerOptions<Context, ErrorContext>, config?: Configuration);
295
+ constructor(options?: BasicCrawlerOptions<Context>, config?: Configuration);
307
296
  /**
308
297
  * Runs the crawler. Returns a promise that gets resolved once all the requests are processed.
298
+ * We can use the `requests` parameter to enqueue the initial requests - it is a shortcut for
299
+ * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before the {@link BasicCrawler.run|`crawler.run()`}.
300
+ *
301
+ * @param [requests] The requests to add
302
+ * @param [options] Options for the request queue
309
303
  */
310
- run(): Promise<FinalStatistics>;
304
+ run(requests?: (string | Request | RequestOptions)[], options?: CrawlerAddRequestsOptions): Promise<FinalStatistics>;
311
305
  getRequestQueue(): Promise<RequestQueue>;
306
+ useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
312
307
  /**
313
308
  * Adds requests to be processed by the crawler
314
309
  * @param requests The requests to add
@@ -326,7 +321,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
326
321
  * Fetches request from either RequestList or RequestQueue. If request comes from a RequestList
327
322
  * and RequestQueue is present then enqueues it to the queue first.
328
323
  */
329
- protected _fetchNextRequest(): Promise<Request | null>;
324
+ protected _fetchNextRequest(): Promise<Request<Dictionary<any>> | null>;
330
325
  /**
331
326
  * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
332
327
  * then retries them in a case of an error, etc.
@@ -349,7 +344,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawler
349
344
  * Handles errors thrown by user provided requestHandler()
350
345
  */
351
346
  protected _requestFunctionErrorHandler(error: Error, crawlingContext: Context, source: RequestList | RequestQueue): Promise<void>;
352
- protected _handleFailedRequestHandler(crawlingContext: ErrorContext): Promise<void>;
347
+ protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
348
+ protected _handleFailedRequestHandler(crawlingContext: Context, error: Error): Promise<void>;
353
349
  /**
354
350
  * Updates handledRequestsCount from possibly stored counts,
355
351
  * usually after worker migration. Since one of the stores
@@ -376,8 +372,7 @@ export interface CreateContextOptions {
376
372
  export interface CrawlerAddRequestsOptions extends RequestQueueOperationOptions {
377
373
  /**
378
374
  * Whether to wait for all the provided requests to be added, instead of waiting just for the initial batch of up to 1000.
379
- *
380
- * By default, this is set to `false`.
375
+ @default false
381
376
  */
382
377
  waitForAllRequestsToBeAdded?: boolean;
383
378
  }
@@ -386,13 +381,16 @@ export interface CrawlerAddRequestsResult {
386
381
  /**
387
382
  * A promise which will resolve with the rest of the requests that were added to the queue.
388
383
  *
389
- * @example
384
+ * Alternatively, we can set {@link CrawlerAddRequestsOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} to `true`
385
+ * in the {@link BasicCrawler.addRequests|`crawler.addRequests()`} options.
386
+ *
387
+ * **Example:**
388
+ *
390
389
  * ```ts
391
390
  * // Assuming `requests` is a list of requests.
392
391
  * const result = await crawler.addRequests(requests);
393
392
  *
394
- * // If you want to wait for the rest of the requests to be added with a condition, you can do so:
395
- * // Alternatively, consider setting `waitForAllRequestsToBeAdded` to `true` in the options for `addRequests`.
393
+ * // If we want to wait for the rest of the requests to be added to the queue:
396
394
  * await result.waitForAllRequestsToBeAdded;
397
395
  * ```
398
396
  */
@@ -406,5 +404,30 @@ interface HandlePropertyNameChangeData<New, Old> {
406
404
  propertyKey: string;
407
405
  allowUndefined?: boolean;
408
406
  }
407
+ /**
408
+ * Creates new {@link Router} instance that works based on request labels.
409
+ * This instance can then serve as a {@link BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@link BasicCrawler}.
410
+ * Defaults to the {@link BasicCrawlingContext}.
411
+ *
412
+ * > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
413
+ *
414
+ * ```ts
415
+ * import { BasicCrawler, createBasicRouter } from 'crawlee';
416
+ *
417
+ * const router = createBasicRouter();
418
+ * router.addHandler('label-a', async (ctx) => {
419
+ * ctx.log.info('...');
420
+ * });
421
+ * router.addDefaultHandler(async (ctx) => {
422
+ * ctx.log.info('...');
423
+ * });
424
+ *
425
+ * const crawler = new BasicCrawler({
426
+ * requestHandler: router,
427
+ * });
428
+ * await crawler.run();
429
+ * ```
430
+ */
431
+ export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext>(): RouterHandler<Context>;
409
432
  export {};
410
433
  //# sourceMappingURL=basic-crawler.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"basic-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":"AAAA,OAAmB,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAG7C,OAAO,EACH,cAAc,EACd,qBAAqB,EACrB,mBAAmB,EACnB,+BAA+B,EAE/B,SAAS,EACT,kBAAkB,EAClB,OAAO,EACP,WAAW,EACX,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,UAAU,EAEV,KAAK,eAAe,EACpB,cAAc,EACd,4BAA4B,EAE5B,aAAa,EACb,YAAY,EAEZ,eAAe,EAClB,MAAM,eAAe,CAAC;AACvB,OAAO,EAA0C,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAC/F,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAgB,MAAM,gBAAgB,CAAC;AAGzD,MAAM,WAAW,2BAA4B,SAAQ,eAAe;IAChE,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,CAAC,OAAO,EAAE,+BAA+B,KAAK,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAC1F,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,oCAAqC,SAAQ,+BAA+B;IACzF,OAAO,EAAE,YAAY,CAAC;CACzB;AAED,gBAAgB;AAChB,oBAAY,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAA;AAavF,oBAAY,cAAc,CAAC,OAAO,SAAS,eAAe,GAAG,2BAA2B,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEjI,oBAAY,oBAAoB,CAAC,MAAM,SAAS,+BAA+B,GAAG,oCAAoC,IAAI,CAAC,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE9J,MAAM,WAAW,mBAAmB,CAChC,OAAO,SAAS,eAAe,GAAG,2BAA2B,EAC7D,YAAY,SAAS,+BAA+B,GAAG,oCAAoC;IAE3F;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,cAAc,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEhD;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;;;;;;;;;;;;;OAkBG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAE1D;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,2BAA2B,CAAC,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAEjE;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAE9C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiEG;AACH,qBAAa,YAAY,CACrB,OAAO,SAAS,eAAe,GAAG,2BAA2B,EAC7D,YAAY,SAAS,+BAA+B,GAAG,oCAAoC;IAmF1B,QAAQ,CAAC,MAAM;IAjFhF;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAE3B;;;;OAIG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,cAAc,EAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnD,SAAS,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IACpE,SAAS,CAAC,2BAA2B,EAAG,MAAM,CAAC;IAC/C,SAAS,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACxC,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC;IAClC,SAAS,CAAC,gBAAgB,uBAA8B;IACxD,SAAS,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IACvD,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;;;;;;;;;MA4B3B;IAEF;;OAEG;gBACS,OAAO,EAAE,mBAAmB,CAAC,OAAO,EAAE,YAAY,CAAC,EAAW,MAAM,gBAAkC;IAsJlH;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;IAuB/B,eAAe;IAMrB;;;;OAIG;IACG,WAAW,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,GAAE,yBAA8B,GAAG,OAAO,CAAC,wBAAwB,CAAC;cAmE9H,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;cAetB,kBAAkB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3E;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM;cAQrD,iBAAiB;IAqCjC;;;OAGG;cACa,iBAAiB;IAmBjC;;;OAGG;cACa,gBAAgB;IA+GhC;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,UAAU,SAAI,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAarJ;;OAEG;cACa,oBAAoB;IASpC;;OAEG;cACa,0BAA0B;IAY1C;;OAEG;cACa,4BAA4B,CACxC,KAAK,EAAE,KAAK,EACZ,eAAe,EAAE,OAAO,EACxB,MAAM,EAAE,WAAW,GAAG,YAAY,GACnC,OAAO,CAAC,IAAI,CAAC;cA+BA,2BAA2B,CAAC,eAAe,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAezF;;;;;;;OAOG;cACa,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;cAQzC,aAAa,CAAC,QAAQ,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;IAQpI;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAM/B,SAAS,CAAC,yBAAyB,CAAC,GAAG,EAAE,GAAG,EAAE,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAsB,GACzB,EAAE,4BAA4B,CAAC,GAAG,EAAE,GAAG,CAAC;IA0BzC,SAAS,CAAC,2BAA2B,CAAC,OAAO,EAAE,OAAO;CAGzD;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;;OAIG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;OAYG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D;AAED,UAAU,4BAA4B,CAAC,GAAG,EAAE,GAAG;IAC3C,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B"}
1
+ {"version":3,"file":"basic-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/basic-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,KAAK,EACR,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,OAAO,EACP,WAAW,EACX,cAAc,EACd,4BAA4B,EAC5B,aAAa,EACb,OAAO,EACP,kBAAkB,EACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EACH,cAAc,EACd,aAAa,EACb,KAAK,eAAe,EAOpB,YAAY,EAEZ,WAAW,EACX,UAAU,EAGb,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,cAAc,EAA6B,QAAQ,IAAI,WAAW,EAAE,MAAM,cAAc,CAAC;AAEvG,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,SAAS,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAItG,MAAM,WAAW,oBAAoB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC7G,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,CAAC,OAAO,EAAE,+BAA+B,KAAK,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC5F,WAAW,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;CAC5F;AAED,gBAAgB;AAChB,oBAAY,+BAA+B,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAA;AAavF,oBAAY,cAAc,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE1H,oBAAY,oBAAoB,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE9I,MAAM,WAAW,mBAAmB,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IACvF;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEzC;;;;;;;;;;;;;;;;;;OAkBG;IACH,qBAAqB,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IAEhD;;;;;OAKG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAErD;;;;;;;;;OASG;IACH,2BAA2B,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAE5D;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAE9C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;CACb;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,YAAY,CAAC,OAAO,SAAS,eAAe,GAAG,oBAAoB;IA2FpB,QAAQ,CAAC,MAAM;IA1FvE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAmB;IAE5D;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAE3B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B;;;;;;OAMG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAA4B;IAEnE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,cAAc,EAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnD,SAAS,CAAC,YAAY,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACvD,SAAS,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC/D,SAAS,CAAC,2BAA2B,EAAG,MAAM,CAAC;IAC/C,SAAS,CAAC,qBAAqB,EAAE,MAAM,CAAC;IACxC,SAAS,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACvC,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC;IAClC,SAAS,CAAC,gBAAgB,uBAA8B;IACxD,SAAS,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IACvD,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,OAAO,CAAC,YAAY,CAAC,CAAU;IAE/B,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;;;;;;;;;;;;;MA6B3B;IAEF;;OAEG;gBACS,OAAO,GAAE,mBAAmB,CAAC,OAAO,CAAM,EAAW,MAAM,gBAAkC;IAiKzG;;;;;;;OAOG;IACG,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,eAAe,CAAC;IA6BpH,eAAe;IAMf,QAAQ,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,QAAc,GAAG,OAAO,CAAC,KAAK,CAAC;IAKjG;;;;OAIG;IACG,WAAW,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,OAAO,GAAE,yBAA8B,GAAG,OAAO,CAAC,wBAAwB,CAAC;cAmE9H,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;cAoBtB,kBAAkB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3E;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM;cAQrD,iBAAiB;IAqCjC;;;OAGG;cACa,iBAAiB;IAmBjC;;;OAGG;cACa,gBAAgB;IAoHhC;;;OAGG;cACa,gBAAgB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,UAAU,SAAI,EAAE,OAAO,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAarJ;;OAEG;cACa,oBAAoB;IASpC;;OAEG;cACa,0BAA0B;IAY1C;;OAEG;cACa,4BAA4B,CACxC,KAAK,EAAE,KAAK,EACZ,eAAe,EAAE,OAAO,EACxB,MAAM,EAAE,WAAW,GAAG,YAAY,GACnC,OAAO,CAAC,IAAI,CAAC;cAoCA,oBAAoB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC;cAStD,2BAA2B,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAclG;;;;;;;OAOG;cACa,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;cAQzC,aAAa,CAAC,QAAQ,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;IAQpI;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAY/B,SAAS,CAAC,yBAAyB,CAAC,GAAG,EAAE,GAAG,EAAE,EAC1C,WAAW,EACX,OAAO,EACP,WAAW,EACX,OAAO,EACP,WAAW,EACX,cAAsB,GACzB,EAAE,4BAA4B,CAAC,GAAG,EAAE,GAAG,CAAC;IA0BzC,SAAS,CAAC,2BAA2B,CAAC,OAAO,EAAE,OAAO;CAGzD;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D;AAED,UAAU,4BAA4B,CAAC,GAAG,EAAE,GAAG;IAC3C,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,WAAW,CAAC,EAAE,GAAG,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,SAAS,oBAAoB,GAAG,oBAAoB,4BAE5F"}