@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +4 -11
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +0 -7
- package/configuration.d.ts.map +1 -1
- package/configuration.js +0 -4
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +4 -6
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +61 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +99 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +12 -16
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +0 -8
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/enqueue_links/enqueue_links.d.ts +22 -15
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +33 -14
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +8 -2
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +40 -18
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +12 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +20 -0
- package/errors.js.map +1 -1
- package/events/local_event_manager.d.ts +0 -2
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +3 -32
- package/events/local_event_manager.js.map +1 -1
- package/http_clients/base-http-client.d.ts +26 -69
- package/http_clients/base-http-client.d.ts.map +1 -1
- package/http_clients/base-http-client.js +20 -9
- package/http_clients/base-http-client.js.map +1 -1
- package/http_clients/got-scraping-http-client.d.ts +10 -3
- package/http_clients/got-scraping-http-client.d.ts.map +1 -1
- package/http_clients/got-scraping-http-client.js +86 -39
- package/http_clients/got-scraping-http-client.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/log.d.ts +4 -2
- package/log.d.ts.map +1 -1
- package/log.js.map +1 -1
- package/package.json +6 -6
- package/proxy_configuration.d.ts +21 -40
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +15 -51
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +123 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +140 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +31 -5
- package/request.d.ts.map +1 -1
- package/request.js +35 -12
- package/request.js.map +1 -1
- package/session_pool/session.d.ts +6 -2
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +9 -2
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +9 -2
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +13 -1
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +7 -0
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +10 -1
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +13 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +11 -6
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +0 -6
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +68 -4
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +74 -44
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue_v2.d.ts +2 -2
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +1 -1
- package/storages/storage_manager.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { log } from '../log.js';
|
|
2
|
+
/**
|
|
3
|
+
* A request manager that combines a RequestList and a RequestQueue.
|
|
4
|
+
* It first reads requests from the RequestList and then, when needed,
|
|
5
|
+
* transfers them in batches to the RequestQueue.
|
|
6
|
+
*/
|
|
7
|
+
export class RequestManagerTandem {
|
|
8
|
+
log;
|
|
9
|
+
requestList;
|
|
10
|
+
requestQueue;
|
|
11
|
+
constructor(requestList, requestQueue) {
|
|
12
|
+
this.log = log.child({ prefix: 'RequestManagerTandem' });
|
|
13
|
+
this.requestList = requestList;
|
|
14
|
+
this.requestQueue = requestQueue;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Transfers a batch of requests from the RequestList to the RequestQueue.
|
|
18
|
+
* Handles both successful transfers and failures appropriately.
|
|
19
|
+
* @private
|
|
20
|
+
*/
|
|
21
|
+
async transferNextBatchToQueue() {
|
|
22
|
+
const request = await this.requestList.fetchNextRequest();
|
|
23
|
+
if (request === null) {
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
try {
|
|
27
|
+
await this.requestQueue.addRequest(request, { forefront: true });
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
// If requestQueue.addRequest() fails here then we must reclaim it back to
|
|
31
|
+
// the RequestList because probably it's not yet in the queue!
|
|
32
|
+
this.log.error('Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.', { request });
|
|
33
|
+
await this.requestList.reclaimRequest(request);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
await this.requestList.markRequestHandled(request);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
|
|
40
|
+
* is not finished, it will transfer a batch of requests from the list to the queue first.
|
|
41
|
+
* @inheritdoc
|
|
42
|
+
*/
|
|
43
|
+
async fetchNextRequest() {
|
|
44
|
+
// First, try to transfer a request from the requestList
|
|
45
|
+
const [listEmpty, listFinished] = await Promise.all([
|
|
46
|
+
this.requestList.isEmpty(),
|
|
47
|
+
this.requestList.isFinished(),
|
|
48
|
+
]);
|
|
49
|
+
if (!listEmpty && !listFinished) {
|
|
50
|
+
await this.transferNextBatchToQueue();
|
|
51
|
+
}
|
|
52
|
+
// Try to fetch from queue after potential transfer
|
|
53
|
+
return this.requestQueue.fetchNextRequest();
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* @inheritdoc
|
|
57
|
+
*/
|
|
58
|
+
async isFinished() {
|
|
59
|
+
const storagesFinished = await Promise.all([this.requestList.isFinished(), this.requestQueue.isFinished()]);
|
|
60
|
+
return storagesFinished.every(Boolean);
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* @inheritdoc
|
|
64
|
+
*/
|
|
65
|
+
async isEmpty() {
|
|
66
|
+
const storagesEmpty = await Promise.all([this.requestList.isEmpty(), this.requestQueue.isEmpty()]);
|
|
67
|
+
return storagesEmpty.every(Boolean);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* @inheritdoc
|
|
71
|
+
*/
|
|
72
|
+
async handledCount() {
|
|
73
|
+
// Since one of the stores needs to have priority when both are present, we query the request queue - the request list will first be dumped into the queue and then left empty.
|
|
74
|
+
return await this.requestQueue.handledCount();
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* @inheritdoc
|
|
78
|
+
*/
|
|
79
|
+
getTotalCount() {
|
|
80
|
+
return this.requestQueue.getTotalCount();
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* @inheritdoc
|
|
84
|
+
*/
|
|
85
|
+
getPendingCount() {
|
|
86
|
+
return this.requestQueue.getPendingCount() + this.requestList.length() - this.requestList.handledCount();
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* @inheritdoc
|
|
90
|
+
*/
|
|
91
|
+
async *[Symbol.asyncIterator]() {
|
|
92
|
+
while (true) {
|
|
93
|
+
const req = await this.fetchNextRequest();
|
|
94
|
+
if (!req)
|
|
95
|
+
break;
|
|
96
|
+
yield req;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* @inheritdoc
|
|
101
|
+
*/
|
|
102
|
+
async markRequestHandled(request) {
|
|
103
|
+
return this.requestQueue.markRequestHandled(request);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* @inheritdoc
|
|
107
|
+
*/
|
|
108
|
+
async reclaimRequest(request, options) {
|
|
109
|
+
return await this.requestQueue.reclaimRequest(request, options);
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* @inheritdoc
|
|
113
|
+
*/
|
|
114
|
+
async addRequest(requestLike, options) {
|
|
115
|
+
return await this.requestQueue.addRequest(requestLike, options);
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* @inheritdoc
|
|
119
|
+
*/
|
|
120
|
+
async addRequestsBatched(requests, options) {
|
|
121
|
+
return await this.requestQueue.addRequestsBatched(requests, options);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
//# sourceMappingURL=request_manager_tandem.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request_manager_tandem.js","sourceRoot":"","sources":["../../src/storages/request_manager_tandem.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAYhC;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IACrB,GAAG,CAAM;IACT,WAAW,CAAe;IAC1B,YAAY,CAAkB;IAEtC,YAAY,WAAyB,EAAE,YAA6B;QAChE,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,wBAAwB;QAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,yGAAyG,EACzG,EAAE,OAAO,EAAE,CACd,CAAC;YACF,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QAClB,wDAAwD;QACxD,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChD,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE;YAC1B,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAC1C,CAAC;QAED,mDAAmD;QACnD,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAK,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC5G,OAAO,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnG,OAAO,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,+KAA+K;QAC/K,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,aAAa;QACT,OAAO,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,eAAe;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;IAC7G,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,OAAsC;QAEtC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,OAAsC;QACxE,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAAsB,EACtB,OAAmC;QAEnC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;CACJ"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueClient, RequestQueueInfo, StorageClient } from '@crawlee/types';
|
|
2
|
+
import type { ReadonlyDeep } from 'type-fest';
|
|
2
3
|
import { ListDictionary, LruCache } from '@apify/datastructures';
|
|
3
4
|
import type { Log } from '@apify/log';
|
|
4
5
|
import { Configuration } from '../configuration.js';
|
|
@@ -6,7 +7,58 @@ import type { ProxyConfiguration } from '../proxy_configuration.js';
|
|
|
6
7
|
import type { InternalSource, RequestOptions, Source } from '../request.js';
|
|
7
8
|
import { Request } from '../request.js';
|
|
8
9
|
import type { IStorage, StorageManagerOptions } from './storage_manager.js';
|
|
9
|
-
export
|
|
10
|
+
export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
|
|
11
|
+
/**
|
|
12
|
+
* Represents a provider of requests/URLs to crawl.
|
|
13
|
+
*/
|
|
14
|
+
export interface IRequestManager {
|
|
15
|
+
/**
|
|
16
|
+
* Returns `true` if all requests were already handled and there are no more left.
|
|
17
|
+
*/
|
|
18
|
+
isFinished(): Promise<boolean>;
|
|
19
|
+
/**
|
|
20
|
+
* Resolves to `true` if the next call to {@link IRequestManager.fetchNextRequest} function
|
|
21
|
+
* would return `null`, otherwise it resolves to `false`.
|
|
22
|
+
* Note that even if the provider is empty, there might be some pending requests currently being processed.
|
|
23
|
+
*/
|
|
24
|
+
isEmpty(): Promise<boolean>;
|
|
25
|
+
/**
|
|
26
|
+
* Returns number of handled requests.
|
|
27
|
+
*/
|
|
28
|
+
handledCount(): Promise<number>;
|
|
29
|
+
/**
|
|
30
|
+
* Get the total number of requests known to the request manager.
|
|
31
|
+
*/
|
|
32
|
+
getTotalCount(): number;
|
|
33
|
+
/**
|
|
34
|
+
* Get an offline approximation of the number of pending requests.
|
|
35
|
+
*/
|
|
36
|
+
getPendingCount(): number;
|
|
37
|
+
/**
|
|
38
|
+
* Gets the next {@link Request} to process.
|
|
39
|
+
*
|
|
40
|
+
* The function's `Promise` resolves to `null` if there are no more
|
|
41
|
+
* requests to process.
|
|
42
|
+
*/
|
|
43
|
+
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
|
|
44
|
+
/**
|
|
45
|
+
* Can be used to iterate over the `RequestManager` instance in a `for await .. of` loop.
|
|
46
|
+
* Provides an alternative for the repeated use of `fetchNextRequest`.
|
|
47
|
+
*/
|
|
48
|
+
[Symbol.asyncIterator](): AsyncGenerator<Request>;
|
|
49
|
+
/**
|
|
50
|
+
* Marks request as handled after successful processing.
|
|
51
|
+
*/
|
|
52
|
+
markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
|
|
53
|
+
/**
|
|
54
|
+
* Reclaims request to the provider if its processing failed.
|
|
55
|
+
* The request will become available in the next `fetchNextRequest()`.
|
|
56
|
+
*/
|
|
57
|
+
reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
|
|
58
|
+
addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
|
|
59
|
+
addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
|
|
60
|
+
}
|
|
61
|
+
export declare abstract class RequestProvider implements IStorage, IRequestManager {
|
|
10
62
|
readonly config: Configuration;
|
|
11
63
|
id: string;
|
|
12
64
|
name?: string;
|
|
@@ -20,6 +72,7 @@ export declare abstract class RequestProvider implements IStorage {
|
|
|
20
72
|
assumedTotalCount: number;
|
|
21
73
|
assumedHandledCount: number;
|
|
22
74
|
private initialCount;
|
|
75
|
+
private initialHandledCount;
|
|
23
76
|
protected queueHeadIds: ListDictionary<string>;
|
|
24
77
|
protected requestCache: LruCache<RequestLruItem>;
|
|
25
78
|
protected recentlyHandledRequestsCache: LruCache<boolean>;
|
|
@@ -34,6 +87,12 @@ export declare abstract class RequestProvider implements IStorage {
|
|
|
34
87
|
* Survives restarts and actor migrations.
|
|
35
88
|
*/
|
|
36
89
|
getTotalCount(): number;
|
|
90
|
+
/**
|
|
91
|
+
* Returns an offline approximation of the total number of pending requests in the queue.
|
|
92
|
+
*
|
|
93
|
+
* Survives restarts and Actor migrations.
|
|
94
|
+
*/
|
|
95
|
+
getPendingCount(): number;
|
|
37
96
|
/**
|
|
38
97
|
* Adds a request to the queue.
|
|
39
98
|
*
|
|
@@ -63,7 +122,7 @@ export declare abstract class RequestProvider implements IStorage {
|
|
|
63
122
|
* Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
|
|
64
123
|
* @param [options] Request queue operation options.
|
|
65
124
|
*/
|
|
66
|
-
addRequests(requestsLike:
|
|
125
|
+
addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
67
126
|
/**
|
|
68
127
|
* Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
|
|
69
128
|
* adding the rest in the background. You can configure the batch size via `batchSize` option and the sleep time in between
|
|
@@ -73,7 +132,7 @@ export declare abstract class RequestProvider implements IStorage {
|
|
|
73
132
|
* @param requests The requests to add
|
|
74
133
|
* @param options Options for the request queue
|
|
75
134
|
*/
|
|
76
|
-
addRequestsBatched(requests:
|
|
135
|
+
addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
|
|
77
136
|
/**
|
|
78
137
|
* Gets the request from the queue specified by ID.
|
|
79
138
|
*
|
|
@@ -98,7 +157,7 @@ export declare abstract class RequestProvider implements IStorage {
|
|
|
98
157
|
* @returns
|
|
99
158
|
* Returns the request object or `null` if there are no more pending requests.
|
|
100
159
|
*/
|
|
101
|
-
abstract fetchNextRequest<T extends Dictionary = Dictionary>(
|
|
160
|
+
abstract fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
|
|
102
161
|
/**
|
|
103
162
|
* Marks a request that was previously returned by the
|
|
104
163
|
* {@link RequestQueue.fetchNextRequest}
|
|
@@ -142,6 +201,10 @@ export declare abstract class RequestProvider implements IStorage {
|
|
|
142
201
|
* depending on the mode of operation.
|
|
143
202
|
*/
|
|
144
203
|
drop(): Promise<void>;
|
|
204
|
+
/**
|
|
205
|
+
* @inheritdoc
|
|
206
|
+
*/
|
|
207
|
+
[Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
|
|
145
208
|
/**
|
|
146
209
|
* Returns the number of handled requests.
|
|
147
210
|
*
|
|
@@ -150,6 +213,7 @@ export declare abstract class RequestProvider implements IStorage {
|
|
|
150
213
|
* ```javascript
|
|
151
214
|
* const { handledRequestCount } = await queue.getInfo();
|
|
152
215
|
* ```
|
|
216
|
+
* @inheritdoc
|
|
153
217
|
*/
|
|
154
218
|
handledCount(): Promise<number>;
|
|
155
219
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAChB,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAChB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAI5E,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE5G;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhC;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAElF;;;OAGG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAElD;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAEvF;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC;IAEpH,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE5G,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtH;AAED,8BAAsB,eAAgB,YAAW,QAAQ,EAAE,eAAe;IAmClE,QAAQ,CAAC,MAAM;IAlCnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,GAAG,CAAC;IACT,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,mBAAmB,CAAK;IAEhC,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;gBAGtC,OAAO,EAAE,8BAA8B,EAC9B,MAAM,gBAAkC;IAsBrD;;;;OAIG;IACH,aAAa;IAIb;;;;OAIG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IA4GlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,CAAC,YAAY,CAAC,EACpC,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IAmIpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAE1F;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;;;;;;;;OASG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAMtD;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAQjC;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC;CA6BlH;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IAEtB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { inspect } from 'node:util';
|
|
2
|
-
import {
|
|
2
|
+
import { chunkedAsyncIterable, downloadListOfUrls, getObjectType, isAsyncIterable, isIterable, peekableAsyncIterable, sleep, } from '@crawlee/utils';
|
|
3
3
|
import ow from 'ow';
|
|
4
4
|
import { ListDictionary, LruCache } from '@apify/datastructures';
|
|
5
5
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
@@ -25,6 +25,7 @@ export class RequestProvider {
|
|
|
25
25
|
assumedTotalCount = 0;
|
|
26
26
|
assumedHandledCount = 0;
|
|
27
27
|
initialCount = 0;
|
|
28
|
+
initialHandledCount = 0; // We track this separately from `assumedHandledCount` which is used non-trivially by RequestQueueV1
|
|
28
29
|
queueHeadIds = new ListDictionary();
|
|
29
30
|
requestCache;
|
|
30
31
|
recentlyHandledRequestsCache;
|
|
@@ -57,6 +58,14 @@ export class RequestProvider {
|
|
|
57
58
|
getTotalCount() {
|
|
58
59
|
return this.assumedTotalCount + this.initialCount;
|
|
59
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Returns an offline approximation of the total number of pending requests in the queue.
|
|
63
|
+
*
|
|
64
|
+
* Survives restarts and Actor migrations.
|
|
65
|
+
*/
|
|
66
|
+
getPendingCount() {
|
|
67
|
+
return this.getTotalCount() - this.initialHandledCount - this.assumedHandledCount;
|
|
68
|
+
}
|
|
60
69
|
/**
|
|
61
70
|
* Adds a request to the queue.
|
|
62
71
|
*
|
|
@@ -134,7 +143,9 @@ export class RequestProvider {
|
|
|
134
143
|
async addRequests(requestsLike, options = {}) {
|
|
135
144
|
checkStorageAccess();
|
|
136
145
|
this.lastActivity = new Date();
|
|
137
|
-
ow(requestsLike, ow.
|
|
146
|
+
ow(requestsLike, ow.object
|
|
147
|
+
.is((value) => isIterable(value) || isAsyncIterable(value))
|
|
148
|
+
.message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
|
|
138
149
|
ow(options, ow.object.exactShape({
|
|
139
150
|
forefront: ow.optional.boolean,
|
|
140
151
|
cache: ow.optional.boolean,
|
|
@@ -153,17 +164,19 @@ export class RequestProvider {
|
|
|
153
164
|
processedRequests: [],
|
|
154
165
|
unprocessedRequests: [],
|
|
155
166
|
};
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
167
|
+
const requests = [];
|
|
168
|
+
for await (const requestLike of requestsLike) {
|
|
169
|
+
if (typeof requestLike === 'string') {
|
|
170
|
+
requests.push(new Request({ url: requestLike }));
|
|
171
|
+
}
|
|
172
|
+
else if ('requestsFromUrl' in requestLike) {
|
|
173
|
+
const fetchedRequests = await this._fetchRequestsFromUrl(requestLike);
|
|
174
|
+
await this._addFetchedRequests(requestLike, fetchedRequests, options);
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike));
|
|
160
178
|
}
|
|
161
179
|
}
|
|
162
|
-
const requests = requestsLike
|
|
163
|
-
.filter((requestLike) => !('requestsFromUrl' in requestLike))
|
|
164
|
-
.map((requestLike) => {
|
|
165
|
-
return requestLike instanceof Request ? requestLike : new Request(requestLike);
|
|
166
|
-
});
|
|
167
180
|
const requestsToAdd = new Map();
|
|
168
181
|
for (const request of requests) {
|
|
169
182
|
const cacheKey = getCachedRequestId(request.uniqueKey);
|
|
@@ -219,43 +232,44 @@ export class RequestProvider {
|
|
|
219
232
|
async addRequestsBatched(requests, options = {}) {
|
|
220
233
|
checkStorageAccess();
|
|
221
234
|
this.lastActivity = new Date();
|
|
235
|
+
ow(requests, ow.object
|
|
236
|
+
.is((value) => isIterable(value) || isAsyncIterable(value))
|
|
237
|
+
.message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
|
|
222
238
|
ow(options, ow.object.exactShape({
|
|
223
239
|
forefront: ow.optional.boolean,
|
|
224
240
|
waitForAllRequestsToBeAdded: ow.optional.boolean,
|
|
225
241
|
batchSize: ow.optional.number,
|
|
226
242
|
waitBetweenBatchesMillis: ow.optional.number,
|
|
227
243
|
}));
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
244
|
+
const addRequest = this.addRequest.bind(this);
|
|
245
|
+
async function* generateRequests() {
|
|
246
|
+
for await (const opts of requests) {
|
|
247
|
+
// Validate the input
|
|
248
|
+
if (typeof opts === 'object' && opts !== null) {
|
|
249
|
+
if (opts.url !== undefined && typeof opts.url !== 'string') {
|
|
250
|
+
throw new Error(`Request options are not valid, the 'url' property is not a string. Input: ${inspect(opts)}`);
|
|
251
|
+
}
|
|
252
|
+
if (opts.id !== undefined) {
|
|
253
|
+
throw new Error(`Request options are not valid, the 'id' property must not be present. Input: ${inspect(opts)}`);
|
|
254
|
+
}
|
|
255
|
+
if (opts.requestsFromUrl !== undefined &&
|
|
256
|
+
typeof opts.requestsFromUrl !== 'string') {
|
|
257
|
+
throw new Error(`Request options are not valid, the 'requestsFromUrl' property is not a string. Input: ${inspect(opts)}`);
|
|
258
|
+
}
|
|
242
259
|
}
|
|
243
|
-
if (typeof
|
|
244
|
-
|
|
260
|
+
if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
|
|
261
|
+
// Handle URL lists right away
|
|
262
|
+
await addRequest(opts, { forefront: options.forefront });
|
|
263
|
+
}
|
|
264
|
+
else {
|
|
265
|
+
// Yield valid requests
|
|
266
|
+
yield typeof opts === 'string' ? { url: opts } : opts;
|
|
245
267
|
}
|
|
246
268
|
}
|
|
247
|
-
throw new Error(`Request options are not valid, provide either a URL or an object with 'url' property (but without 'id' property), or an object with 'requestsFromUrl' property. Input: ${inspect(request)}`);
|
|
248
269
|
}
|
|
249
270
|
const { batchSize = 1000, waitBetweenBatchesMillis = 1000 } = options;
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
|
|
253
|
-
await this.addRequest(opts, { forefront: options.forefront });
|
|
254
|
-
}
|
|
255
|
-
else {
|
|
256
|
-
sources.push(typeof opts === 'string' ? { url: opts } : opts);
|
|
257
|
-
}
|
|
258
|
-
}
|
|
271
|
+
const chunks = peekableAsyncIterable(chunkedAsyncIterable(generateRequests(), batchSize));
|
|
272
|
+
const chunksIterator = chunks[Symbol.asyncIterator]();
|
|
259
273
|
const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true) => {
|
|
260
274
|
const resultsToReturn = [];
|
|
261
275
|
const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
|
|
@@ -266,11 +280,15 @@ export class RequestProvider {
|
|
|
266
280
|
}
|
|
267
281
|
return resultsToReturn;
|
|
268
282
|
};
|
|
269
|
-
const initialChunk = sources.splice(0, batchSize);
|
|
270
283
|
// Add initial batch of `batchSize` to process them right away
|
|
284
|
+
const initialChunk = await chunksIterator.peek();
|
|
285
|
+
if (initialChunk === undefined) {
|
|
286
|
+
return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]) };
|
|
287
|
+
}
|
|
271
288
|
const addedRequests = await attemptToAddToQueueAndAddAnyUnprocessed(initialChunk);
|
|
272
|
-
|
|
273
|
-
|
|
289
|
+
await chunksIterator.next();
|
|
290
|
+
// If we have no more requests to add, return immediately
|
|
291
|
+
if ((await chunksIterator.peek()) === undefined) {
|
|
274
292
|
return {
|
|
275
293
|
addedRequests,
|
|
276
294
|
waitForAllRequestsToBeAdded: Promise.resolve([]),
|
|
@@ -278,9 +296,8 @@ export class RequestProvider {
|
|
|
278
296
|
}
|
|
279
297
|
// eslint-disable-next-line no-async-promise-executor
|
|
280
298
|
const promise = new Promise(async (resolve) => {
|
|
281
|
-
const chunks = chunk(sources, batchSize);
|
|
282
299
|
const finalAddedRequests = [];
|
|
283
|
-
for (const requestChunk of chunks) {
|
|
300
|
+
for await (const requestChunk of chunks) {
|
|
284
301
|
finalAddedRequests.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(requestChunk, false)));
|
|
285
302
|
await sleep(waitBetweenBatchesMillis);
|
|
286
303
|
}
|
|
@@ -429,6 +446,17 @@ export class RequestProvider {
|
|
|
429
446
|
const manager = StorageManager.getManager(this.constructor, this.config);
|
|
430
447
|
manager.closeStorage(this);
|
|
431
448
|
}
|
|
449
|
+
/**
|
|
450
|
+
* @inheritdoc
|
|
451
|
+
*/
|
|
452
|
+
async *[Symbol.asyncIterator]() {
|
|
453
|
+
while (true) {
|
|
454
|
+
const req = await this.fetchNextRequest();
|
|
455
|
+
if (!req)
|
|
456
|
+
break;
|
|
457
|
+
yield req;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
432
460
|
/**
|
|
433
461
|
* Returns the number of handled requests.
|
|
434
462
|
*
|
|
@@ -437,6 +465,7 @@ export class RequestProvider {
|
|
|
437
465
|
* ```javascript
|
|
438
466
|
* const { handledRequestCount } = await queue.getInfo();
|
|
439
467
|
* ```
|
|
468
|
+
* @inheritdoc
|
|
440
469
|
*/
|
|
441
470
|
async handledCount() {
|
|
442
471
|
// NOTE: We keep this function for compatibility with RequestList.handledCount()
|
|
@@ -547,8 +576,9 @@ export class RequestProvider {
|
|
|
547
576
|
const manager = StorageManager.getManager(this, options.config);
|
|
548
577
|
const queue = await manager.openStorage(queueIdOrName, options.storageClient);
|
|
549
578
|
queue.proxyConfiguration = options.proxyConfiguration;
|
|
550
|
-
|
|
551
|
-
queue
|
|
579
|
+
const queueInfo = await queue.client.get();
|
|
580
|
+
queue.initialCount = queueInfo?.totalRequestCount ?? 0;
|
|
581
|
+
queue.initialHandledCount = queueInfo?.handledRequestCount ?? 0;
|
|
552
582
|
return queue;
|
|
553
583
|
}
|
|
554
584
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_provider.js","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,OAAO,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEvF,MAAM,OAAgB,eAAe;IAkCpB;IAjCb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,WAAW,GAAG,EAAE,CAAC;IACjB,SAAS,GAAG,oBAAoB,EAAE,CAAC;IACnC,MAAM,CAAqB;IACjB,kBAAkB,CAAsB;IAElD,GAAG,CAAM;IACT,qBAAqB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,4DAA4D;IAChG,eAAe,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,4DAA4D;IAEtF,mFAAmF;IACnF,mFAAmF;IACnF,iBAAiB,GAAG,CAAC,CAAC;IACtB,mBAAmB,GAAG,CAAC,CAAC;IAEhB,YAAY,GAAG,CAAC,CAAC;IAEf,YAAY,GAAG,IAAI,cAAc,EAAU,CAAC;IAC5C,YAAY,CAA2B;IAEvC,4BAA4B,CAAoB;IAEhD,uBAAuB,GAAG,KAAK,CAAC;IAEhC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;IAE1B,oCAAoC,GAAG,CAAC,CAAC;IAEzC,2BAA2B,GAAG,CAAC,CAAC;IAE1C,YACI,OAAuC,EAC9B,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC,4BAA4B,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,8BAA8B,EAAE,CAAC,CAAC;QACxG,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,IAAI,SAAS,GAAG,EAAE,CAAC,CAAC;QAEhG,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAE9C,YAAY,CAAC,EAAE,wCAAsB,KAAK,IAAI,EAAE;YAC5C,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,aAAa;QACT,OAAO,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,UAAU,CACZ,WAAmB,EACnB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;YACjF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE3G,OAAO,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QAClD,CAAC;QAED,EAAE,CACE,WAAW,EACX,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,GAAG,EAAE,EAAE,CAAC,MAAM;YACd,EAAE,EAAE,EAAE,CAAC,SAAS;SACnB,CAAC,CACL,CAAC;QAEF,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;QAExF,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,UAAU,EAAE,CAAC;YACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,IAAI;gBACvB,mFAAmF;gBACnF,0FAA0F;gBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;gBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,SAAS;aACZ,CAAC;QACN,CAAC;QAED,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;YACzD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QAEtC,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,kBAAkB,CAAC;QAC5D,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAEjD,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEzB,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,WAAW,CACb,YAAsB,EACtB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QAC3B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtD,MAAM,kBAAkB,GAAG,CAAC,SAAiB,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAElD,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;YAE1B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YAC5C,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAEhD,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,MAAM,OAAO,GAA2B;YACpC,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QAEF,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACrC,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;gBACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;gBACjF,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrF,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,YAAY;aACxB,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,IAAI,WAAW,CAAC,CAAC;aAC5D,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YACjB,OAAO,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAA6B,CAAC,CAAC;QACrG,CAAC,CAAC,CAAC;QAEP,MAAM,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEnD,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC3B,iBAAiB,EAAE,IAAI;oBACvB,mFAAmF;oBACnF,0FAA0F;oBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;oBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;oBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;iBAClC,CAAC,CAAC;YACP,CAAC;iBAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;QACL,CAAC;QAED,yDAAyD;QACzD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAElG,8BAA8B;QAC9B,OAAO,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;QAE7D,2CAA2C;QAC3C,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,iBAAiB,EAAE,CAAC;YACpD,4CAA4C;YAC5C,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE1D,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC;YAEpD,IAAI,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/D,CAAC;YAED,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAEzB,qEAAqE;gBACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAA6B,EAC7B,UAAqC,EAAE;QAEvC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAChD,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC/C,CAAC,CACL,CAAC;QAEF,qFAAqF;QACrF,iGAAiG;QAEjG,uCAAuC;QACvC,iBAAiB;QACjB,oEAAoE;QACpE,yFAAyF;QACzF,OAAO;QAEP,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC9B,SAAS;YACb,CAAC;YAED,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBAClD,IAAI,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,OAAO,CAAC,EAAE,KAAK,WAAW,EAAE,CAAC;oBACvE,SAAS;gBACb,CAAC;gBAED,IAAI,OAAQ,OAAe,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;oBACvD,SAAS;gBACb,CAAC;YACL,CAAC;YAED,MAAM,IAAI,KAAK,CACX,0KAA0K,OAAO,CAC7K,OAAO,CACV,EAAE,CACN,CAAC;QACN,CAAC;QAED,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,wBAAwB,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;YAClE,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,IAAuB,CAAC,CAAC;YACtF,CAAC;QACL,CAAC;QAED,MAAM,uCAAuC,GAAG,KAAK,EAAE,gBAA0B,EAAE,KAAK,GAAG,IAAI,EAAE,EAAE;YAC/F,MAAM,eAAe,GAAuB,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACpG,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAErD,IAAI,SAAS,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC;gBACvC,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAEtC,eAAe,CAAC,IAAI,CAChB,GAAG,CAAC,MAAM,uCAAuC,CAC7C,gBAAgB,CAAC,MAAM,CACnB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC,CACjF,EACD,KAAK,CACR,CAAC,CACL,CAAC;YACN,CAAC;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAElD,8DAA8D;QAC9D,MAAM,aAAa,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC;QAElF,mDAAmD;QACnD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,OAAO;gBACH,aAAa;gBACb,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;QACN,CAAC;QAED,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAqB,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACzC,MAAM,kBAAkB,GAAuB,EAAE,CAAC;YAElD,KAAK,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;gBAChC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,uCAAuC,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBAEjG,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC1C,CAAC;YAED,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QACtC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;YACtB,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,0GAA0G;QAC1G,IAAI,OAAO,CAAC,2BAA2B,EAAE,CAAC;YACtC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO;YACH,aAAa;YACb,2BAA2B,EAAE,OAAO;SACvC,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAoC,EAAU;QAC1D,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,OAAO,IAAI,OAAO,CAAC,cAA2C,CAAC,CAAC;IACpE,CAAC;IAqBD;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;YACpB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAChC,CAAC,CACL,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;QAE7F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAChE,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;gBAChC,GAAG,OAAO;gBACV,SAAS;aACZ,CAAC,CAAC;YACH,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAExD,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE,CAAC;YACxC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAErC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;SACvB,CAAC,CACL,CAAC;QACF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,oEAAoE;QACpE,wEAAwE;QACxE,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE;gBACzC,SAAS;aACZ,CAAC,CAAC;YACH,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAID;;;;;OAKG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC5C,CAAC;IAUS,MAAM;QACZ,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,4BAA4B,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,QAAgB,EAAE,kBAA6C;QACnF,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC5B,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,kBAAkB,CAAC,SAAS;SAC1C,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACO,2BAA2B,CAAC,SAAiB,EAAE,SAAkB;QACvE,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACtD,CAAC;aAAM,IAAI,IAAI,CAAC,iBAAiB,GAAG,qBAAqB,EAAE,CAAC;YACxD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,WAAoC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY;QACd,gFAAgF;QAChF,MAAM,EAAE,mBAAmB,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7D,OAAO,mBAAmB,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBACrC,GAAG,EAAE,eAAe;gBACpB,SAAS,EAAE,KAAK;gBAChB,QAAQ,EAAE,MAAM,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE;aACpD,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0CAA0C,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YACzF,OAAO,EAAE,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAC/B,MAAsB,EACtB,eAAiC,EACjC,OAAqC;QAErC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QAElF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY,EAAE,eAAe,CAAC,MAAM;YACpC,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,cAAc,EAAE,eAAe,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;YAC7D,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAA+D;QAG7F,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAmC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/F,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9E,KAAK,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAEtD,wCAAwC;QACxC,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,CAAC;QAE3E,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"request_provider.js","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,OAAO,EACH,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,UAAU,EACV,qBAAqB,EACrB,KAAK,GACR,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAiEvF,MAAM,OAAgB,eAAe;IAmCpB;IAlCb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,WAAW,GAAG,EAAE,CAAC;IACjB,SAAS,GAAG,oBAAoB,EAAE,CAAC;IACnC,MAAM,CAAqB;IACjB,kBAAkB,CAAsB;IAElD,GAAG,CAAM;IACT,qBAAqB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,4DAA4D;IAChG,eAAe,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,4DAA4D;IAEtF,mFAAmF;IACnF,mFAAmF;IACnF,iBAAiB,GAAG,CAAC,CAAC;IACtB,mBAAmB,GAAG,CAAC,CAAC;IAEhB,YAAY,GAAG,CAAC,CAAC;IACjB,mBAAmB,GAAG,CAAC,CAAC,CAAC,oGAAoG;IAE3H,YAAY,GAAG,IAAI,cAAc,EAAU,CAAC;IAC5C,YAAY,CAA2B;IAEvC,4BAA4B,CAAoB;IAEhD,uBAAuB,GAAG,KAAK,CAAC;IAEhC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;IAE1B,oCAAoC,GAAG,CAAC,CAAC;IAEzC,2BAA2B,GAAG,CAAC,CAAC;IAE1C,YACI,OAAuC,EAC9B,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC,4BAA4B,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,8BAA8B,EAAE,CAAC,CAAC;QACxG,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,IAAI,SAAS,GAAG,EAAE,CAAC,CAAC;QAEhG,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAE9C,YAAY,CAAC,EAAE,wCAAsB,KAAK,IAAI,EAAE;YAC5C,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,aAAa;QACT,OAAO,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,eAAe;QACX,OAAO,IAAI,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;IACtF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,UAAU,CACZ,WAAmB,EACnB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;YACjF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE3G,OAAO,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QAClD,CAAC;QAED,EAAE,CACE,WAAW,EACX,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,GAAG,EAAE,EAAE,CAAC,MAAM;YACd,EAAE,EAAE,EAAE,CAAC,SAAS;SACnB,CAAC,CACL,CAAC;QAEF,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;QAExF,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,UAAU,EAAE,CAAC;YACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,IAAI;gBACvB,mFAAmF;gBACnF,0FAA0F;gBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;gBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,SAAS;aACZ,CAAC;QACN,CAAC;QAED,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;YACzD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QAEtC,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,kBAAkB,CAAC;QAC5D,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAEjD,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEzB,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,WAAW,CACb,YAA0B,EAC1B,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,YAAY,EACZ,EAAE,CAAC,MAAM;aACJ,EAAE,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;aACnE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,+CAA+C,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CACjG,CAAC;QACF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtD,MAAM,kBAAkB,GAAG,CAAC,SAAiB,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAElD,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;YAE1B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YAC5C,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAEhD,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,MAAM,OAAO,GAA2B;YACpC,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QAEF,MAAM,QAAQ,GAA0B,EAAE,CAAC;QAE3C,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YAC3C,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAClC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;gBAC1C,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;gBACxF,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YAC5F,CAAC;iBAAM,CAAC;gBACJ,QAAQ,CAAC,IAAI,CACT,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAA6B,CAAC,CAC5F,CAAC;YACN,CAAC;QACL,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEnD,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC3B,iBAAiB,EAAE,IAAI;oBACvB,mFAAmF;oBACnF,0FAA0F;oBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;oBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;oBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;iBAClC,CAAC,CAAC;YACP,CAAC;iBAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;QACL,CAAC;QAED,yDAAyD;QACzD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAElG,8BAA8B;QAC9B,OAAO,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;QAE7D,2CAA2C;QAC3C,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,iBAAiB,EAAE,CAAC;YACpD,4CAA4C;YAC5C,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE1D,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC;YAEpD,IAAI,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/D,CAAC;YAED,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAEzB,qEAAqE;gBACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAAoC,EACpC,UAAqC,EAAE;QAEvC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,EAAE,CACE,QAAQ,EACR,EAAE,CAAC,MAAM;aACJ,EAAE,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;aACnE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,+CAA+C,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CACjG,CAAC;QAEF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAChD,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC/C,CAAC,CACL,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,KAAK,SAAS,CAAC,CAAC,gBAAgB;YAC5B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAChC,qBAAqB;gBACrB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;wBACzD,MAAM,IAAI,KAAK,CACX,6EAA6E,OAAO,CAAC,IAAI,CAAC,EAAE,CAC/F,CAAC;oBACN,CAAC;oBAED,IAAI,IAAI,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;wBACxB,MAAM,IAAI,KAAK,CACX,gFAAgF,OAAO,CAAC,IAAI,CAAC,EAAE,CAClG,CAAC;oBACN,CAAC;oBAED,IACK,IAAY,CAAC,eAAe,KAAK,SAAS;wBAC3C,OAAQ,IAAY,CAAC,eAAe,KAAK,QAAQ,EACnD,CAAC;wBACC,MAAM,IAAI,KAAK,CACX,yFAAyF,OAAO,CAAC,IAAI,CAAC,EAAE,CAC3G,CAAC;oBACN,CAAC;gBACL,CAAC;gBAED,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;oBAChE,8BAA8B;oBAC9B,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACJ,uBAAuB;oBACvB,MAAM,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,IAAuB,CAAC;gBAC9E,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,wBAAwB,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEtE,MAAM,MAAM,GAAG,qBAAqB,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;QAC1F,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAEtD,MAAM,uCAAuC,GAAG,KAAK,EAAE,gBAA0B,EAAE,KAAK,GAAG,IAAI,EAAE,EAAE;YAC/F,MAAM,eAAe,GAAuB,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACpG,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAErD,IAAI,SAAS,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC;gBACvC,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAEtC,eAAe,CAAC,IAAI,CAChB,GAAG,CAAC,MAAM,uCAAuC,CAC7C,gBAAgB,CAAC,MAAM,CACnB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC,CACjF,EACD,KAAK,CACR,CAAC,CACL,CAAC;YACN,CAAC;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,8DAA8D;QAC9D,MAAM,YAAY,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACjD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;QACnF,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE5B,yDAAyD;QACzD,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO;gBACH,aAAa;gBACb,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;QACN,CAAC;QAED,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAqB,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9D,MAAM,kBAAkB,GAAuB,EAAE,CAAC;YAElD,IAAI,KAAK,EAAE,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;gBACtC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,uCAAuC,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBAEjG,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC1C,CAAC;YAED,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QACtC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;YACtB,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,0GAA0G;QAC1G,IAAI,OAAO,CAAC,2BAA2B,EAAE,CAAC;YACtC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO;YACH,aAAa;YACb,2BAA2B,EAAE,OAAO;SACvC,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAoC,EAAU;QAC1D,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,OAAO,IAAI,OAAO,CAAC,cAA2C,CAAC,CAAC;IACpE,CAAC;IAqBD;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;YACpB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAChC,CAAC,CACL,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;QAE7F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAChE,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;gBAChC,GAAG,OAAO;gBACV,SAAS;aACZ,CAAC,CAAC;YACH,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAExD,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE,CAAC;YACxC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAErC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;SACvB,CAAC,CACL,CAAC;QACF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,oEAAoE;QACpE,wEAAwE;QACxE,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE;gBACzC,SAAS;aACZ,CAAC,CAAC;YACH,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAID;;;;;OAKG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC5C,CAAC;IAUS,MAAM;QACZ,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,4BAA4B,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,QAAgB,EAAE,kBAA6C;QACnF,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC5B,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,kBAAkB,CAAC,SAAS;SAC1C,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACO,2BAA2B,CAAC,SAAiB,EAAE,SAAkB;QACvE,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACtD,CAAC;aAAM,IAAI,IAAI,CAAC,iBAAiB,GAAG,qBAAqB,EAAE,CAAC;YACxD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,WAAoC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,YAAY;QACd,gFAAgF;QAChF,MAAM,EAAE,mBAAmB,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7D,OAAO,mBAAmB,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBACrC,GAAG,EAAE,eAAe;gBACpB,SAAS,EAAE,KAAK;gBAChB,QAAQ,EAAE,MAAM,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE;aACpD,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0CAA0C,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YACzF,OAAO,EAAE,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAC/B,MAAsB,EACtB,eAAiC,EACjC,OAAqC;QAErC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QAElF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY,EAAE,eAAe,CAAC,MAAM;YACpC,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,cAAc,EAAE,eAAe,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;YAC7D,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAIjC;QACG,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAmC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/F,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9E,KAAK,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAEtD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAE3C,KAAK,CAAC,YAAY,GAAG,SAAS,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACvD,KAAK,CAAC,mBAAmB,GAAG,SAAS,EAAE,mBAAmB,IAAI,CAAC,CAAC;QAEhE,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { Request, Source } from '../request.js';
|
|
4
|
-
import type { RequestProviderOptions, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_provider.js';
|
|
4
|
+
import type { RequestProviderOptions, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike } from './request_provider.js';
|
|
5
5
|
import { RequestProvider } from './request_provider.js';
|
|
6
6
|
/**
|
|
7
7
|
* Represents a queue of URLs to crawl, which is used for deep crawling of websites
|
|
@@ -54,7 +54,7 @@ export declare class RequestQueue extends RequestProvider {
|
|
|
54
54
|
/**
|
|
55
55
|
* @inheritDoc
|
|
56
56
|
*/
|
|
57
|
-
addRequests(requestsLike:
|
|
57
|
+
addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
58
58
|
/**
|
|
59
59
|
* @inheritDoc
|
|
60
60
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue_v2.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,KAAK,EACR,sBAAsB,EACtB,yBAAyB,EACzB,4BAA4B,
|
|
1
|
+
{"version":3,"file":"request_queue_v2.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,KAAK,EACR,sBAAsB,EACtB,yBAAyB,EACzB,4BAA4B,EAC5B,YAAY,EACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAiBxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,YAAa,SAAQ,eAAe;IAC7C,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,OAAO,CAAC,sBAAsB,CAAkC;IAChE,OAAO,CAAC,+BAA+B,CAAS;IAChD,OAAO,CAAC,oBAAoB,CAAK;gBAErB,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAkC;IAsBrF;;OAEG;cACgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAevG;;OAEG;IACY,UAAU,CACrB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAQrC;;OAEG;IACY,WAAW,CACtB,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAWlC;;OAEG;IACY,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAiDhG;;OAEG;IACY,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAK9F;;OAEG;IACY,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IA+E7C;;OAEG;IACY,cAAc,CACzB,GAAG,IAAI,EAAE,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC,GACvD,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;cAqBhC,oBAAoB;YAqBtB,UAAU;YAYV,gBAAgB;YAmFhB,mBAAmB;YAuFnB,mBAAmB;cAiBd,MAAM;cAMN,2BAA2B;cAI9B,mBAAmB;IAcnC;;OAEG;WACmB,IAAI,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;CAGtG"}
|