@crawlee/core 4.0.0-beta.24 → 4.0.0-beta.240

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (265) hide show
  1. package/README.md +15 -15
  2. package/byte-utils.d.ts +17 -0
  3. package/byte-utils.js +42 -0
  4. package/configuration.d.ts +77 -235
  5. package/configuration.js +157 -227
  6. package/debug.d.ts +36 -0
  7. package/debug.js +70 -0
  8. package/errors.d.ts +45 -18
  9. package/errors.js +58 -27
  10. package/events/event-manager.d.ts +78 -0
  11. package/events/{event_manager.js → event-manager.js} +12 -13
  12. package/events/index.d.ts +3 -3
  13. package/events/index.js +2 -3
  14. package/events/local-event-manager.d.ts +32 -0
  15. package/events/local-event-manager.js +101 -0
  16. package/events/system-info.d.ts +38 -0
  17. package/events/system-info.js +1 -0
  18. package/index.d.ts +9 -12
  19. package/index.js +10 -11
  20. package/internal.d.ts +8 -0
  21. package/internal.js +9 -0
  22. package/iterables.d.ts +79 -0
  23. package/iterables.js +134 -0
  24. package/log.d.ts +77 -1
  25. package/log.js +134 -1
  26. package/memory-storage/consts.d.ts +4 -0
  27. package/memory-storage/consts.js +4 -0
  28. package/memory-storage/index.d.ts +1 -0
  29. package/memory-storage/index.js +1 -0
  30. package/memory-storage/memory-storage.d.ts +43 -0
  31. package/memory-storage/memory-storage.js +158 -0
  32. package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
  33. package/memory-storage/resource-clients/common/base-client.js +6 -0
  34. package/memory-storage/resource-clients/dataset.d.ts +35 -0
  35. package/memory-storage/resource-clients/dataset.js +105 -0
  36. package/memory-storage/resource-clients/key-value-store.d.ts +54 -0
  37. package/memory-storage/resource-clients/key-value-store.js +179 -0
  38. package/memory-storage/resource-clients/request-queue.d.ts +55 -0
  39. package/memory-storage/resource-clients/request-queue.js +413 -0
  40. package/memory-storage/utils.d.ts +16 -0
  41. package/memory-storage/utils.js +41 -0
  42. package/owned-or-injected.d.ts +58 -0
  43. package/owned-or-injected.js +98 -0
  44. package/package.json +20 -20
  45. package/proxy-configuration.d.ts +118 -0
  46. package/proxy-configuration.js +141 -0
  47. package/recoverable-state.d.ts +177 -0
  48. package/recoverable-state.js +268 -0
  49. package/request.d.ts +48 -93
  50. package/request.js +112 -230
  51. package/serialization.d.ts +0 -1
  52. package/serialization.js +18 -18
  53. package/service-locator.d.ts +168 -0
  54. package/service-locator.js +255 -0
  55. package/storages/batched-adds.d.ts +37 -0
  56. package/storages/batched-adds.js +73 -0
  57. package/storages/dataset.d.ts +105 -58
  58. package/storages/dataset.js +292 -150
  59. package/storages/index.d.ts +11 -12
  60. package/storages/index.js +7 -12
  61. package/storages/key-value-store-codec.d.ts +32 -0
  62. package/storages/key-value-store-codec.js +108 -0
  63. package/storages/{key_value_store.d.ts → key-value-store.d.ts} +169 -72
  64. package/storages/key-value-store.js +767 -0
  65. package/storages/request-dedup-cache.d.ts +20 -0
  66. package/storages/request-dedup-cache.js +48 -0
  67. package/storages/{request_list.d.ts → request-list.d.ts} +33 -173
  68. package/storages/{request_list.js → request-list.js} +276 -255
  69. package/storages/request-loader.d.ts +127 -0
  70. package/storages/request-loader.js +36 -0
  71. package/storages/request-manager-tandem.d.ts +82 -0
  72. package/storages/request-manager-tandem.js +200 -0
  73. package/storages/request-manager.d.ts +119 -0
  74. package/storages/request-manager.js +1 -0
  75. package/storages/{request_provider.d.ts → request-queue.d.ts} +128 -159
  76. package/storages/request-queue.js +870 -0
  77. package/storages/storage-instance-manager.d.ts +86 -0
  78. package/storages/storage-instance-manager.js +256 -0
  79. package/storages/storage-stats.d.ts +48 -0
  80. package/storages/storage-stats.js +29 -0
  81. package/storages/transaction.d.ts +270 -0
  82. package/storages/transaction.js +296 -0
  83. package/storages/utils.d.ts +59 -11
  84. package/storages/utils.js +76 -16
  85. package/system-info/cpu-info.d.ts +67 -0
  86. package/system-info/cpu-info.js +216 -0
  87. package/system-info/memory-info.d.ts +31 -0
  88. package/system-info/memory-info.js +115 -0
  89. package/system-info/ps-tree.d.ts +17 -0
  90. package/system-info/ps-tree.js +144 -0
  91. package/system-info/runtime.d.ts +14 -0
  92. package/system-info/runtime.js +80 -0
  93. package/typedefs.d.ts +0 -6
  94. package/typedefs.js +0 -1
  95. package/url.d.ts +9 -0
  96. package/url.js +11 -0
  97. package/validators.d.ts +23 -18
  98. package/validators.js +14 -18
  99. package/autoscaling/autoscaled_pool.d.ts +0 -300
  100. package/autoscaling/autoscaled_pool.d.ts.map +0 -1
  101. package/autoscaling/autoscaled_pool.js +0 -533
  102. package/autoscaling/autoscaled_pool.js.map +0 -1
  103. package/autoscaling/index.d.ts +0 -4
  104. package/autoscaling/index.d.ts.map +0 -1
  105. package/autoscaling/index.js +0 -4
  106. package/autoscaling/index.js.map +0 -1
  107. package/autoscaling/snapshotter.d.ts +0 -189
  108. package/autoscaling/snapshotter.d.ts.map +0 -1
  109. package/autoscaling/snapshotter.js +0 -285
  110. package/autoscaling/snapshotter.js.map +0 -1
  111. package/autoscaling/system_status.d.ts +0 -181
  112. package/autoscaling/system_status.d.ts.map +0 -1
  113. package/autoscaling/system_status.js +0 -169
  114. package/autoscaling/system_status.js.map +0 -1
  115. package/configuration.d.ts.map +0 -1
  116. package/configuration.js.map +0 -1
  117. package/cookie_utils.d.ts +0 -43
  118. package/cookie_utils.d.ts.map +0 -1
  119. package/cookie_utils.js +0 -111
  120. package/cookie_utils.js.map +0 -1
  121. package/crawlers/context_pipeline.d.ts +0 -61
  122. package/crawlers/context_pipeline.d.ts.map +0 -1
  123. package/crawlers/context_pipeline.js +0 -99
  124. package/crawlers/context_pipeline.js.map +0 -1
  125. package/crawlers/crawler_commons.d.ts +0 -191
  126. package/crawlers/crawler_commons.d.ts.map +0 -1
  127. package/crawlers/crawler_commons.js +0 -108
  128. package/crawlers/crawler_commons.js.map +0 -1
  129. package/crawlers/crawler_utils.d.ts +0 -10
  130. package/crawlers/crawler_utils.d.ts.map +0 -1
  131. package/crawlers/crawler_utils.js +0 -12
  132. package/crawlers/crawler_utils.js.map +0 -1
  133. package/crawlers/error_snapshotter.d.ts +0 -60
  134. package/crawlers/error_snapshotter.d.ts.map +0 -1
  135. package/crawlers/error_snapshotter.js +0 -118
  136. package/crawlers/error_snapshotter.js.map +0 -1
  137. package/crawlers/error_tracker.d.ts +0 -55
  138. package/crawlers/error_tracker.d.ts.map +0 -1
  139. package/crawlers/error_tracker.js +0 -309
  140. package/crawlers/error_tracker.js.map +0 -1
  141. package/crawlers/index.d.ts +0 -7
  142. package/crawlers/index.d.ts.map +0 -1
  143. package/crawlers/index.js +0 -7
  144. package/crawlers/index.js.map +0 -1
  145. package/crawlers/internals/types.d.ts +0 -8
  146. package/crawlers/internals/types.d.ts.map +0 -1
  147. package/crawlers/internals/types.js +0 -2
  148. package/crawlers/internals/types.js.map +0 -1
  149. package/crawlers/statistics.d.ts +0 -213
  150. package/crawlers/statistics.d.ts.map +0 -1
  151. package/crawlers/statistics.js +0 -349
  152. package/crawlers/statistics.js.map +0 -1
  153. package/enqueue_links/enqueue_links.d.ts +0 -249
  154. package/enqueue_links/enqueue_links.d.ts.map +0 -1
  155. package/enqueue_links/enqueue_links.js +0 -269
  156. package/enqueue_links/enqueue_links.js.map +0 -1
  157. package/enqueue_links/index.d.ts +0 -3
  158. package/enqueue_links/index.d.ts.map +0 -1
  159. package/enqueue_links/index.js +0 -3
  160. package/enqueue_links/index.js.map +0 -1
  161. package/enqueue_links/shared.d.ts +0 -73
  162. package/enqueue_links/shared.d.ts.map +0 -1
  163. package/enqueue_links/shared.js +0 -212
  164. package/enqueue_links/shared.js.map +0 -1
  165. package/errors.d.ts.map +0 -1
  166. package/errors.js.map +0 -1
  167. package/events/event_manager.d.ts +0 -52
  168. package/events/event_manager.d.ts.map +0 -1
  169. package/events/event_manager.js.map +0 -1
  170. package/events/index.d.ts.map +0 -1
  171. package/events/index.js.map +0 -1
  172. package/events/local_event_manager.d.ts +0 -27
  173. package/events/local_event_manager.d.ts.map +0 -1
  174. package/events/local_event_manager.js +0 -75
  175. package/events/local_event_manager.js.map +0 -1
  176. package/index.d.ts.map +0 -1
  177. package/index.js.map +0 -1
  178. package/log.d.ts.map +0 -1
  179. package/log.js.map +0 -1
  180. package/proxy_configuration.d.ts +0 -169
  181. package/proxy_configuration.d.ts.map +0 -1
  182. package/proxy_configuration.js +0 -247
  183. package/proxy_configuration.js.map +0 -1
  184. package/recoverable_state.d.ts +0 -123
  185. package/recoverable_state.d.ts.map +0 -1
  186. package/recoverable_state.js +0 -140
  187. package/recoverable_state.js.map +0 -1
  188. package/request.d.ts.map +0 -1
  189. package/request.js.map +0 -1
  190. package/router.d.ts +0 -134
  191. package/router.d.ts.map +0 -1
  192. package/router.js +0 -164
  193. package/router.js.map +0 -1
  194. package/serialization.d.ts.map +0 -1
  195. package/serialization.js.map +0 -1
  196. package/session_pool/consts.d.ts +0 -4
  197. package/session_pool/consts.d.ts.map +0 -1
  198. package/session_pool/consts.js +0 -4
  199. package/session_pool/consts.js.map +0 -1
  200. package/session_pool/errors.d.ts +0 -8
  201. package/session_pool/errors.d.ts.map +0 -1
  202. package/session_pool/errors.js +0 -12
  203. package/session_pool/errors.js.map +0 -1
  204. package/session_pool/events.d.ts +0 -3
  205. package/session_pool/events.d.ts.map +0 -1
  206. package/session_pool/events.js +0 -3
  207. package/session_pool/events.js.map +0 -1
  208. package/session_pool/index.d.ts +0 -6
  209. package/session_pool/index.d.ts.map +0 -1
  210. package/session_pool/index.js +0 -6
  211. package/session_pool/index.js.map +0 -1
  212. package/session_pool/session.d.ts +0 -186
  213. package/session_pool/session.d.ts.map +0 -1
  214. package/session_pool/session.js +0 -278
  215. package/session_pool/session.js.map +0 -1
  216. package/session_pool/session_pool.d.ts +0 -258
  217. package/session_pool/session_pool.d.ts.map +0 -1
  218. package/session_pool/session_pool.js +0 -387
  219. package/session_pool/session_pool.js.map +0 -1
  220. package/storages/access_checking.d.ts +0 -13
  221. package/storages/access_checking.d.ts.map +0 -1
  222. package/storages/access_checking.js +0 -14
  223. package/storages/access_checking.js.map +0 -1
  224. package/storages/dataset.d.ts.map +0 -1
  225. package/storages/dataset.js.map +0 -1
  226. package/storages/index.d.ts.map +0 -1
  227. package/storages/index.js.map +0 -1
  228. package/storages/key_value_store.d.ts.map +0 -1
  229. package/storages/key_value_store.js +0 -512
  230. package/storages/key_value_store.js.map +0 -1
  231. package/storages/request_list.d.ts.map +0 -1
  232. package/storages/request_list.js.map +0 -1
  233. package/storages/request_list_adapter.d.ts +0 -58
  234. package/storages/request_list_adapter.d.ts.map +0 -1
  235. package/storages/request_list_adapter.js +0 -81
  236. package/storages/request_list_adapter.js.map +0 -1
  237. package/storages/request_manager_tandem.d.ts +0 -68
  238. package/storages/request_manager_tandem.d.ts.map +0 -1
  239. package/storages/request_manager_tandem.js +0 -124
  240. package/storages/request_manager_tandem.js.map +0 -1
  241. package/storages/request_provider.d.ts.map +0 -1
  242. package/storages/request_provider.js +0 -591
  243. package/storages/request_provider.js.map +0 -1
  244. package/storages/request_queue.d.ts +0 -128
  245. package/storages/request_queue.d.ts.map +0 -1
  246. package/storages/request_queue.js +0 -320
  247. package/storages/request_queue.js.map +0 -1
  248. package/storages/request_queue_v2.d.ts +0 -87
  249. package/storages/request_queue_v2.d.ts.map +0 -1
  250. package/storages/request_queue_v2.js +0 -438
  251. package/storages/request_queue_v2.js.map +0 -1
  252. package/storages/sitemap_request_list.d.ts +0 -250
  253. package/storages/sitemap_request_list.d.ts.map +0 -1
  254. package/storages/sitemap_request_list.js +0 -431
  255. package/storages/sitemap_request_list.js.map +0 -1
  256. package/storages/storage_manager.d.ts +0 -62
  257. package/storages/storage_manager.d.ts.map +0 -1
  258. package/storages/storage_manager.js +0 -105
  259. package/storages/storage_manager.js.map +0 -1
  260. package/storages/utils.d.ts.map +0 -1
  261. package/storages/utils.js.map +0 -1
  262. package/typedefs.d.ts.map +0 -1
  263. package/typedefs.js.map +0 -1
  264. package/validators.d.ts.map +0 -1
  265. package/validators.js.map +0 -1
@@ -1,591 +0,0 @@
1
- import { inspect } from 'node:util';
2
- import { chunkedAsyncIterable, downloadListOfUrls, getObjectType, isAsyncIterable, isIterable, peekableAsyncIterable, sleep, } from '@crawlee/utils';
3
- import ow from 'ow';
4
- import { ListDictionary, LruCache } from '@apify/datastructures';
5
- import { cryptoRandomObjectId } from '@apify/utilities';
6
- import { Configuration } from '../configuration.js';
7
- import { log } from '../log.js';
8
- import { Request } from '../request.js';
9
- import { checkStorageAccess } from './access_checking.js';
10
- import { StorageManager } from './storage_manager.js';
11
- import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './utils.js';
12
- export class RequestProvider {
13
- config;
14
- id;
15
- name;
16
- timeoutSecs = 30;
17
- clientKey = cryptoRandomObjectId();
18
- client;
19
- proxyConfiguration;
20
- log;
21
- internalTimeoutMillis = 5 * 60_000; // defaults to 5 minutes, will be overridden by BasicCrawler
22
- requestLockSecs = 3 * 60; // defaults to 3 minutes, will be overridden by BasicCrawler
23
- // We can trust these numbers only in a case that queue is used by a single client.
24
- // This information is returned by getHead() under the hadMultipleClients property.
25
- assumedTotalCount = 0;
26
- assumedHandledCount = 0;
27
- initialCount = 0;
28
- initialHandledCount = 0; // We track this separately from `assumedHandledCount` which is used non-trivially by RequestQueueV1
29
- queueHeadIds = new ListDictionary();
30
- requestCache;
31
- recentlyHandledRequestsCache;
32
- queuePausedForMigration = false;
33
- lastActivity = new Date();
34
- isFinishedCalledWhileHeadWasNotEmpty = 0;
35
- inProgressRequestBatchCount = 0;
36
- httpClient;
37
- constructor(options, config = Configuration.getGlobalConfig()) {
38
- this.config = config;
39
- this.id = options.id;
40
- this.name = options.name;
41
- this.client = options.client.requestQueue(this.id, {
42
- clientKey: this.clientKey,
43
- timeoutSecs: this.timeoutSecs,
44
- });
45
- this.proxyConfiguration = options.proxyConfiguration;
46
- this.requestCache = new LruCache({ maxLength: options.requestCacheMaxSize });
47
- this.recentlyHandledRequestsCache = new LruCache({ maxLength: options.recentlyHandledRequestsMaxSize });
48
- this.log = log.child({ prefix: `${options.logPrefix}(${this.id}, ${this.name ?? 'no-name'})` });
49
- const eventManager = config.getEventManager();
50
- eventManager.on("migrating" /* EventType.MIGRATING */, async () => {
51
- this.queuePausedForMigration = true;
52
- });
53
- }
54
- /**
55
- * Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).
56
- *
57
- * Survives restarts and actor migrations.
58
- */
59
- getTotalCount() {
60
- return this.assumedTotalCount + this.initialCount;
61
- }
62
- /**
63
- * Returns an offline approximation of the total number of pending requests in the queue.
64
- *
65
- * Survives restarts and Actor migrations.
66
- */
67
- getPendingCount() {
68
- return this.getTotalCount() - this.initialHandledCount - this.assumedHandledCount;
69
- }
70
- /**
71
- * Adds a request to the queue.
72
- *
73
- * If a request with the same `uniqueKey` property is already present in the queue,
74
- * it will not be updated. You can find out whether this happened from the resulting
75
- * {@link QueueOperationInfo} object.
76
- *
77
- * To add multiple requests to the queue by extracting links from a webpage,
78
- * see the {@link enqueueLinks} helper function.
79
- *
80
- * @param requestLike {@link Request} object or vanilla object with request data.
81
- * Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
82
- * @param [options] Request queue operation options.
83
- */
84
- async addRequest(requestLike, options = {}) {
85
- checkStorageAccess();
86
- this.lastActivity = new Date();
87
- ow(requestLike, ow.object);
88
- ow(options, ow.object.exactShape({
89
- forefront: ow.optional.boolean,
90
- }));
91
- const { forefront = false } = options;
92
- if ('requestsFromUrl' in requestLike) {
93
- const requests = await this._fetchRequestsFromUrl(requestLike);
94
- const processedRequests = await this._addFetchedRequests(requestLike, requests, options);
95
- return { ...processedRequests[0], forefront };
96
- }
97
- ow(requestLike, ow.object.partialShape({
98
- url: ow.string,
99
- id: ow.undefined,
100
- }));
101
- const request = requestLike instanceof Request ? requestLike : new Request(requestLike);
102
- const cacheKey = getRequestId(request.uniqueKey);
103
- const cachedInfo = this.requestCache.get(cacheKey);
104
- if (cachedInfo) {
105
- request.id = cachedInfo.id;
106
- return {
107
- wasAlreadyPresent: true,
108
- // We may assume that if request is in local cache then also the information if the
109
- // request was already handled is there because just one client should be using one queue.
110
- wasAlreadyHandled: cachedInfo.isHandled,
111
- requestId: cachedInfo.id,
112
- uniqueKey: cachedInfo.uniqueKey,
113
- forefront,
114
- };
115
- }
116
- const queueOperationInfo = {
117
- ...(await this.client.addRequest(request, { forefront })),
118
- uniqueKey: request.uniqueKey,
119
- forefront,
120
- };
121
- const { requestId, wasAlreadyPresent } = queueOperationInfo;
122
- this._cacheRequest(cacheKey, queueOperationInfo);
123
- if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
124
- this.assumedTotalCount++;
125
- // Performance optimization: add request straight to head if possible
126
- this._maybeAddRequestToQueueHead(requestId, forefront);
127
- }
128
- return queueOperationInfo;
129
- }
130
- /**
131
- * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
132
- * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`
133
- * if you don't want to block the processing, as those methods will only wait for the initial 1000 requests,
134
- * start processing right after that happens, and continue adding more in the background.
135
- *
136
- * If a request passed in is already present due to its `uniqueKey` property being the same,
137
- * it will not be updated. You can find out whether this happened by finding the request in the resulting
138
- * {@link BatchAddRequestsResult} object.
139
- *
140
- * @param requestsLike {@link Request} objects or vanilla objects with request data.
141
- * Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
142
- * @param [options] Request queue operation options.
143
- */
144
- async addRequests(requestsLike, options = {}) {
145
- checkStorageAccess();
146
- this.lastActivity = new Date();
147
- ow(requestsLike, ow.object
148
- .is((value) => isIterable(value) || isAsyncIterable(value))
149
- .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
150
- ow(options, ow.object.exactShape({
151
- forefront: ow.optional.boolean,
152
- cache: ow.optional.boolean,
153
- }));
154
- const { forefront = false, cache = true } = options;
155
- const uniqueKeyToCacheKey = new Map();
156
- const getCachedRequestId = (uniqueKey) => {
157
- const cached = uniqueKeyToCacheKey.get(uniqueKey);
158
- if (cached)
159
- return cached;
160
- const newCacheKey = getRequestId(uniqueKey);
161
- uniqueKeyToCacheKey.set(uniqueKey, newCacheKey);
162
- return newCacheKey;
163
- };
164
- const results = {
165
- processedRequests: [],
166
- unprocessedRequests: [],
167
- };
168
- const requests = [];
169
- for await (const requestLike of requestsLike) {
170
- if (typeof requestLike === 'string') {
171
- requests.push(new Request({ url: requestLike }));
172
- }
173
- else if ('requestsFromUrl' in requestLike) {
174
- const fetchedRequests = await this._fetchRequestsFromUrl(requestLike);
175
- await this._addFetchedRequests(requestLike, fetchedRequests, options);
176
- }
177
- else {
178
- requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike));
179
- }
180
- }
181
- const requestsToAdd = new Map();
182
- for (const request of requests) {
183
- const cacheKey = getCachedRequestId(request.uniqueKey);
184
- const cachedInfo = this.requestCache.get(cacheKey);
185
- if (cachedInfo) {
186
- request.id = cachedInfo.id;
187
- results.processedRequests.push({
188
- wasAlreadyPresent: true,
189
- // We may assume that if request is in local cache then also the information if the
190
- // request was already handled is there because just one client should be using one queue.
191
- wasAlreadyHandled: cachedInfo.isHandled,
192
- requestId: cachedInfo.id,
193
- uniqueKey: cachedInfo.uniqueKey,
194
- });
195
- }
196
- else if (!requestsToAdd.has(request.uniqueKey)) {
197
- requestsToAdd.set(request.uniqueKey, request);
198
- }
199
- }
200
- // Early exit if all provided requests were already added
201
- if (!requestsToAdd.size) {
202
- return results;
203
- }
204
- const apiResults = await this.client.batchAddRequests([...requestsToAdd.values()], { forefront });
205
- // Report unprocessed requests
206
- results.unprocessedRequests = apiResults.unprocessedRequests;
207
- // Add all new requests to the requestCache
208
- for (const newRequest of apiResults.processedRequests) {
209
- // Add the new request to the processed list
210
- results.processedRequests.push(newRequest);
211
- const cacheKey = getCachedRequestId(newRequest.uniqueKey);
212
- const { requestId, wasAlreadyPresent } = newRequest;
213
- if (cache) {
214
- this._cacheRequest(cacheKey, { ...newRequest, forefront });
215
- }
216
- if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
217
- this.assumedTotalCount++;
218
- // Performance optimization: add request straight to head if possible
219
- this._maybeAddRequestToQueueHead(requestId, forefront);
220
- }
221
- }
222
- return results;
223
- }
224
- /**
225
- * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
226
- * adding the rest in the background. You can configure the batch size via `batchSize` option and the sleep time in between
227
- * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
228
- * the `waitForAllRequestsToBeAdded` promise you get in the response object.
229
- *
230
- * @param requests The requests to add
231
- * @param options Options for the request queue
232
- */
233
- async addRequestsBatched(requests, options = {}) {
234
- checkStorageAccess();
235
- this.lastActivity = new Date();
236
- ow(requests, ow.object
237
- .is((value) => isIterable(value) || isAsyncIterable(value))
238
- .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
239
- ow(options, ow.object.exactShape({
240
- forefront: ow.optional.boolean,
241
- waitForAllRequestsToBeAdded: ow.optional.boolean,
242
- batchSize: ow.optional.number,
243
- waitBetweenBatchesMillis: ow.optional.number,
244
- }));
245
- const addRequest = this.addRequest.bind(this);
246
- async function* generateRequests() {
247
- for await (const opts of requests) {
248
- // Validate the input
249
- if (typeof opts === 'object' && opts !== null) {
250
- if (opts.url !== undefined && typeof opts.url !== 'string') {
251
- throw new Error(`Request options are not valid, the 'url' property is not a string. Input: ${inspect(opts)}`);
252
- }
253
- if (opts.id !== undefined) {
254
- throw new Error(`Request options are not valid, the 'id' property must not be present. Input: ${inspect(opts)}`);
255
- }
256
- if (opts.requestsFromUrl !== undefined &&
257
- typeof opts.requestsFromUrl !== 'string') {
258
- throw new Error(`Request options are not valid, the 'requestsFromUrl' property is not a string. Input: ${inspect(opts)}`);
259
- }
260
- }
261
- if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
262
- // Handle URL lists right away
263
- await addRequest(opts, { forefront: options.forefront });
264
- }
265
- else {
266
- // Yield valid requests
267
- yield typeof opts === 'string' ? { url: opts } : opts;
268
- }
269
- }
270
- }
271
- const { batchSize = 1000, waitBetweenBatchesMillis = 1000 } = options;
272
- const chunks = peekableAsyncIterable(chunkedAsyncIterable(generateRequests(), batchSize));
273
- const chunksIterator = chunks[Symbol.asyncIterator]();
274
- const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true) => {
275
- const resultsToReturn = [];
276
- const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
277
- resultsToReturn.push(...apiResult.processedRequests);
278
- if (apiResult.unprocessedRequests.length) {
279
- await sleep(waitBetweenBatchesMillis);
280
- resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false)));
281
- }
282
- return resultsToReturn;
283
- };
284
- // Add initial batch of `batchSize` to process them right away
285
- const initialChunk = await chunksIterator.peek();
286
- if (initialChunk === undefined) {
287
- return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]) };
288
- }
289
- const addedRequests = await attemptToAddToQueueAndAddAnyUnprocessed(initialChunk);
290
- await chunksIterator.next();
291
- // If we have no more requests to add, return immediately
292
- if ((await chunksIterator.peek()) === undefined) {
293
- return {
294
- addedRequests,
295
- waitForAllRequestsToBeAdded: Promise.resolve([]),
296
- };
297
- }
298
- // eslint-disable-next-line no-async-promise-executor
299
- const promise = new Promise(async (resolve) => {
300
- const finalAddedRequests = [];
301
- for await (const requestChunk of chunks) {
302
- finalAddedRequests.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(requestChunk, false)));
303
- await sleep(waitBetweenBatchesMillis);
304
- }
305
- resolve(finalAddedRequests);
306
- });
307
- this.inProgressRequestBatchCount += 1;
308
- void promise.finally(() => {
309
- this.inProgressRequestBatchCount -= 1;
310
- });
311
- // If the user wants to wait for all the requests to be added, we wait for the promise to resolve for them
312
- if (options.waitForAllRequestsToBeAdded) {
313
- addedRequests.push(...(await promise));
314
- }
315
- return {
316
- addedRequests,
317
- waitForAllRequestsToBeAdded: promise,
318
- };
319
- }
320
- /**
321
- * Gets the request from the queue specified by ID.
322
- *
323
- * @param id ID of the request.
324
- * @returns Returns the request object, or `null` if it was not found.
325
- */
326
- async getRequest(id) {
327
- checkStorageAccess();
328
- ow(id, ow.string);
329
- const requestOptions = await this.client.getRequest(id);
330
- if (!requestOptions)
331
- return null;
332
- return new Request(requestOptions);
333
- }
334
- /**
335
- * Marks a request that was previously returned by the
336
- * {@link RequestQueue.fetchNextRequest}
337
- * function as handled after successful processing.
338
- * Handled requests will never again be returned by the `fetchNextRequest` function.
339
- */
340
- async markRequestHandled(request) {
341
- checkStorageAccess();
342
- this.lastActivity = new Date();
343
- ow(request, ow.object.partialShape({
344
- id: ow.string,
345
- uniqueKey: ow.string,
346
- handledAt: ow.optional.string,
347
- }));
348
- const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
349
- const handledAt = request.handledAt ?? new Date().toISOString();
350
- const queueOperationInfo = {
351
- ...(await this.client.updateRequest({
352
- ...request,
353
- handledAt,
354
- })),
355
- uniqueKey: request.uniqueKey,
356
- forefront,
357
- };
358
- request.handledAt = handledAt;
359
- this.recentlyHandledRequestsCache.add(request.id, true);
360
- if (!queueOperationInfo.wasAlreadyHandled) {
361
- this.assumedHandledCount++;
362
- }
363
- this.queueHeadIds.remove(request.id);
364
- this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
365
- return queueOperationInfo;
366
- }
367
- /**
368
- * Reclaims a failed request back to the queue, so that it can be returned for processing later again
369
- * by another call to {@link RequestQueue.fetchNextRequest}.
370
- * The request record in the queue is updated using the provided `request` parameter.
371
- * For example, this lets you store the number of retries or error messages for the request.
372
- */
373
- async reclaimRequest(request, options = {}) {
374
- checkStorageAccess();
375
- this.lastActivity = new Date();
376
- ow(request, ow.object.partialShape({
377
- id: ow.string,
378
- uniqueKey: ow.string,
379
- }));
380
- ow(options, ow.object.exactShape({
381
- forefront: ow.optional.boolean,
382
- }));
383
- const { forefront = false } = options;
384
- // TODO: If request hasn't been changed since the last getRequest(),
385
- // we don't need to call updateRequest() and thus improve performance.
386
- const queueOperationInfo = {
387
- ...(await this.client.updateRequest(request, {
388
- forefront,
389
- })),
390
- uniqueKey: request.uniqueKey,
391
- forefront,
392
- };
393
- this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
394
- return queueOperationInfo;
395
- }
396
- /**
397
- * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest}
398
- * would return `null`, otherwise it resolves to `false`.
399
- * Note that even if the queue is empty, there might be some pending requests currently being processed.
400
- * If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
401
- */
402
- async isEmpty() {
403
- await this.ensureHeadIsNonEmpty();
404
- return this.queueHeadIds.length() === 0;
405
- }
406
- _reset() {
407
- this.lastActivity = new Date();
408
- this.queueHeadIds.clear();
409
- this.recentlyHandledRequestsCache.clear();
410
- this.assumedTotalCount = 0;
411
- this.assumedHandledCount = 0;
412
- this.requestCache.clear();
413
- }
414
- /**
415
- * Caches information about request to beware of unneeded addRequest() calls.
416
- */
417
- _cacheRequest(cacheKey, queueOperationInfo) {
418
- // Remove the previous entry, as otherwise our cache will never update 👀
419
- this.requestCache.remove(cacheKey);
420
- this.requestCache.add(cacheKey, {
421
- id: queueOperationInfo.requestId,
422
- isHandled: queueOperationInfo.wasAlreadyHandled,
423
- uniqueKey: queueOperationInfo.uniqueKey,
424
- hydrated: null,
425
- lockExpiresAt: null,
426
- forefront: queueOperationInfo.forefront,
427
- });
428
- }
429
- /**
430
- * Adds a request straight to the queueHeadDict, to improve performance.
431
- */
432
- _maybeAddRequestToQueueHead(requestId, forefront) {
433
- if (forefront) {
434
- this.queueHeadIds.add(requestId, requestId, true);
435
- }
436
- else if (this.assumedTotalCount < QUERY_HEAD_MIN_LENGTH) {
437
- this.queueHeadIds.add(requestId, requestId, false);
438
- }
439
- }
440
- /**
441
- * Removes the queue either from the Apify Cloud storage or from the local database,
442
- * depending on the mode of operation.
443
- */
444
- async drop() {
445
- checkStorageAccess();
446
- await this.client.delete();
447
- const manager = StorageManager.getManager(this.constructor, this.config);
448
- manager.closeStorage(this);
449
- }
450
- /**
451
- * @inheritdoc
452
- */
453
- async *[Symbol.asyncIterator]() {
454
- while (true) {
455
- const req = await this.fetchNextRequest();
456
- if (!req)
457
- break;
458
- yield req;
459
- }
460
- }
461
- /**
462
- * Returns the number of handled requests.
463
- *
464
- * This function is just a convenient shortcut for:
465
- *
466
- * ```javascript
467
- * const { handledRequestCount } = await queue.getInfo();
468
- * ```
469
- * @inheritdoc
470
- */
471
- async handledCount() {
472
- // NOTE: We keep this function for compatibility with RequestList.handledCount()
473
- const { handledRequestCount } = (await this.getInfo()) ?? {};
474
- return handledRequestCount ?? 0;
475
- }
476
- /**
477
- * Returns an object containing general information about the request queue.
478
- *
479
- * The function returns the same object as the Apify API Client's
480
- * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
481
- * function, which in turn calls the
482
- * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
483
- * API endpoint.
484
- *
485
- * **Example:**
486
- * ```
487
- * {
488
- * id: "WkzbQMuFYuamGv3YF",
489
- * name: "my-queue",
490
- * userId: "wRsJZtadYvn4mBZmm",
491
- * createdAt: new Date("2015-12-12T07:34:14.202Z"),
492
- * modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
493
- * accessedAt: new Date("2015-12-14T08:36:13.202Z"),
494
- * totalRequestCount: 25,
495
- * handledRequestCount: 5,
496
- * pendingRequestCount: 20,
497
- * }
498
- * ```
499
- */
500
- async getInfo() {
501
- checkStorageAccess();
502
- return this.client.get();
503
- }
504
- /**
505
- * Fetches URLs from requestsFromUrl and returns them in format of list of requests
506
- */
507
- async _fetchRequestsFromUrl(source) {
508
- const { requestsFromUrl, regex, ...sharedOpts } = source;
509
- // Download remote resource and parse URLs.
510
- let urlsArr;
511
- try {
512
- urlsArr = await this._downloadListOfUrls({
513
- url: requestsFromUrl,
514
- urlRegExp: regex,
515
- proxyUrl: await this.proxyConfiguration?.newUrl(),
516
- });
517
- }
518
- catch (err) {
519
- throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);
520
- }
521
- // Skip if resource contained no URLs.
522
- if (!urlsArr.length) {
523
- this.log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex });
524
- return [];
525
- }
526
- return urlsArr.map((url) => ({ url, ...sharedOpts }));
527
- }
528
- /**
529
- * Adds all fetched requests from a URL from a remote resource.
530
- */
531
- async _addFetchedRequests(source, fetchedRequests, options) {
532
- const { requestsFromUrl, regex } = source;
533
- const { addedRequests } = await this.addRequestsBatched(fetchedRequests, options);
534
- this.log.info('Fetched and loaded Requests from a remote resource.', {
535
- requestsFromUrl,
536
- regex,
537
- fetchedCount: fetchedRequests.length,
538
- importedCount: addedRequests.length,
539
- duplicateCount: fetchedRequests.length - addedRequests.length,
540
- sample: JSON.stringify(fetchedRequests.slice(0, 5)),
541
- });
542
- return addedRequests;
543
- }
544
- /**
545
- * @internal wraps public utility for mocking purposes
546
- */
547
- async _downloadListOfUrls(options) {
548
- return downloadListOfUrls({
549
- ...options,
550
- httpClient: this.httpClient,
551
- });
552
- }
553
- /**
554
- * Opens a request queue and returns a promise resolving to an instance
555
- * of the {@link RequestQueue} class.
556
- *
557
- * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
558
- * The queue is used for deep crawling of websites, where you start with several URLs and then
559
- * recursively follow links to other pages. The data structure supports both breadth-first
560
- * and depth-first crawling orders.
561
- *
562
- * For more details and code examples, see the {@link RequestQueue} class.
563
- *
564
- * @param [queueIdOrName]
565
- * ID or name of the request queue to be opened. If `null` or `undefined`,
566
- * the function returns the default request queue associated with the crawler run.
567
- * @param [options] Open Request Queue options.
568
- */
569
- static async open(queueIdOrName, options = {}) {
570
- checkStorageAccess();
571
- ow(queueIdOrName, ow.optional.any(ow.string, ow.null));
572
- ow(options, ow.object.exactShape({
573
- config: ow.optional.object.instanceOf(Configuration),
574
- storageClient: ow.optional.object,
575
- proxyConfiguration: ow.optional.object,
576
- httpClient: ow.optional.object,
577
- }));
578
- options.config ??= Configuration.getGlobalConfig();
579
- options.storageClient ??= options.config.getStorageClient();
580
- await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
581
- const manager = StorageManager.getManager(this, options.config);
582
- const queue = await manager.openStorage(queueIdOrName, options.storageClient);
583
- queue.proxyConfiguration = options.proxyConfiguration;
584
- const queueInfo = await queue.client.get();
585
- queue.initialCount = queueInfo?.totalRequestCount ?? 0;
586
- queue.initialHandledCount = queueInfo?.handledRequestCount ?? 0;
587
- queue.httpClient = options.httpClient;
588
- return queue;
589
- }
590
- }
591
- //# sourceMappingURL=request_provider.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"request_provider.js","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,OAAO,EACH,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,UAAU,EACV,qBAAqB,EACrB,KAAK,GACR,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAiEvF,MAAM,OAAgB,eAAe;IAqCpB;IApCb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,WAAW,GAAG,EAAE,CAAC;IACjB,SAAS,GAAG,oBAAoB,EAAE,CAAC;IACnC,MAAM,CAAqB;IACjB,kBAAkB,CAAsB;IAElD,GAAG,CAAM;IACT,qBAAqB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,4DAA4D;IAChG,eAAe,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,4DAA4D;IAEtF,mFAAmF;IACnF,mFAAmF;IACnF,iBAAiB,GAAG,CAAC,CAAC;IACtB,mBAAmB,GAAG,CAAC,CAAC;IAEhB,YAAY,GAAG,CAAC,CAAC;IACjB,mBAAmB,GAAG,CAAC,CAAC,CAAC,oGAAoG;IAE3H,YAAY,GAAG,IAAI,cAAc,EAAU,CAAC;IAC5C,YAAY,CAA2B;IAEvC,4BAA4B,CAAoB;IAEhD,uBAAuB,GAAG,KAAK,CAAC;IAEhC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;IAE1B,oCAAoC,GAAG,CAAC,CAAC;IAEzC,2BAA2B,GAAG,CAAC,CAAC;IAEhC,UAAU,CAAkB;IAEtC,YACI,OAAuC,EAC9B,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC,4BAA4B,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,8BAA8B,EAAE,CAAC,CAAC;QACxG,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,IAAI,SAAS,GAAG,EAAE,CAAC,CAAC;QAEhG,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAE9C,YAAY,CAAC,EAAE,wCAAsB,KAAK,IAAI,EAAE;YAC5C,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,aAAa;QACT,OAAO,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,eAAe;QACX,OAAO,IAAI,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;IACtF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,UAAU,CACZ,WAAmB,EACnB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;YACjF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE3G,OAAO,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QAClD,CAAC;QAED,EAAE,CACE,WAAW,EACX,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,GAAG,EAAE,EAAE,CAAC,MAAM;YACd,EAAE,EAAE,EAAE,CAAC,SAAS;SACnB,CAAC,CACL,CAAC;QAEF,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;QAExF,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,UAAU,EAAE,CAAC;YACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,IAAI;gBACvB,mFAAmF;gBACnF,0FAA0F;gBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;gBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,SAAS;aACZ,CAAC;QACN,CAAC;QAED,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;YACzD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QAEtC,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,kBAAkB,CAAC;QAC5D,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAEjD,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEzB,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,WAAW,CACb,YAA0B,EAC1B,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,YAAY,EACZ,EAAE,CAAC,MAAM;aACJ,EAAE,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;aACnE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,+CAA+C,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CACjG,CAAC;QACF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtD,MAAM,kBAAkB,GAAG,CAAC,SAAiB,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAElD,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;YAE1B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YAC5C,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAEhD,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,MAAM,OAAO,GAA2B;YACpC,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QAEF,MAAM,QAAQ,GAA0B,EAAE,CAAC;QAE3C,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YAC3C,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;gBAClC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;gBAC1C,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;gBACxF,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YAC5F,CAAC;iBAAM,CAAC;gBACJ,QAAQ,CAAC,IAAI,CACT,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAA6B,CAAC,CAC5F,CAAC;YACN,CAAC;QACL,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEnD,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC3B,iBAAiB,EAAE,IAAI;oBACvB,mFAAmF;oBACnF,0FAA0F;oBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;oBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;oBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;iBAClC,CAAC,CAAC;YACP,CAAC;iBAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;QACL,CAAC;QAED,yDAAyD;QACzD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAElG,8BAA8B;QAC9B,OAAO,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;QAE7D,2CAA2C;QAC3C,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,iBAAiB,EAAE,CAAC;YACpD,4CAA4C;YAC5C,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE1D,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC;YAEpD,IAAI,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/D,CAAC;YAED,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAEzB,qEAAqE;gBACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAAoC,EACpC,UAAqC,EAAE;QAEvC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,EAAE,CACE,QAAQ,EACR,EAAE,CAAC,MAAM;aACJ,EAAE,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;aACnE,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,+CAA+C,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CACjG,CAAC;QAEF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAChD,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC/C,CAAC,CACL,CAAC;QAEF,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,KAAK,SAAS,CAAC,CAAC,gBAAgB;YAC5B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAChC,qBAAqB;gBACrB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;wBACzD,MAAM,IAAI,KAAK,CACX,6EAA6E,OAAO,CAAC,IAAI,CAAC,EAAE,CAC/F,CAAC;oBACN,CAAC;oBAED,IAAI,IAAI,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;wBACxB,MAAM,IAAI,KAAK,CACX,gFAAgF,OAAO,CAAC,IAAI,CAAC,EAAE,CAClG,CAAC;oBACN,CAAC;oBAED,IACK,IAAY,CAAC,eAAe,KAAK,SAAS;wBAC3C,OAAQ,IAAY,CAAC,eAAe,KAAK,QAAQ,EACnD,CAAC;wBACC,MAAM,IAAI,KAAK,CACX,yFAAyF,OAAO,CAAC,IAAI,CAAC,EAAE,CAC3G,CAAC;oBACN,CAAC;gBACL,CAAC;gBAED,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;oBAChE,8BAA8B;oBAC9B,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACJ,uBAAuB;oBACvB,MAAM,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,IAAuB,CAAC;gBAC9E,CAAC;YACL,CAAC;QACL,CAAC;QAED,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,wBAAwB,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEtE,MAAM,MAAM,GAAG,qBAAqB,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;QAC1F,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAEtD,MAAM,uCAAuC,GAAG,KAAK,EAAE,gBAA0B,EAAE,KAAK,GAAG,IAAI,EAAE,EAAE;YAC/F,MAAM,eAAe,GAAuB,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACpG,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAErD,IAAI,SAAS,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC;gBACvC,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAEtC,eAAe,CAAC,IAAI,CAChB,GAAG,CAAC,MAAM,uCAAuC,CAC7C,gBAAgB,CAAC,MAAM,CACnB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC,CACjF,EACD,KAAK,CACR,CAAC,CACL,CAAC;YACN,CAAC;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,8DAA8D;QAC9D,MAAM,YAAY,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QACjD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;QACnF,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE5B,yDAAyD;QACzD,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,SAAS,EAAE,CAAC;YAC9C,OAAO;gBACH,aAAa;gBACb,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;QACN,CAAC;QAED,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAqB,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9D,MAAM,kBAAkB,GAAuB,EAAE,CAAC;YAElD,IAAI,KAAK,EAAE,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;gBACtC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,uCAAuC,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBAEjG,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC1C,CAAC;YAED,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QACtC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;YACtB,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,0GAA0G;QAC1G,IAAI,OAAO,CAAC,2BAA2B,EAAE,CAAC;YACtC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO;YACH,aAAa;YACb,2BAA2B,EAAE,OAAO;SACvC,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAoC,EAAU;QAC1D,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,OAAO,IAAI,OAAO,CAAC,cAA2C,CAAC,CAAC;IACpE,CAAC;IAqBD;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;YACpB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAChC,CAAC,CACL,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;QAE7F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAChE,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;gBAChC,GAAG,OAAO;gBACV,SAAS;aACZ,CAAC,CAAC;YACH,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAExD,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE,CAAC;YACxC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAErC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;SACvB,CAAC,CACL,CAAC;QACF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,oEAAoE;QACpE,wEAAwE;QACxE,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE;gBACzC,SAAS;aACZ,CAAC,CAAC;YACH,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAID;;;;;OAKG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC5C,CAAC;IAUS,MAAM;QACZ,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,4BAA4B,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,QAAgB,EAAE,kBAA6C;QACnF,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC5B,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,kBAAkB,CAAC,SAAS;SAC1C,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACO,2BAA2B,CAAC,SAAiB,EAAE,SAAkB;QACvE,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACtD,CAAC;aAAM,IAAI,IAAI,CAAC,iBAAiB,GAAG,qBAAqB,EAAE,CAAC;YACxD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,WAAoC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,YAAY;QACd,gFAAgF;QAChF,MAAM,EAAE,mBAAmB,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7D,OAAO,mBAAmB,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBACrC,GAAG,EAAE,eAAe;gBACpB,SAAS,EAAE,KAAK;gBAChB,QAAQ,EAAE,MAAM,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE;aACpD,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0CAA0C,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YACzF,OAAO,EAAE,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAC/B,MAAsB,EACtB,eAAiC,EACjC,OAAqC;QAErC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QAElF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY,EAAE,eAAe,CAAC,MAAM;YACpC,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,cAAc,EAAE,eAAe,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;YAC7D,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAIjC;QACG,OAAO,kBAAkB,CAAC;YACtB,GAAG,OAAO;YACV,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACjC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAmC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/F,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9E,KAAK,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAEtD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAE3C,KAAK,CAAC,YAAY,GAAG,SAAS,EAAE,iBAAiB,IAAI,CAAC,CAAC;QACvD,KAAK,CAAC,mBAAmB,GAAG,SAAS,EAAE,mBAAmB,IAAI,CAAC,CAAC;QAChE,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAEtC,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}