@crawlee/core 4.0.0-beta.64 → 4.0.0-beta.66

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/crawlers/crawler_commons.d.ts +3 -3
  2. package/crawlers/crawler_commons.d.ts.map +1 -1
  3. package/enqueue_links/enqueue_links.d.ts +7 -6
  4. package/enqueue_links/enqueue_links.d.ts.map +1 -1
  5. package/enqueue_links/enqueue_links.js +4 -4
  6. package/enqueue_links/enqueue_links.js.map +1 -1
  7. package/package.json +5 -5
  8. package/storages/index.d.ts +4 -6
  9. package/storages/index.d.ts.map +1 -1
  10. package/storages/index.js +2 -6
  11. package/storages/index.js.map +1 -1
  12. package/storages/request_list.d.ts +23 -72
  13. package/storages/request_list.d.ts.map +1 -1
  14. package/storages/request_list.js +34 -29
  15. package/storages/request_list.js.map +1 -1
  16. package/storages/request_loader.d.ts +97 -0
  17. package/storages/request_loader.d.ts.map +1 -0
  18. package/storages/request_loader.js +2 -0
  19. package/storages/request_loader.js.map +1 -0
  20. package/storages/request_manager.d.ts +34 -0
  21. package/storages/request_manager.d.ts.map +1 -0
  22. package/storages/request_manager.js +2 -0
  23. package/storages/request_manager.js.map +1 -0
  24. package/storages/request_manager_tandem.d.ts +56 -17
  25. package/storages/request_manager_tandem.d.ts.map +1 -1
  26. package/storages/request_manager_tandem.js +114 -41
  27. package/storages/request_manager_tandem.js.map +1 -1
  28. package/storages/request_queue.d.ts +276 -44
  29. package/storages/request_queue.d.ts.map +1 -1
  30. package/storages/request_queue.js +576 -212
  31. package/storages/request_queue.js.map +1 -1
  32. package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +24 -19
  33. package/storages/sitemap_request_loader.d.ts.map +1 -0
  34. package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +41 -40
  35. package/storages/sitemap_request_loader.js.map +1 -0
  36. package/validators.d.ts +4 -0
  37. package/validators.d.ts.map +1 -1
  38. package/validators.js +4 -0
  39. package/validators.js.map +1 -1
  40. package/storages/request_list_adapter.d.ts +0 -58
  41. package/storages/request_list_adapter.d.ts.map +0 -1
  42. package/storages/request_list_adapter.js +0 -81
  43. package/storages/request_list_adapter.js.map +0 -1
  44. package/storages/request_provider.d.ts +0 -384
  45. package/storages/request_provider.d.ts.map +0 -1
  46. package/storages/request_provider.js +0 -624
  47. package/storages/request_provider.js.map +0 -1
  48. package/storages/request_queue_v2.d.ts +0 -87
  49. package/storages/request_queue_v2.d.ts.map +0 -1
  50. package/storages/request_queue_v2.js +0 -437
  51. package/storages/request_queue_v2.js.map +0 -1
  52. package/storages/sitemap_request_list.d.ts.map +0 -1
  53. package/storages/sitemap_request_list.js.map +0 -1
@@ -1,87 +0,0 @@
1
- import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
2
- import type { Configuration } from '../configuration.js';
3
- import type { Request, Source } from '../request.js';
4
- import type { RequestProviderOptions, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike } from './request_provider.js';
5
- import { RequestProvider } from './request_provider.js';
6
- /**
7
- * Represents a queue of URLs to crawl, which is used for deep crawling of websites
8
- * where you start with several URLs and then recursively
9
- * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
10
- *
11
- * Each URL is represented using an instance of the {@link Request} class.
12
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
13
- * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
14
- * To add a single URL multiple times to the queue,
15
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
16
- *
17
- * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
18
- *
19
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
20
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
21
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
22
- * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
23
- *
24
- * **Example usage:**
25
- *
26
- * ```javascript
27
- * // Open the default request queue associated with the crawler run
28
- * const queue = await RequestQueue.open();
29
- *
30
- * // Open a named request queue
31
- * const queueWithName = await RequestQueue.open('some-name');
32
- *
33
- * // Enqueue few requests
34
- * await queue.addRequest({ url: 'http://example.com/aaa' });
35
- * await queue.addRequest({ url: 'http://example.com/bbb' });
36
- * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
37
- * ```
38
- * @category Sources
39
- */
40
- export declare class RequestQueue extends RequestProvider {
41
- private listHeadAndLockPromise;
42
- private queueHasLockedRequests;
43
- private shouldCheckForForefrontRequests;
44
- private dequeuedRequestCount;
45
- constructor(options: RequestProviderOptions, config?: Configuration);
46
- /**
47
- * Caches information about request to beware of unneeded addRequest() calls.
48
- */
49
- protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
50
- /**
51
- * @inheritDoc
52
- */
53
- addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
54
- /**
55
- * @inheritDoc
56
- */
57
- addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
58
- /**
59
- * @inheritDoc
60
- */
61
- fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
62
- /**
63
- * @inheritDoc
64
- */
65
- markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
66
- /**
67
- * @inheritDoc
68
- */
69
- isFinished(): Promise<boolean>;
70
- /**
71
- * @inheritDoc
72
- */
73
- reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>): ReturnType<RequestProvider['reclaimRequest']>;
74
- protected ensureHeadIsNonEmpty(): Promise<void>;
75
- private giveUpLock;
76
- private _listHeadAndLock;
77
- private getOrHydrateRequest;
78
- private _prolongRequestLock;
79
- protected _reset(): void;
80
- protected _maybeAddRequestToQueueHead(): void;
81
- protected _clearPossibleLocks(): Promise<void>;
82
- /**
83
- * @inheritDoc
84
- */
85
- static open(...args: Parameters<typeof RequestProvider.open>): Promise<RequestQueue>;
86
- }
87
- //# sourceMappingURL=request_queue_v2.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"request_queue_v2.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAGrD,OAAO,KAAK,EACR,sBAAsB,EACtB,yBAAyB,EACzB,4BAA4B,EAC5B,YAAY,EACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAiBxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,YAAa,SAAQ,eAAe;IAC7C,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,OAAO,CAAC,sBAAsB,CAAkC;IAChE,OAAO,CAAC,+BAA+B,CAAS;IAChD,OAAO,CAAC,oBAAoB,CAAK;gBAErB,OAAO,EAAE,sBAAsB,EAAE,MAAM,GAAE,aAAiD;IAoBtG;;OAEG;cACgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAevG;;OAEG;IACY,UAAU,CACrB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAQrC;;OAEG;IACY,WAAW,CACtB,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAWlC;;OAEG;IACY,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAiDhG;;OAEG;IACY,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAK9F;;OAEG;IACY,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IA+E7C;;OAEG;IACY,cAAc,CACzB,GAAG,IAAI,EAAE,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC,GACvD,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;cAqBhC,oBAAoB;YAqBtB,UAAU;YAYV,gBAAgB;YAmFhB,mBAAmB;YAuFnB,mBAAmB;cAiBd,MAAM;cAMN,2BAA2B;cAI9B,mBAAmB;IAcnC;;OAEG;WACmB,IAAI,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;CAGtG"}
@@ -1,437 +0,0 @@
1
- import { serviceLocator } from '../service_locator.js';
2
- import { checkStorageAccess } from './access_checking.js';
3
- import { RequestProvider } from './request_provider.js';
4
- import { getRequestId } from './utils.js';
5
- // Double the limit of RequestQueue v1 (1_000_000) as we also store keyed by request.id, not just from uniqueKey
6
- const MAX_CACHED_REQUESTS = 2_000_000;
7
- /**
8
- * This number must be large enough so that processing of all these requests cannot be done in
9
- * a time lower than expected maximum latency of DynamoDB, but low enough not to waste too much memory.
10
- * @internal
11
- */
12
- const RECENTLY_HANDLED_CACHE_SIZE = 1000;
13
- const LIST_AND_LOCK_HEAD_LIMIT = 25;
14
- const QUEUE_HEAD_REFILL_THRESHOLD = 1;
15
- /**
16
- * Represents a queue of URLs to crawl, which is used for deep crawling of websites
17
- * where you start with several URLs and then recursively
18
- * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
19
- *
20
- * Each URL is represented using an instance of the {@link Request} class.
21
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
22
- * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
23
- * To add a single URL multiple times to the queue,
24
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
25
- *
26
- * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
27
- *
28
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
29
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
30
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
31
- * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
32
- *
33
- * **Example usage:**
34
- *
35
- * ```javascript
36
- * // Open the default request queue associated with the crawler run
37
- * const queue = await RequestQueue.open();
38
- *
39
- * // Open a named request queue
40
- * const queueWithName = await RequestQueue.open('some-name');
41
- *
42
- * // Enqueue few requests
43
- * await queue.addRequest({ url: 'http://example.com/aaa' });
44
- * await queue.addRequest({ url: 'http://example.com/bbb' });
45
- * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
46
- * ```
47
- * @category Sources
48
- */
49
- export class RequestQueue extends RequestProvider {
50
- listHeadAndLockPromise = null;
51
- queueHasLockedRequests = undefined;
52
- shouldCheckForForefrontRequests = false;
53
- dequeuedRequestCount = 0;
54
- constructor(options, config = serviceLocator.getConfiguration()) {
55
- super({
56
- ...options,
57
- logPrefix: 'RequestQueue2',
58
- recentlyHandledRequestsMaxSize: RECENTLY_HANDLED_CACHE_SIZE,
59
- requestCacheMaxSize: MAX_CACHED_REQUESTS,
60
- }, config);
61
- this.events.on("migrating" /* EventType.MIGRATING */, async () => {
62
- await this._clearPossibleLocks();
63
- });
64
- this.events.on("aborting" /* EventType.ABORTING */, async () => {
65
- await this._clearPossibleLocks();
66
- });
67
- }
68
- /**
69
- * Caches information about request to beware of unneeded addRequest() calls.
70
- */
71
- _cacheRequest(cacheKey, queueOperationInfo) {
72
- super._cacheRequest(cacheKey, queueOperationInfo);
73
- this.requestCache.remove(queueOperationInfo.requestId);
74
- this.requestCache.add(queueOperationInfo.requestId, {
75
- id: queueOperationInfo.requestId,
76
- isHandled: queueOperationInfo.wasAlreadyHandled,
77
- uniqueKey: queueOperationInfo.uniqueKey,
78
- forefront: queueOperationInfo.forefront,
79
- hydrated: null,
80
- lockExpiresAt: null,
81
- });
82
- }
83
- /**
84
- * @inheritDoc
85
- */
86
- async addRequest(requestLike, options = {}) {
87
- const result = await super.addRequest(requestLike, options);
88
- if (!result.wasAlreadyPresent && options.forefront) {
89
- this.shouldCheckForForefrontRequests = true;
90
- }
91
- return result;
92
- }
93
- /**
94
- * @inheritDoc
95
- */
96
- async addRequests(requestsLike, options = {}) {
97
- const result = await super.addRequests(requestsLike, options);
98
- for (const request of result.processedRequests) {
99
- if (!request.wasAlreadyPresent && options.forefront) {
100
- this.shouldCheckForForefrontRequests = true;
101
- break;
102
- }
103
- }
104
- return result;
105
- }
106
- /**
107
- * @inheritDoc
108
- */
109
- async fetchNextRequest() {
110
- checkStorageAccess();
111
- if (this.queuePausedForMigration) {
112
- return null;
113
- }
114
- this.lastActivity = new Date();
115
- await this.ensureHeadIsNonEmpty();
116
- const nextRequestId = this.queueHeadIds.removeFirst();
117
- // We are likely done at this point.
118
- if (!nextRequestId) {
119
- return null;
120
- }
121
- const request = await this.getOrHydrateRequest(nextRequestId);
122
- // NOTE: It can happen that the queue head index is inconsistent with the main queue table. This can occur in two situations:
123
- // 1) Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
124
- // In this case, keep the request marked as in progress for a short while,
125
- // so that isFinished() doesn't return true and _ensureHeadIsNonEmpty() doesn't not load the request
126
- // into the queueHeadDict straight again. After the interval expires, fetchNextRequest()
127
- // will try to fetch this request again, until it eventually appears in the main table.
128
- if (!request) {
129
- this.log.debug('Cannot find a request from the beginning of queue or lost lock, will be retried later', {
130
- nextRequestId,
131
- });
132
- return null;
133
- }
134
- // 2) Queue head index is behind the main table and the underlying request was already handled
135
- // (by some other client, since we keep the track of handled requests in recentlyHandled dictionary).
136
- // We just add the request to the recentlyHandled dictionary so that next call to _ensureHeadIsNonEmpty()
137
- // will not put the request again to queueHeadDict.
138
- if (request.handledAt) {
139
- this.log.debug('Request fetched from the beginning of queue was already handled', { nextRequestId });
140
- return null;
141
- }
142
- this.dequeuedRequestCount += 1;
143
- return request;
144
- }
145
- /**
146
- * @inheritDoc
147
- */
148
- async markRequestHandled(request) {
149
- this.dequeuedRequestCount -= 1;
150
- return await super.markRequestHandled(request);
151
- }
152
- /**
153
- * @inheritDoc
154
- */
155
- async isFinished() {
156
- // We are not finished if we're still adding new requests in the background
157
- if (this.inProgressRequestBatchCount > 0) {
158
- return false;
159
- }
160
- // If the local queue head is non-empty, we don't need to query the "upstream" queue to know we are not finished yet
161
- if (this.queueHeadIds.length() > 0) {
162
- return false;
163
- }
164
- // Local queue head is empty - try to fetch and lock more requests
165
- await this.ensureHeadIsNonEmpty();
166
- // We managed to lock something - we are not finished
167
- if (this.queueHeadIds.length() > 0) {
168
- return false;
169
- }
170
- // We could not lock any new requests - decide based on whether the queue contains requests locked by another client
171
- if (this.queueHasLockedRequests !== undefined) {
172
- // The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much.
173
- if (this.queueHasLockedRequests &&
174
- this.dequeuedRequestCount === 0 &&
175
- ++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {
176
- this.log.info('The queue still contains requests locked by another client');
177
- }
178
- return !this.queueHasLockedRequests;
179
- }
180
- // The following is a legacy algorithm for checking if the queue is finished. It is used only for request queue clients that do not provide the `queueHasLockedRequests` flag.
181
- const currentHead = await this.client.listHead({ limit: 2 });
182
- if (currentHead.items.length === 0) {
183
- return true;
184
- }
185
- // Give users some more concrete info as to why their crawlers seem to be "hanging" doing nothing while we're waiting because the queue is technically
186
- // not empty. We decided that a queue with elements in its head but that are also locked shouldn't return true in this function.
187
- // If that ever changes, this function might need a rewrite
188
- // The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much. This is also a very specific path that most crawlers shouldn't hit
189
- if (++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {
190
- const requests = await Promise.all(currentHead.items.map(async (item) => this.client.getRequest(item.id)));
191
- this.log.info(`Queue head still returned requests that need to be processed (or that are locked by other clients)`, {
192
- requests: requests
193
- .map((r) => {
194
- if (!r) {
195
- return null;
196
- }
197
- return {
198
- id: r.id,
199
- lockExpiresAt: r.lockExpiresAt,
200
- lockedBy: r.lockByClient,
201
- };
202
- })
203
- .filter(Boolean),
204
- clientKey: this.clientKey,
205
- });
206
- }
207
- else {
208
- this.log.debug('Queue head still returned requests that need to be processed (or that are locked by other clients)', {
209
- requestIds: currentHead.items.map((item) => item.id),
210
- });
211
- }
212
- return false;
213
- }
214
- /**
215
- * @inheritDoc
216
- */
217
- async reclaimRequest(...args) {
218
- const res = await super.reclaimRequest(...args);
219
- if (res) {
220
- const [request, options] = args;
221
- if (options?.forefront) {
222
- this.shouldCheckForForefrontRequests = true;
223
- }
224
- // Try to delete the request lock if possible
225
- try {
226
- await this.client.deleteRequestLock(request.id, { forefront: options?.forefront ?? false });
227
- }
228
- catch (err) {
229
- this.log.debug(`Failed to delete request lock for request ${request.id}`, { err });
230
- }
231
- }
232
- return res;
233
- }
234
- async ensureHeadIsNonEmpty() {
235
- checkStorageAccess();
236
- // Stop fetching if we are paused for migration
237
- if (this.queuePausedForMigration) {
238
- return;
239
- }
240
- // We want to fetch ahead of time to minimize dead time
241
- // If we need to check for newly added forefront requests, we do it even if we already have some locked requests
242
- if (this.queueHeadIds.length() > QUEUE_HEAD_REFILL_THRESHOLD && !this.shouldCheckForForefrontRequests) {
243
- return;
244
- }
245
- this.listHeadAndLockPromise ??= this._listHeadAndLock().finally(() => {
246
- this.listHeadAndLockPromise = null;
247
- });
248
- await this.listHeadAndLockPromise;
249
- }
250
- async giveUpLock(id, uniqueKey) {
251
- if (id === undefined) {
252
- return;
253
- }
254
- try {
255
- await this.client.deleteRequestLock(id);
256
- }
257
- catch {
258
- this.log.debug('Failed to delete request lock', { id, uniqueKey });
259
- }
260
- }
261
- async _listHeadAndLock() {
262
- // Make a copy so that we can clear the flag only if the whole method executes after the flag was set
263
- // (i.e, it was not set in the middle of the execution of the method)
264
- const shouldCheckForForefrontRequests = this.shouldCheckForForefrontRequests;
265
- // NOTE: in theory, if we're not checking for forefront requests, we could fetch just enough requests to fill the local queue head to the limit.
266
- // We chose not to do this because 1. it's simpler and 2. the queue is being processed while we're fetching and we want to avoid underruns.
267
- // If we are checking for forefront requests, we need to fetch enough requests to be sure that we won't miss any new forefront ones.
268
- const headData = await this.client.listAndLockHead({
269
- limit: LIST_AND_LOCK_HEAD_LIMIT,
270
- lockSecs: this.requestLockSecs,
271
- });
272
- this.queueHasLockedRequests = headData.queueHasLockedRequests;
273
- const headIdBuffer = [];
274
- const forefrontHeadIdBuffer = [];
275
- // Go through the fetched requests, ensure they are cached locally and sort them into normal and forefront groups
276
- for (const { id, uniqueKey } of headData.items) {
277
- if (!id || !uniqueKey) {
278
- this.log.warning(`Skipping request from queue head as it's invalid. Please report this with the provided metadata!`, {
279
- id,
280
- uniqueKey,
281
- });
282
- // Remove the lock from the request for now, so that it can be picked up later
283
- // This may/may not succeed, but that's fine
284
- await this.giveUpLock(id, uniqueKey);
285
- continue;
286
- }
287
- // If we remember that we added the request ourselves and we added it to the forefront,
288
- // we will put it to the beginning of the local queue head to preserve the expected order.
289
- // If we do not remember that, we will enqueue it normally.
290
- const forefront = this.requestCache.get(getRequestId(uniqueKey))?.forefront ?? false;
291
- if (forefront) {
292
- forefrontHeadIdBuffer.unshift(id);
293
- }
294
- else {
295
- headIdBuffer.push(id);
296
- }
297
- // Ensure that the request is cached locally
298
- this._cacheRequest(getRequestId(uniqueKey), {
299
- requestId: id,
300
- uniqueKey,
301
- wasAlreadyPresent: true,
302
- wasAlreadyHandled: false,
303
- forefront,
304
- });
305
- }
306
- // Insert the newly fetched requests into the local queue head
307
- for (const id of headIdBuffer) {
308
- this.queueHeadIds.add(id, id, false);
309
- }
310
- for (const id of forefrontHeadIdBuffer) {
311
- this.queueHeadIds.add(id, id, true);
312
- }
313
- // Unlock and forget requests that would make the local queue head grow over the limit
314
- const toUnlock = [];
315
- const limit = shouldCheckForForefrontRequests
316
- ? LIST_AND_LOCK_HEAD_LIMIT // we may have received up to LIST_AND_LOCK_HEAD_LIMIT newly added forefront requests - we need to make sure that anything we already had in the queue gets unlocked
317
- : LIST_AND_LOCK_HEAD_LIMIT + QUEUE_HEAD_REFILL_THRESHOLD; // we tolerate up to QUEUE_HEAD_REFILL_THRESHOLD additional requests to avoid frequent, yet unnecessary unlocks
318
- while (this.queueHeadIds.length() > limit) {
319
- toUnlock.push(this.queueHeadIds.removeLast());
320
- }
321
- if (toUnlock.length > 0) {
322
- await Promise.all(toUnlock.map(async (id) => await this.giveUpLock(id)));
323
- }
324
- // We went through the whole procedure after `this.shouldCheckForForefrontRequests` was set -> we can clear the flag now
325
- if (shouldCheckForForefrontRequests) {
326
- this.shouldCheckForForefrontRequests = false;
327
- }
328
- }
329
- async getOrHydrateRequest(requestId) {
330
- checkStorageAccess();
331
- const cachedEntry = this.requestCache.get(requestId);
332
- if (!cachedEntry) {
333
- // 2.1. Attempt to prolong the request lock to see if we still own the request
334
- const prolongResult = await this._prolongRequestLock(requestId);
335
- if (!prolongResult) {
336
- return null;
337
- }
338
- // 2.1.1. If successful, hydrate the request and return it
339
- const hydratedRequest = await this.getRequest(requestId);
340
- // Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
341
- if (!hydratedRequest) {
342
- // Remove the lock from the request for now, so that it can be picked up later
343
- // This may/may not succeed, but that's fine
344
- try {
345
- await this.client.deleteRequestLock(requestId);
346
- }
347
- catch {
348
- // Ignore
349
- }
350
- return null;
351
- }
352
- this.requestCache.add(requestId, {
353
- id: requestId,
354
- uniqueKey: hydratedRequest.uniqueKey,
355
- hydrated: hydratedRequest,
356
- isHandled: hydratedRequest.handledAt !== null,
357
- lockExpiresAt: prolongResult.getTime(),
358
- forefront: false,
359
- });
360
- return hydratedRequest;
361
- }
362
- // 1.1. If hydrated, prolong the lock more and return it
363
- if (cachedEntry.hydrated) {
364
- // 1.1.1. If the lock expired on the hydrated requests, try to prolong. If we fail, we lost the request (or it was handled already)
365
- if (cachedEntry.lockExpiresAt && cachedEntry.lockExpiresAt < Date.now()) {
366
- const prolonged = await this._prolongRequestLock(cachedEntry.id);
367
- if (!prolonged) {
368
- return null;
369
- }
370
- cachedEntry.lockExpiresAt = prolonged.getTime();
371
- }
372
- return cachedEntry.hydrated;
373
- }
374
- // 1.2. If not hydrated, try to prolong the lock first (to ensure we keep it in our queue), hydrate and return it
375
- const prolonged = await this._prolongRequestLock(cachedEntry.id);
376
- if (!prolonged) {
377
- return null;
378
- }
379
- // This might still return null if the queue head is inconsistent with the main queue table.
380
- const hydratedRequest = await this.getRequest(cachedEntry.id);
381
- cachedEntry.hydrated = hydratedRequest;
382
- // Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
383
- if (!hydratedRequest) {
384
- // Remove the lock from the request for now, so that it can be picked up later
385
- // This may/may not succeed, but that's fine
386
- try {
387
- await this.client.deleteRequestLock(cachedEntry.id);
388
- }
389
- catch {
390
- // Ignore
391
- }
392
- return null;
393
- }
394
- return hydratedRequest;
395
- }
396
- async _prolongRequestLock(requestId) {
397
- try {
398
- const res = await this.client.prolongRequestLock(requestId, { lockSecs: this.requestLockSecs });
399
- return res.lockExpiresAt;
400
- }
401
- catch (err) {
402
- // Most likely we do not own the lock anymore
403
- this.log.warning(`Failed to prolong lock for cached request ${requestId}, either lost the lock or the request was already handled\n`, {
404
- err,
405
- });
406
- return null;
407
- }
408
- }
409
- _reset() {
410
- super._reset();
411
- this.listHeadAndLockPromise = null;
412
- this.queueHasLockedRequests = undefined;
413
- }
414
- _maybeAddRequestToQueueHead() {
415
- // Do nothing for request queue v2, as we are only able to lock requests when listing the head
416
- }
417
- async _clearPossibleLocks() {
418
- this.queuePausedForMigration = true;
419
- let requestId;
420
- // eslint-disable-next-line no-cond-assign
421
- while ((requestId = this.queueHeadIds.removeFirst()) !== null) {
422
- try {
423
- await this.client.deleteRequestLock(requestId);
424
- }
425
- catch {
426
- // We don't have the lock, or the request was never locked. Either way it's fine
427
- }
428
- }
429
- }
430
- /**
431
- * @inheritDoc
432
- */
433
- static async open(...args) {
434
- return super.open(...args);
435
- }
436
- }
437
- //# sourceMappingURL=request_queue_v2.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"request_queue_v2.js","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAO1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,gHAAgH;AAChH,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC,MAAM,wBAAwB,GAAG,EAAE,CAAC;AAEpC,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAEtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,OAAO,YAAa,SAAQ,eAAe;IACrC,sBAAsB,GAAyB,IAAI,CAAC;IACpD,sBAAsB,GAAwB,SAAS,CAAC;IACxD,+BAA+B,GAAG,KAAK,CAAC;IACxC,oBAAoB,GAAG,CAAC,CAAC;IAEjC,YAAY,OAA+B,EAAE,SAAwB,cAAc,CAAC,gBAAgB,EAAE;QAClG,KAAK,CACD;YACI,GAAG,OAAO;YACV,SAAS,EAAE,eAAe;YAC1B,8BAA8B,EAAE,2BAA2B;YAC3D,mBAAmB,EAAE,mBAAmB;SAC3C,EACD,MAAM,CACT,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,EAAE,wCAAsB,KAAK,IAAI,EAAE;YAC3C,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,sCAAqB,KAAK,IAAI,EAAE;YAC1C,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACgB,aAAa,CAAC,QAAgB,EAAE,kBAA6C;QAC5F,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAElD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,kBAAkB,CAAC,SAAS,EAAE;YAChD,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,IAAI;SACtB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,UAAU,CACrB,WAAmB,EACnB,UAAwC,EAAE;QAE1C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACjD,IAAI,CAAC,+BAA+B,GAAG,IAAI,CAAC;QAChD,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,WAAW,CACtB,YAA0B,EAC1B,UAAwC,EAAE;QAE1C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAC9D,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,iBAAiB,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAClD,IAAI,CAAC,+BAA+B,GAAG,IAAI,CAAC;gBAC5C,MAAM;YACV,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,gBAAgB;QAC3B,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAEtD,oCAAoC;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,OAAO,GAAmB,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC;QAE9E,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uFAAuF,EAAE;gBACpG,aAAa;aAChB,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;QAE/B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QAC9C,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;QAC/B,OAAO,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,UAAU;QACrB,2EAA2E;QAC3E,IAAI,IAAI,CAAC,2BAA2B,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,oHAAoH;QACpH,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,kEAAkE;QAClE,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,qDAAqD;QACrD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,oHAAoH;QACpH,IAAI,IAAI,CAAC,sBAAsB,KAAK,SAAS,EAAE,CAAC;YAC5C,yFAAyF;YACzF,IACI,IAAI,CAAC,sBAAsB;gBAC3B,IAAI,CAAC,oBAAoB,KAAK,CAAC;gBAC/B,EAAE,IAAI,CAAC,oCAAoC,GAAG,EAAE,KAAK,CAAC,EACxD,CAAC;gBACC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;YAChF,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC;QACxC,CAAC;QAED,8KAA8K;QAE9K,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAE7D,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,sJAAsJ;QACtJ,gIAAgI;QAChI,2DAA2D;QAC3D,4JAA4J;QAC5J,IAAI,EAAE,IAAI,CAAC,oCAAoC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACzD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAE3G,IAAI,CAAC,GAAG,CAAC,IAAI,CACT,oGAAoG,EACpG;gBACI,QAAQ,EAAE,QAAQ;qBACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACP,IAAI,CAAC,CAAC,EAAE,CAAC;wBACL,OAAO,IAAI,CAAC;oBAChB,CAAC;oBAED,OAAO;wBACH,EAAE,EAAE,CAAC,CAAC,EAAE;wBACR,aAAa,EAAE,CAAC,CAAC,aAAa;wBAC9B,QAAQ,EAAE,CAAC,CAAC,YAAY;qBAC3B,CAAC;gBACN,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;gBACpB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CACJ,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,oGAAoG,EACpG;gBACI,UAAU,EAAE,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;aACvD,CACJ,CAAC;QACN,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,cAAc,CACzB,GAAG,IAAmD;QAEtD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;QAEhD,IAAI,GAAG,EAAE,CAAC;YACN,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;YAEhC,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;gBACrB,IAAI,CAAC,+BAA+B,GAAG,IAAI,CAAC;YAChD,CAAC;YAED,6CAA6C;YAC7C,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,KAAK,EAAE,CAAC,CAAC;YACjG,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,6CAA6C,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACvF,CAAC;QACL,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAES,KAAK,CAAC,oBAAoB;QAChC,kBAAkB,EAAE,CAAC;QAErB,+CAA+C;QAC/C,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO;QACX,CAAC;QAED,uDAAuD;QACvD,gHAAgH;QAChH,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,2BAA2B,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC;YACpG,OAAO;QACX,CAAC;QAED,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACjE,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,sBAAsB,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,EAAW,EAAE,SAAkB;QACpD,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACL,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,qGAAqG;QACrG,qEAAqE;QACrE,MAAM,+BAA+B,GAAG,IAAI,CAAC,+BAA+B,CAAC;QAE7E,gJAAgJ;QAChJ,2IAA2I;QAC3I,oIAAoI;QACpI,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;YAC/C,KAAK,EAAE,wBAAwB;YAC/B,QAAQ,EAAE,IAAI,CAAC,eAAe;SACjC,CAAC,CAAC;QAEH,IAAI,CAAC,sBAAsB,GAAG,QAAQ,CAAC,sBAAsB,CAAC;QAE9D,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,MAAM,qBAAqB,GAAG,EAAE,CAAC;QAEjC,iHAAiH;QACjH,KAAK,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,kGAAkG,EAClG;oBACI,EAAE;oBACF,SAAS;iBACZ,CACJ,CAAC;gBAEF,8EAA8E;gBAC9E,4CAA4C;gBAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;gBACrC,SAAS;YACb,CAAC;YAED,uFAAuF;YACvF,0FAA0F;YAC1F,2DAA2D;YAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;YACrF,IAAI,SAAS,EAAE,CAAC;gBACZ,qBAAqB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACJ,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;YAED,4CAA4C;YAC5C,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE;gBACxC,SAAS,EAAE,EAAE;gBACb,SAAS;gBACT,iBAAiB,EAAE,IAAI;gBACvB,iBAAiB,EAAE,KAAK;gBACxB,SAAS;aACZ,CAAC,CAAC;QACP,CAAC;QAED,8DAA8D;QAC9D,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;QACzC,CAAC;QAED,KAAK,MAAM,EAAE,IAAI,qBAAqB,EAAE,CAAC;YACrC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QACxC,CAAC;QAED,sFAAsF;QACtF,MAAM,QAAQ,GAAG,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,+BAA+B;YACzC,CAAC,CAAC,wBAAwB,CAAC,oKAAoK;YAC/L,CAAC,CAAC,wBAAwB,GAAG,2BAA2B,CAAC,CAAC,+GAA+G;QAC7K,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAG,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,wHAAwH;QACxH,IAAI,+BAA+B,EAAE,CAAC;YAClC,IAAI,CAAC,+BAA+B,GAAG,KAAK,CAAC;QACjD,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,SAAiB;QAEjB,kBAAkB,EAAE,CAAC;QAErB,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAErD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,8EAA8E;YAC9E,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAEhE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,0DAA0D;YAC1D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAEzD,sIAAsI;YACtI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnB,8EAA8E;gBAC9E,4CAA4C;gBAC5C,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;gBACnD,CAAC;gBAAC,MAAM,CAAC;oBACL,SAAS;gBACb,CAAC;gBAED,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC7B,EAAE,EAAE,SAAS;gBACb,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,QAAQ,EAAE,eAAe;gBACzB,SAAS,EAAE,eAAe,CAAC,SAAS,KAAK,IAAI;gBAC7C,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE;gBACtC,SAAS,EAAE,KAAK;aACnB,CAAC,CAAC;YAEH,OAAO,eAAe,CAAC;QAC3B,CAAC;QAED,wDAAwD;QACxD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;YACvB,mIAAmI;YACnI,IAAI,WAAW,CAAC,aAAa,IAAI,WAAW,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBACtE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBAChB,CAAC;gBAED,WAAW,CAAC,aAAa,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;YACpD,CAAC;YAED,OAAO,WAAW,CAAC,QAAQ,CAAC;QAChC,CAAC;QAED,iHAAiH;QACjH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,4FAA4F;QAC5F,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAE9D,WAAW,CAAC,QAAQ,GAAG,eAAe,CAAC;QAEvC,sIAAsI;QACtI,IAAI,CAAC,eAAe,EAAE,CAAC;YACnB,8EAA8E;YAC9E,4CAA4C;YAC5C,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YACxD,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;YAED,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,eAAe,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,SAAiB;QAC/C,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;YAChG,OAAO,GAAG,CAAC,aAAa,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,6CAA6C;YAC7C,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,6CAA6C,SAAS,6DAA6D,EACnH;gBACI,GAAG;aACN,CACJ,CAAC;YAEF,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEkB,MAAM;QACrB,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACnC,IAAI,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAC5C,CAAC;IAEkB,2BAA2B;QAC1C,8FAA8F;IAClG,CAAC;IAES,KAAK,CAAC,mBAAmB;QAC/B,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACpC,IAAI,SAAwB,CAAC;QAE7B,0CAA0C;QAC1C,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5D,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACnD,CAAC;YAAC,MAAM,CAAC;gBACL,gFAAgF;YACpF,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAU,KAAK,CAAC,IAAI,CAAC,GAAG,IAA6C;QACvE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAA0B,CAAC;IACxD,CAAC;CACJ"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"sitemap_request_list.d.ts","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAgB,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAKxE,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAoB,MAAM,4BAA4B,CAAC;AAI3F,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAMtD,UAAU,cAAc;IACpB;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC;IAE1C;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,yBAA0B,SAAQ,cAAc;IAC7D;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,kBAAkB,CAAC,EAAE;QACjB;;;WAGG;QACH,MAAM,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,mBAAmB,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,oBAAoB,GAAG,UAAU,CAAC,CAAC;IACnF;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC/B;AAiBD;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACnD;;;OAGG;IACH,UAAU,cAAqB;IAE/B,2DAA2D;IAC3D,OAAO,CAAC,SAAS,CAAqB;IAEtC;;;;OAIG;IACH,OAAO,CAAC,WAAW,CAA8B;IAEjD;;OAEG;IACH,OAAO,CAAC,sBAAsB,CAa5B;IAEF;;;;;OAKG;IACH,OAAO,CAAC,cAAc,CAAY;IAElC;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY,CAAS;IAE7B,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,eAAe,CAAC,CAAS;IAEjC,OAAO,CAAC,KAAK,CAAC,CAAgB;IAE9B,OAAO,CAAC,MAAM,CAAS;IAEvB;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,CAAS;IAE1B;;OAEG;IACH,OAAO,CAAC,GAAG,CAAgB;IAE3B,OAAO,CAAC,wBAAwB,CAA0B;IAC1D,OAAO,CAAC,iBAAiB,CAA0B;IAEnD,8CAA8C;IAC9C,OAAO,CAAC,MAAM,CAAe;IAE7B,OAAO,CAAC,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,OAAO;IAwDP;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAO7B;;;;OAIG;YACW,WAAW;IAkBzB;;;;;OAKG;YACW,WAAW;IAqBzB;;;;OAIG;IACH,oBAAoB,IAAI,OAAO;IAO/B;;;;OAIG;YACW,IAAI;IAuClB;;;;;;OAMG;WACU,IAAI,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA6BlF;;OAEG;IACH,MAAM,IAAI,MAAM;IAIhB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAIjC;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;YA6CrB,YAAY;IA+B1B;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAkBjD;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAS7B;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAMrD;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAOzD,OAAO,CAAC,+BAA+B;CAQ1C"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"sitemap_request_list.js","sourceRoot":"","sources":["../../src/storages/sitemap_request_list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,EAAE,YAAY,EAA4B,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,6BAA6B,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAG9G,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,qBAAqB,GAAG,4BAA4B,CAAC;AAuG3D;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAC3B;;;OAGG;IACH,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,2DAA2D;IACnD,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC;;;;OAIG;IACK,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEjD;;OAEG;IACK,sBAAsB,GAA2B;QACrD;;WAEG;QACH,oBAAoB,EAAE,IAAI;QAC1B;;WAEG;QACH,iBAAiB,EAAE,IAAI,GAAG,EAAU;QACpC;;WAEG;QACH,kBAAkB,EAAE,IAAI,GAAG,EAAU;KACxC,CAAC;IAEF;;;;;OAKG;IACK,cAAc,CAAY;IAElC;;;;;;;OAOG;IACK,YAAY,GAAG,KAAK,CAAC;IAE7B,iDAAiD;IACzC,eAAe,GAAG,CAAC,CAAC;IAEpB,eAAe,CAAU;IAEzB,KAAK,CAAiB;IAEtB,MAAM,GAAG,KAAK,CAAC;IAEvB;;OAEG;IACK,QAAQ,CAAU;IAE1B;;OAEG;IACK,GAAG,CAAgB;IAEnB,wBAAwB,GAAuB,EAAE,CAAC;IAClD,iBAAiB,GAAuB,EAAE,CAAC;IAEnD,8CAA8C;IACtC,MAAM,CAAe;IAErB,kBAAkB,CAAgE;IAE1F,gBAAgB;IAChB,YAAoB,OAAkC;QAClD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzB,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7E,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;YACD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAE5C,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAE9E,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;oBAC7C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjF,CAAC;qBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACpD,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAE1E,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC,CAAC;QAEzE,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAE/C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,aAAqB;QACzC,OAAO,IAAI,SAAS,CAAC;YACjB,UAAU,EAAE,IAAI;YAChB,aAAa;SAChB,CAAC,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC1B,OAAO,CAAC,aAAa,EAAE,EAAE;YACrB,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC;YAEvC,MAAM,YAAY,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC;YAC5D,MAAM,WAAW,GAAG,CAAC,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC;YAE9E,OAAO,OAAO,CAAC,YAAY,IAAI,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAAC,GAAW;QACrC,OAAO,CACH,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACzD,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAC7F,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,GAAkB;QACxC,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3D,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,+EAA+E;gBAC/E,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,EAAE;oBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3C,OAAO,CAAC,OAAO,CAAC,CAAC;gBACrB,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,oBAAoB;QAChB,OAAO,CACH,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,KAAK,IAAI;YACzD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,KAAK,CAAC,CAC5D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,IAAI,CAAC,EACf,mBAAmB,GAGtB;QACG,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACxD,MAAM,UAAU,GACZ,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBAChD,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAM,CAAC;YAE1E,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE;oBACrF,GAAG,mBAAmB;oBACtB,QAAQ,EAAE,CAAC;oBACX,kBAAkB,EAAE,IAAI;iBAC3B,CAAC,EAAE,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBACzB,2BAA2B;wBAC3B,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC7D,SAAS;oBACb,CAAC;oBAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC/D,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChE,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAClE,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YACtD,IAAI,CAAC,sBAAsB,CAAC,oBAAoB,GAAG,IAAI,CAAC;QAC5D,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAkC;QAChD,MAAM,EAAE,UAAU,EAAE,GAAG,WAAW,EAAE,GAAG,OAAO,CAAC;QAE/C,MAAM,WAAW,GAAG,IAAI,kBAAkB,CAAC;YACvC,GAAG,WAAW;YACd,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,qBAAqB;SACpE,CAAC,CAAC;QACH,MAAM,WAAW,CAAC,YAAY,EAAE,CAAC;QACjC,KAAK,WAAW,CAAC,IAAI,CAAC;YAClB,mBAAmB,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,mBAAmB,EAAE,UAAU,EAAE;SAC1G,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACxC,WAAW,CAAC,MAAM,CAAC,EAAE,+CAA0B,WAAW,CAAC,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;YAC5C,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,UAAU,CAAC,GAAG,EAAE;gBACZ,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC;YACpC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM;QACF,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAClH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,OAAO,CACH,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,YAAY,CAAC,CAC7G,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,KAAK,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACH,YAAY;QACR,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACf,MAAM;YACV,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,0EAA0E;QAC1E,2FAA2F;QAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;QAElF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YACzB,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,CAAC;YACpC,SAAS,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAEhC,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE;YAC5C,sBAAsB,EAAE;gBACpB,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;gBAC9E,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAC,oBAAoB;gBACtE,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;aAC/E;YACD,QAAQ;YACR,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,+EAA+E;YACnI,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACnD,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;SACY,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,KAAK,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEvF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,sBAAsB,GAAG;YAC1B,kBAAkB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,kBAAkB,CAAC;YAC5E,oBAAoB,EAAE,KAAK,CAAC,sBAAsB,CAAC,oBAAoB;YACvE,iBAAiB,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,iBAAiB,CAAC;SAC7E,CAAC;QAEF,IAAI,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAEpD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QAClB,0CAA0C;QAC1C,IAAI,OAAO,GAA8B,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QAC9E,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACJ,0CAA0C;YAC1C,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEpB,MAAM,OAAO,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,sDAAsD;IAChG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAEO,+BAA+B,CAAC,GAAW;QAC/C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,4CAA4C,GAAG,GAAG,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,GAAG,GAAG,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;CACJ"}