@crawlee/core 4.0.0-beta.65 → 4.0.0-beta.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawlers/crawler_commons.d.ts +1 -1
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +1 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/package.json +5 -5
- package/storages/index.d.ts +1 -4
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +1 -4
- package/storages/index.js.map +1 -1
- package/storages/request_list.d.ts +1 -1
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +2 -2
- package/storages/request_list.js.map +1 -1
- package/storages/request_loader.d.ts +5 -5
- package/storages/request_loader.d.ts.map +1 -1
- package/storages/request_manager.d.ts +10 -1
- package/storages/request_manager.d.ts.map +1 -1
- package/storages/request_manager_tandem.d.ts +13 -2
- package/storages/request_manager_tandem.d.ts.map +1 -1
- package/storages/request_manager_tandem.js +21 -3
- package/storages/request_manager_tandem.js.map +1 -1
- package/storages/request_queue.d.ts +276 -44
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +576 -212
- package/storages/request_queue.js.map +1 -1
- package/storages/sitemap_request_loader.d.ts +1 -1
- package/storages/sitemap_request_loader.d.ts.map +1 -1
- package/storages/sitemap_request_loader.js +2 -2
- package/storages/sitemap_request_loader.js.map +1 -1
- package/validators.d.ts +4 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +4 -0
- package/validators.js.map +1 -1
- package/storages/request_provider.d.ts +0 -325
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -619
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -88
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -437
- package/storages/request_queue_v2.js.map +0 -1
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
|
|
2
|
-
import type { Configuration } from '../configuration.js';
|
|
3
|
-
import type { Request, Source } from '../request.js';
|
|
4
|
-
import type { RequestsLike } from './request_manager.js';
|
|
5
|
-
import type { RequestProviderOptions, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_provider.js';
|
|
6
|
-
import { RequestProvider } from './request_provider.js';
|
|
7
|
-
/**
|
|
8
|
-
* Represents a queue of URLs to crawl, which is used for deep crawling of websites
|
|
9
|
-
* where you start with several URLs and then recursively
|
|
10
|
-
* follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
|
|
11
|
-
*
|
|
12
|
-
* Each URL is represented using an instance of the {@link Request} class.
|
|
13
|
-
* The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
|
|
14
|
-
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
15
|
-
* To add a single URL multiple times to the queue,
|
|
16
|
-
* corresponding {@link Request} objects will need to have different `uniqueKey` properties.
|
|
17
|
-
*
|
|
18
|
-
* Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
|
|
19
|
-
*
|
|
20
|
-
* `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
|
|
21
|
-
* and {@link PlaywrightCrawler} as a source of URLs to crawl.
|
|
22
|
-
* Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
|
|
23
|
-
* On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
|
|
24
|
-
*
|
|
25
|
-
* **Example usage:**
|
|
26
|
-
*
|
|
27
|
-
* ```javascript
|
|
28
|
-
* // Open the default request queue associated with the crawler run
|
|
29
|
-
* const queue = await RequestQueue.open();
|
|
30
|
-
*
|
|
31
|
-
* // Open a named request queue
|
|
32
|
-
* const queueWithName = await RequestQueue.open('some-name');
|
|
33
|
-
*
|
|
34
|
-
* // Enqueue few requests
|
|
35
|
-
* await queue.addRequest({ url: 'http://example.com/aaa' });
|
|
36
|
-
* await queue.addRequest({ url: 'http://example.com/bbb' });
|
|
37
|
-
* await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
|
|
38
|
-
* ```
|
|
39
|
-
* @category Sources
|
|
40
|
-
*/
|
|
41
|
-
export declare class RequestQueue extends RequestProvider {
|
|
42
|
-
private listHeadAndLockPromise;
|
|
43
|
-
private queueHasLockedRequests;
|
|
44
|
-
private shouldCheckForForefrontRequests;
|
|
45
|
-
private dequeuedRequestCount;
|
|
46
|
-
constructor(options: RequestProviderOptions, config?: Configuration);
|
|
47
|
-
/**
|
|
48
|
-
* Caches information about request to beware of unneeded addRequest() calls.
|
|
49
|
-
*/
|
|
50
|
-
protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
|
|
51
|
-
/**
|
|
52
|
-
* @inheritDoc
|
|
53
|
-
*/
|
|
54
|
-
addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
|
|
55
|
-
/**
|
|
56
|
-
* @inheritDoc
|
|
57
|
-
*/
|
|
58
|
-
addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
59
|
-
/**
|
|
60
|
-
* @inheritDoc
|
|
61
|
-
*/
|
|
62
|
-
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
|
|
63
|
-
/**
|
|
64
|
-
* @inheritDoc
|
|
65
|
-
*/
|
|
66
|
-
markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
|
|
67
|
-
/**
|
|
68
|
-
* @inheritDoc
|
|
69
|
-
*/
|
|
70
|
-
isFinished(): Promise<boolean>;
|
|
71
|
-
/**
|
|
72
|
-
* @inheritDoc
|
|
73
|
-
*/
|
|
74
|
-
reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>): ReturnType<RequestProvider['reclaimRequest']>;
|
|
75
|
-
protected ensureHeadIsNonEmpty(): Promise<void>;
|
|
76
|
-
private giveUpLock;
|
|
77
|
-
private _listHeadAndLock;
|
|
78
|
-
private getOrHydrateRequest;
|
|
79
|
-
private _prolongRequestLock;
|
|
80
|
-
protected _reset(): void;
|
|
81
|
-
protected _maybeAddRequestToQueueHead(): void;
|
|
82
|
-
protected _clearPossibleLocks(): Promise<void>;
|
|
83
|
-
/**
|
|
84
|
-
* @inheritDoc
|
|
85
|
-
*/
|
|
86
|
-
static open(...args: Parameters<typeof RequestProvider.open>): Promise<RequestQueue>;
|
|
87
|
-
}
|
|
88
|
-
//# sourceMappingURL=request_queue_v2.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue_v2.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAGrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EACR,sBAAsB,EACtB,yBAAyB,EACzB,4BAA4B,EAC/B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAiBxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,YAAa,SAAQ,eAAe;IAC7C,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,OAAO,CAAC,sBAAsB,CAAkC;IAChE,OAAO,CAAC,+BAA+B,CAAS;IAChD,OAAO,CAAC,oBAAoB,CAAK;gBAErB,OAAO,EAAE,sBAAsB,EAAE,MAAM,GAAE,aAAiD;IAoBtG;;OAEG;cACgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAevG;;OAEG;IACY,UAAU,CACrB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAQrC;;OAEG;IACY,WAAW,CACtB,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAWlC;;OAEG;IACY,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAiDhG;;OAEG;IACY,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAK9F;;OAEG;IACY,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IA+E7C;;OAEG;IACY,cAAc,CACzB,GAAG,IAAI,EAAE,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC,GACvD,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;cAqBhC,oBAAoB;YAqBtB,UAAU;YAYV,gBAAgB;YAmFhB,mBAAmB;YAuFnB,mBAAmB;cAiBd,MAAM;cAMN,2BAA2B;cAI9B,mBAAmB;IAcnC;;OAEG;WACmB,IAAI,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;CAGtG"}
|
|
@@ -1,437 +0,0 @@
|
|
|
1
|
-
import { serviceLocator } from '../service_locator.js';
|
|
2
|
-
import { checkStorageAccess } from './access_checking.js';
|
|
3
|
-
import { RequestProvider } from './request_provider.js';
|
|
4
|
-
import { getRequestId } from './utils.js';
|
|
5
|
-
// Double the limit of RequestQueue v1 (1_000_000) as we also store keyed by request.id, not just from uniqueKey
|
|
6
|
-
const MAX_CACHED_REQUESTS = 2_000_000;
|
|
7
|
-
/**
|
|
8
|
-
* This number must be large enough so that processing of all these requests cannot be done in
|
|
9
|
-
* a time lower than expected maximum latency of DynamoDB, but low enough not to waste too much memory.
|
|
10
|
-
* @internal
|
|
11
|
-
*/
|
|
12
|
-
const RECENTLY_HANDLED_CACHE_SIZE = 1000;
|
|
13
|
-
const LIST_AND_LOCK_HEAD_LIMIT = 25;
|
|
14
|
-
const QUEUE_HEAD_REFILL_THRESHOLD = 1;
|
|
15
|
-
/**
|
|
16
|
-
* Represents a queue of URLs to crawl, which is used for deep crawling of websites
|
|
17
|
-
* where you start with several URLs and then recursively
|
|
18
|
-
* follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
|
|
19
|
-
*
|
|
20
|
-
* Each URL is represented using an instance of the {@link Request} class.
|
|
21
|
-
* The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
|
|
22
|
-
* with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
|
|
23
|
-
* To add a single URL multiple times to the queue,
|
|
24
|
-
* corresponding {@link Request} objects will need to have different `uniqueKey` properties.
|
|
25
|
-
*
|
|
26
|
-
* Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
|
|
27
|
-
*
|
|
28
|
-
* `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
|
|
29
|
-
* and {@link PlaywrightCrawler} as a source of URLs to crawl.
|
|
30
|
-
* Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
|
|
31
|
-
* On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
|
|
32
|
-
*
|
|
33
|
-
* **Example usage:**
|
|
34
|
-
*
|
|
35
|
-
* ```javascript
|
|
36
|
-
* // Open the default request queue associated with the crawler run
|
|
37
|
-
* const queue = await RequestQueue.open();
|
|
38
|
-
*
|
|
39
|
-
* // Open a named request queue
|
|
40
|
-
* const queueWithName = await RequestQueue.open('some-name');
|
|
41
|
-
*
|
|
42
|
-
* // Enqueue few requests
|
|
43
|
-
* await queue.addRequest({ url: 'http://example.com/aaa' });
|
|
44
|
-
* await queue.addRequest({ url: 'http://example.com/bbb' });
|
|
45
|
-
* await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
|
|
46
|
-
* ```
|
|
47
|
-
* @category Sources
|
|
48
|
-
*/
|
|
49
|
-
export class RequestQueue extends RequestProvider {
|
|
50
|
-
listHeadAndLockPromise = null;
|
|
51
|
-
queueHasLockedRequests = undefined;
|
|
52
|
-
shouldCheckForForefrontRequests = false;
|
|
53
|
-
dequeuedRequestCount = 0;
|
|
54
|
-
constructor(options, config = serviceLocator.getConfiguration()) {
|
|
55
|
-
super({
|
|
56
|
-
...options,
|
|
57
|
-
logPrefix: 'RequestQueue2',
|
|
58
|
-
recentlyHandledRequestsMaxSize: RECENTLY_HANDLED_CACHE_SIZE,
|
|
59
|
-
requestCacheMaxSize: MAX_CACHED_REQUESTS,
|
|
60
|
-
}, config);
|
|
61
|
-
this.events.on("migrating" /* EventType.MIGRATING */, async () => {
|
|
62
|
-
await this._clearPossibleLocks();
|
|
63
|
-
});
|
|
64
|
-
this.events.on("aborting" /* EventType.ABORTING */, async () => {
|
|
65
|
-
await this._clearPossibleLocks();
|
|
66
|
-
});
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Caches information about request to beware of unneeded addRequest() calls.
|
|
70
|
-
*/
|
|
71
|
-
_cacheRequest(cacheKey, queueOperationInfo) {
|
|
72
|
-
super._cacheRequest(cacheKey, queueOperationInfo);
|
|
73
|
-
this.requestCache.remove(queueOperationInfo.requestId);
|
|
74
|
-
this.requestCache.add(queueOperationInfo.requestId, {
|
|
75
|
-
id: queueOperationInfo.requestId,
|
|
76
|
-
isHandled: queueOperationInfo.wasAlreadyHandled,
|
|
77
|
-
uniqueKey: queueOperationInfo.uniqueKey,
|
|
78
|
-
forefront: queueOperationInfo.forefront,
|
|
79
|
-
hydrated: null,
|
|
80
|
-
lockExpiresAt: null,
|
|
81
|
-
});
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* @inheritDoc
|
|
85
|
-
*/
|
|
86
|
-
async addRequest(requestLike, options = {}) {
|
|
87
|
-
const result = await super.addRequest(requestLike, options);
|
|
88
|
-
if (!result.wasAlreadyPresent && options.forefront) {
|
|
89
|
-
this.shouldCheckForForefrontRequests = true;
|
|
90
|
-
}
|
|
91
|
-
return result;
|
|
92
|
-
}
|
|
93
|
-
/**
|
|
94
|
-
* @inheritDoc
|
|
95
|
-
*/
|
|
96
|
-
async addRequests(requestsLike, options = {}) {
|
|
97
|
-
const result = await super.addRequests(requestsLike, options);
|
|
98
|
-
for (const request of result.processedRequests) {
|
|
99
|
-
if (!request.wasAlreadyPresent && options.forefront) {
|
|
100
|
-
this.shouldCheckForForefrontRequests = true;
|
|
101
|
-
break;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
return result;
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* @inheritDoc
|
|
108
|
-
*/
|
|
109
|
-
async fetchNextRequest() {
|
|
110
|
-
checkStorageAccess();
|
|
111
|
-
if (this.queuePausedForMigration) {
|
|
112
|
-
return null;
|
|
113
|
-
}
|
|
114
|
-
this.lastActivity = new Date();
|
|
115
|
-
await this.ensureHeadIsNonEmpty();
|
|
116
|
-
const nextRequestId = this.queueHeadIds.removeFirst();
|
|
117
|
-
// We are likely done at this point.
|
|
118
|
-
if (!nextRequestId) {
|
|
119
|
-
return null;
|
|
120
|
-
}
|
|
121
|
-
const request = await this.getOrHydrateRequest(nextRequestId);
|
|
122
|
-
// NOTE: It can happen that the queue head index is inconsistent with the main queue table. This can occur in two situations:
|
|
123
|
-
// 1) Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
|
|
124
|
-
// In this case, keep the request marked as in progress for a short while,
|
|
125
|
-
// so that isFinished() doesn't return true and _ensureHeadIsNonEmpty() doesn't not load the request
|
|
126
|
-
// into the queueHeadDict straight again. After the interval expires, fetchNextRequest()
|
|
127
|
-
// will try to fetch this request again, until it eventually appears in the main table.
|
|
128
|
-
if (!request) {
|
|
129
|
-
this.log.debug('Cannot find a request from the beginning of queue or lost lock, will be retried later', {
|
|
130
|
-
nextRequestId,
|
|
131
|
-
});
|
|
132
|
-
return null;
|
|
133
|
-
}
|
|
134
|
-
// 2) Queue head index is behind the main table and the underlying request was already handled
|
|
135
|
-
// (by some other client, since we keep the track of handled requests in recentlyHandled dictionary).
|
|
136
|
-
// We just add the request to the recentlyHandled dictionary so that next call to _ensureHeadIsNonEmpty()
|
|
137
|
-
// will not put the request again to queueHeadDict.
|
|
138
|
-
if (request.handledAt) {
|
|
139
|
-
this.log.debug('Request fetched from the beginning of queue was already handled', { nextRequestId });
|
|
140
|
-
return null;
|
|
141
|
-
}
|
|
142
|
-
this.dequeuedRequestCount += 1;
|
|
143
|
-
return request;
|
|
144
|
-
}
|
|
145
|
-
/**
|
|
146
|
-
* @inheritDoc
|
|
147
|
-
*/
|
|
148
|
-
async markRequestHandled(request) {
|
|
149
|
-
this.dequeuedRequestCount -= 1;
|
|
150
|
-
return await super.markRequestHandled(request);
|
|
151
|
-
}
|
|
152
|
-
/**
|
|
153
|
-
* @inheritDoc
|
|
154
|
-
*/
|
|
155
|
-
async isFinished() {
|
|
156
|
-
// We are not finished if we're still adding new requests in the background
|
|
157
|
-
if (this.inProgressRequestBatchCount > 0) {
|
|
158
|
-
return false;
|
|
159
|
-
}
|
|
160
|
-
// If the local queue head is non-empty, we don't need to query the "upstream" queue to know we are not finished yet
|
|
161
|
-
if (this.queueHeadIds.length() > 0) {
|
|
162
|
-
return false;
|
|
163
|
-
}
|
|
164
|
-
// Local queue head is empty - try to fetch and lock more requests
|
|
165
|
-
await this.ensureHeadIsNonEmpty();
|
|
166
|
-
// We managed to lock something - we are not finished
|
|
167
|
-
if (this.queueHeadIds.length() > 0) {
|
|
168
|
-
return false;
|
|
169
|
-
}
|
|
170
|
-
// We could not lock any new requests - decide based on whether the queue contains requests locked by another client
|
|
171
|
-
if (this.queueHasLockedRequests !== undefined) {
|
|
172
|
-
// The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much.
|
|
173
|
-
if (this.queueHasLockedRequests &&
|
|
174
|
-
this.dequeuedRequestCount === 0 &&
|
|
175
|
-
++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {
|
|
176
|
-
this.log.info('The queue still contains requests locked by another client');
|
|
177
|
-
}
|
|
178
|
-
return !this.queueHasLockedRequests;
|
|
179
|
-
}
|
|
180
|
-
// The following is a legacy algorithm for checking if the queue is finished. It is used only for request queue clients that do not provide the `queueHasLockedRequests` flag.
|
|
181
|
-
const currentHead = await this.client.listHead({ limit: 2 });
|
|
182
|
-
if (currentHead.items.length === 0) {
|
|
183
|
-
return true;
|
|
184
|
-
}
|
|
185
|
-
// Give users some more concrete info as to why their crawlers seem to be "hanging" doing nothing while we're waiting because the queue is technically
|
|
186
|
-
// not empty. We decided that a queue with elements in its head but that are also locked shouldn't return true in this function.
|
|
187
|
-
// If that ever changes, this function might need a rewrite
|
|
188
|
-
// The `% 25` was absolutely arbitrarily picked. It's just to not spam the logs too much. This is also a very specific path that most crawlers shouldn't hit
|
|
189
|
-
if (++this.isFinishedCalledWhileHeadWasNotEmpty % 25 === 0) {
|
|
190
|
-
const requests = await Promise.all(currentHead.items.map(async (item) => this.client.getRequest(item.id)));
|
|
191
|
-
this.log.info(`Queue head still returned requests that need to be processed (or that are locked by other clients)`, {
|
|
192
|
-
requests: requests
|
|
193
|
-
.map((r) => {
|
|
194
|
-
if (!r) {
|
|
195
|
-
return null;
|
|
196
|
-
}
|
|
197
|
-
return {
|
|
198
|
-
id: r.id,
|
|
199
|
-
lockExpiresAt: r.lockExpiresAt,
|
|
200
|
-
lockedBy: r.lockByClient,
|
|
201
|
-
};
|
|
202
|
-
})
|
|
203
|
-
.filter(Boolean),
|
|
204
|
-
clientKey: this.clientKey,
|
|
205
|
-
});
|
|
206
|
-
}
|
|
207
|
-
else {
|
|
208
|
-
this.log.debug('Queue head still returned requests that need to be processed (or that are locked by other clients)', {
|
|
209
|
-
requestIds: currentHead.items.map((item) => item.id),
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
return false;
|
|
213
|
-
}
|
|
214
|
-
/**
|
|
215
|
-
* @inheritDoc
|
|
216
|
-
*/
|
|
217
|
-
async reclaimRequest(...args) {
|
|
218
|
-
const res = await super.reclaimRequest(...args);
|
|
219
|
-
if (res) {
|
|
220
|
-
const [request, options] = args;
|
|
221
|
-
if (options?.forefront) {
|
|
222
|
-
this.shouldCheckForForefrontRequests = true;
|
|
223
|
-
}
|
|
224
|
-
// Try to delete the request lock if possible
|
|
225
|
-
try {
|
|
226
|
-
await this.client.deleteRequestLock(request.id, { forefront: options?.forefront ?? false });
|
|
227
|
-
}
|
|
228
|
-
catch (err) {
|
|
229
|
-
this.log.debug(`Failed to delete request lock for request ${request.id}`, { err });
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
return res;
|
|
233
|
-
}
|
|
234
|
-
async ensureHeadIsNonEmpty() {
|
|
235
|
-
checkStorageAccess();
|
|
236
|
-
// Stop fetching if we are paused for migration
|
|
237
|
-
if (this.queuePausedForMigration) {
|
|
238
|
-
return;
|
|
239
|
-
}
|
|
240
|
-
// We want to fetch ahead of time to minimize dead time
|
|
241
|
-
// If we need to check for newly added forefront requests, we do it even if we already have some locked requests
|
|
242
|
-
if (this.queueHeadIds.length() > QUEUE_HEAD_REFILL_THRESHOLD && !this.shouldCheckForForefrontRequests) {
|
|
243
|
-
return;
|
|
244
|
-
}
|
|
245
|
-
this.listHeadAndLockPromise ??= this._listHeadAndLock().finally(() => {
|
|
246
|
-
this.listHeadAndLockPromise = null;
|
|
247
|
-
});
|
|
248
|
-
await this.listHeadAndLockPromise;
|
|
249
|
-
}
|
|
250
|
-
async giveUpLock(id, uniqueKey) {
|
|
251
|
-
if (id === undefined) {
|
|
252
|
-
return;
|
|
253
|
-
}
|
|
254
|
-
try {
|
|
255
|
-
await this.client.deleteRequestLock(id);
|
|
256
|
-
}
|
|
257
|
-
catch {
|
|
258
|
-
this.log.debug('Failed to delete request lock', { id, uniqueKey });
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
async _listHeadAndLock() {
|
|
262
|
-
// Make a copy so that we can clear the flag only if the whole method executes after the flag was set
|
|
263
|
-
// (i.e, it was not set in the middle of the execution of the method)
|
|
264
|
-
const shouldCheckForForefrontRequests = this.shouldCheckForForefrontRequests;
|
|
265
|
-
// NOTE: in theory, if we're not checking for forefront requests, we could fetch just enough requests to fill the local queue head to the limit.
|
|
266
|
-
// We chose not to do this because 1. it's simpler and 2. the queue is being processed while we're fetching and we want to avoid underruns.
|
|
267
|
-
// If we are checking for forefront requests, we need to fetch enough requests to be sure that we won't miss any new forefront ones.
|
|
268
|
-
const headData = await this.client.listAndLockHead({
|
|
269
|
-
limit: LIST_AND_LOCK_HEAD_LIMIT,
|
|
270
|
-
lockSecs: this.requestLockSecs,
|
|
271
|
-
});
|
|
272
|
-
this.queueHasLockedRequests = headData.queueHasLockedRequests;
|
|
273
|
-
const headIdBuffer = [];
|
|
274
|
-
const forefrontHeadIdBuffer = [];
|
|
275
|
-
// Go through the fetched requests, ensure they are cached locally and sort them into normal and forefront groups
|
|
276
|
-
for (const { id, uniqueKey } of headData.items) {
|
|
277
|
-
if (!id || !uniqueKey) {
|
|
278
|
-
this.log.warning(`Skipping request from queue head as it's invalid. Please report this with the provided metadata!`, {
|
|
279
|
-
id,
|
|
280
|
-
uniqueKey,
|
|
281
|
-
});
|
|
282
|
-
// Remove the lock from the request for now, so that it can be picked up later
|
|
283
|
-
// This may/may not succeed, but that's fine
|
|
284
|
-
await this.giveUpLock(id, uniqueKey);
|
|
285
|
-
continue;
|
|
286
|
-
}
|
|
287
|
-
// If we remember that we added the request ourselves and we added it to the forefront,
|
|
288
|
-
// we will put it to the beginning of the local queue head to preserve the expected order.
|
|
289
|
-
// If we do not remember that, we will enqueue it normally.
|
|
290
|
-
const forefront = this.requestCache.get(getRequestId(uniqueKey))?.forefront ?? false;
|
|
291
|
-
if (forefront) {
|
|
292
|
-
forefrontHeadIdBuffer.unshift(id);
|
|
293
|
-
}
|
|
294
|
-
else {
|
|
295
|
-
headIdBuffer.push(id);
|
|
296
|
-
}
|
|
297
|
-
// Ensure that the request is cached locally
|
|
298
|
-
this._cacheRequest(getRequestId(uniqueKey), {
|
|
299
|
-
requestId: id,
|
|
300
|
-
uniqueKey,
|
|
301
|
-
wasAlreadyPresent: true,
|
|
302
|
-
wasAlreadyHandled: false,
|
|
303
|
-
forefront,
|
|
304
|
-
});
|
|
305
|
-
}
|
|
306
|
-
// Insert the newly fetched requests into the local queue head
|
|
307
|
-
for (const id of headIdBuffer) {
|
|
308
|
-
this.queueHeadIds.add(id, id, false);
|
|
309
|
-
}
|
|
310
|
-
for (const id of forefrontHeadIdBuffer) {
|
|
311
|
-
this.queueHeadIds.add(id, id, true);
|
|
312
|
-
}
|
|
313
|
-
// Unlock and forget requests that would make the local queue head grow over the limit
|
|
314
|
-
const toUnlock = [];
|
|
315
|
-
const limit = shouldCheckForForefrontRequests
|
|
316
|
-
? LIST_AND_LOCK_HEAD_LIMIT // we may have received up to LIST_AND_LOCK_HEAD_LIMIT newly added forefront requests - we need to make sure that anything we already had in the queue gets unlocked
|
|
317
|
-
: LIST_AND_LOCK_HEAD_LIMIT + QUEUE_HEAD_REFILL_THRESHOLD; // we tolerate up to QUEUE_HEAD_REFILL_THRESHOLD additional requests to avoid frequent, yet unnecessary unlocks
|
|
318
|
-
while (this.queueHeadIds.length() > limit) {
|
|
319
|
-
toUnlock.push(this.queueHeadIds.removeLast());
|
|
320
|
-
}
|
|
321
|
-
if (toUnlock.length > 0) {
|
|
322
|
-
await Promise.all(toUnlock.map(async (id) => await this.giveUpLock(id)));
|
|
323
|
-
}
|
|
324
|
-
// We went through the whole procedure after `this.shouldCheckForForefrontRequests` was set -> we can clear the flag now
|
|
325
|
-
if (shouldCheckForForefrontRequests) {
|
|
326
|
-
this.shouldCheckForForefrontRequests = false;
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
async getOrHydrateRequest(requestId) {
|
|
330
|
-
checkStorageAccess();
|
|
331
|
-
const cachedEntry = this.requestCache.get(requestId);
|
|
332
|
-
if (!cachedEntry) {
|
|
333
|
-
// 2.1. Attempt to prolong the request lock to see if we still own the request
|
|
334
|
-
const prolongResult = await this._prolongRequestLock(requestId);
|
|
335
|
-
if (!prolongResult) {
|
|
336
|
-
return null;
|
|
337
|
-
}
|
|
338
|
-
// 2.1.1. If successful, hydrate the request and return it
|
|
339
|
-
const hydratedRequest = await this.getRequest(requestId);
|
|
340
|
-
// Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
|
|
341
|
-
if (!hydratedRequest) {
|
|
342
|
-
// Remove the lock from the request for now, so that it can be picked up later
|
|
343
|
-
// This may/may not succeed, but that's fine
|
|
344
|
-
try {
|
|
345
|
-
await this.client.deleteRequestLock(requestId);
|
|
346
|
-
}
|
|
347
|
-
catch {
|
|
348
|
-
// Ignore
|
|
349
|
-
}
|
|
350
|
-
return null;
|
|
351
|
-
}
|
|
352
|
-
this.requestCache.add(requestId, {
|
|
353
|
-
id: requestId,
|
|
354
|
-
uniqueKey: hydratedRequest.uniqueKey,
|
|
355
|
-
hydrated: hydratedRequest,
|
|
356
|
-
isHandled: hydratedRequest.handledAt !== null,
|
|
357
|
-
lockExpiresAt: prolongResult.getTime(),
|
|
358
|
-
forefront: false,
|
|
359
|
-
});
|
|
360
|
-
return hydratedRequest;
|
|
361
|
-
}
|
|
362
|
-
// 1.1. If hydrated, prolong the lock more and return it
|
|
363
|
-
if (cachedEntry.hydrated) {
|
|
364
|
-
// 1.1.1. If the lock expired on the hydrated requests, try to prolong. If we fail, we lost the request (or it was handled already)
|
|
365
|
-
if (cachedEntry.lockExpiresAt && cachedEntry.lockExpiresAt < Date.now()) {
|
|
366
|
-
const prolonged = await this._prolongRequestLock(cachedEntry.id);
|
|
367
|
-
if (!prolonged) {
|
|
368
|
-
return null;
|
|
369
|
-
}
|
|
370
|
-
cachedEntry.lockExpiresAt = prolonged.getTime();
|
|
371
|
-
}
|
|
372
|
-
return cachedEntry.hydrated;
|
|
373
|
-
}
|
|
374
|
-
// 1.2. If not hydrated, try to prolong the lock first (to ensure we keep it in our queue), hydrate and return it
|
|
375
|
-
const prolonged = await this._prolongRequestLock(cachedEntry.id);
|
|
376
|
-
if (!prolonged) {
|
|
377
|
-
return null;
|
|
378
|
-
}
|
|
379
|
-
// This might still return null if the queue head is inconsistent with the main queue table.
|
|
380
|
-
const hydratedRequest = await this.getRequest(cachedEntry.id);
|
|
381
|
-
cachedEntry.hydrated = hydratedRequest;
|
|
382
|
-
// Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
|
|
383
|
-
if (!hydratedRequest) {
|
|
384
|
-
// Remove the lock from the request for now, so that it can be picked up later
|
|
385
|
-
// This may/may not succeed, but that's fine
|
|
386
|
-
try {
|
|
387
|
-
await this.client.deleteRequestLock(cachedEntry.id);
|
|
388
|
-
}
|
|
389
|
-
catch {
|
|
390
|
-
// Ignore
|
|
391
|
-
}
|
|
392
|
-
return null;
|
|
393
|
-
}
|
|
394
|
-
return hydratedRequest;
|
|
395
|
-
}
|
|
396
|
-
async _prolongRequestLock(requestId) {
|
|
397
|
-
try {
|
|
398
|
-
const res = await this.client.prolongRequestLock(requestId, { lockSecs: this.requestLockSecs });
|
|
399
|
-
return res.lockExpiresAt;
|
|
400
|
-
}
|
|
401
|
-
catch (err) {
|
|
402
|
-
// Most likely we do not own the lock anymore
|
|
403
|
-
this.log.warning(`Failed to prolong lock for cached request ${requestId}, either lost the lock or the request was already handled\n`, {
|
|
404
|
-
err,
|
|
405
|
-
});
|
|
406
|
-
return null;
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
_reset() {
|
|
410
|
-
super._reset();
|
|
411
|
-
this.listHeadAndLockPromise = null;
|
|
412
|
-
this.queueHasLockedRequests = undefined;
|
|
413
|
-
}
|
|
414
|
-
_maybeAddRequestToQueueHead() {
|
|
415
|
-
// Do nothing for request queue v2, as we are only able to lock requests when listing the head
|
|
416
|
-
}
|
|
417
|
-
async _clearPossibleLocks() {
|
|
418
|
-
this.queuePausedForMigration = true;
|
|
419
|
-
let requestId;
|
|
420
|
-
// eslint-disable-next-line no-cond-assign
|
|
421
|
-
while ((requestId = this.queueHeadIds.removeFirst()) !== null) {
|
|
422
|
-
try {
|
|
423
|
-
await this.client.deleteRequestLock(requestId);
|
|
424
|
-
}
|
|
425
|
-
catch {
|
|
426
|
-
// We don't have the lock, or the request was never locked. Either way it's fine
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
/**
|
|
431
|
-
* @inheritDoc
|
|
432
|
-
*/
|
|
433
|
-
static async open(...args) {
|
|
434
|
-
return super.open(...args);
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
//# sourceMappingURL=request_queue_v2.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue_v2.js","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAO1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,gHAAgH;AAChH,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC,MAAM,wBAAwB,GAAG,EAAE,CAAC;AAEpC,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAEtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,OAAO,YAAa,SAAQ,eAAe;IACrC,sBAAsB,GAAyB,IAAI,CAAC;IACpD,sBAAsB,GAAwB,SAAS,CAAC;IACxD,+BAA+B,GAAG,KAAK,CAAC;IACxC,oBAAoB,GAAG,CAAC,CAAC;IAEjC,YAAY,OAA+B,EAAE,SAAwB,cAAc,CAAC,gBAAgB,EAAE;QAClG,KAAK,CACD;YACI,GAAG,OAAO;YACV,SAAS,EAAE,eAAe;YAC1B,8BAA8B,EAAE,2BAA2B;YAC3D,mBAAmB,EAAE,mBAAmB;SAC3C,EACD,MAAM,CACT,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,EAAE,wCAAsB,KAAK,IAAI,EAAE;YAC3C,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,sCAAqB,KAAK,IAAI,EAAE;YAC1C,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACrC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACgB,aAAa,CAAC,QAAgB,EAAE,kBAA6C;QAC5F,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAElD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAEvD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,kBAAkB,CAAC,SAAS,EAAE;YAChD,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,IAAI;SACtB,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,UAAU,CACrB,WAAmB,EACnB,UAAwC,EAAE;QAE1C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACjD,IAAI,CAAC,+BAA+B,GAAG,IAAI,CAAC;QAChD,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,WAAW,CACtB,YAA0B,EAC1B,UAAwC,EAAE;QAE1C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAC9D,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,iBAAiB,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAClD,IAAI,CAAC,+BAA+B,GAAG,IAAI,CAAC;gBAC5C,MAAM;YACV,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,gBAAgB;QAC3B,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAEtD,oCAAoC;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,OAAO,GAAmB,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC;QAE9E,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uFAAuF,EAAE;gBACpG,aAAa;aAChB,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;QAE/B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QAC9C,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;QAC/B,OAAO,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,UAAU;QACrB,2EAA2E;QAC3E,IAAI,IAAI,CAAC,2BAA2B,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,oHAAoH;QACpH,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,kEAAkE;QAClE,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,qDAAqD;QACrD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,oHAAoH;QACpH,IAAI,IAAI,CAAC,sBAAsB,KAAK,SAAS,EAAE,CAAC;YAC5C,yFAAyF;YACzF,IACI,IAAI,CAAC,sBAAsB;gBAC3B,IAAI,CAAC,oBAAoB,KAAK,CAAC;gBAC/B,EAAE,IAAI,CAAC,oCAAoC,GAAG,EAAE,KAAK,CAAC,EACxD,CAAC;gBACC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;YAChF,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC;QACxC,CAAC;QAED,8KAA8K;QAE9K,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAE7D,IAAI,WAAW,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,sJAAsJ;QACtJ,gIAAgI;QAChI,2DAA2D;QAC3D,4JAA4J;QAC5J,IAAI,EAAE,IAAI,CAAC,oCAAoC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACzD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAE3G,IAAI,CAAC,GAAG,CAAC,IAAI,CACT,oGAAoG,EACpG;gBACI,QAAQ,EAAE,QAAQ;qBACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACP,IAAI,CAAC,CAAC,EAAE,CAAC;wBACL,OAAO,IAAI,CAAC;oBAChB,CAAC;oBAED,OAAO;wBACH,EAAE,EAAE,CAAC,CAAC,EAAE;wBACR,aAAa,EAAE,CAAC,CAAC,aAAa;wBAC9B,QAAQ,EAAE,CAAC,CAAC,YAAY;qBAC3B,CAAC;gBACN,CAAC,CAAC;qBACD,MAAM,CAAC,OAAO,CAAC;gBACpB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CACJ,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,oGAAoG,EACpG;gBACI,UAAU,EAAE,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;aACvD,CACJ,CAAC;QACN,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,cAAc,CACzB,GAAG,IAAmD;QAEtD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;QAEhD,IAAI,GAAG,EAAE,CAAC;YACN,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;YAEhC,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;gBACrB,IAAI,CAAC,+BAA+B,GAAG,IAAI,CAAC;YAChD,CAAC;YAED,6CAA6C;YAC7C,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,KAAK,EAAE,CAAC,CAAC;YACjG,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,6CAA6C,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACvF,CAAC;QACL,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAES,KAAK,CAAC,oBAAoB;QAChC,kBAAkB,EAAE,CAAC;QAErB,+CAA+C;QAC/C,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO;QACX,CAAC;QAED,uDAAuD;QACvD,gHAAgH;QAChH,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,2BAA2B,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC;YACpG,OAAO;QACX,CAAC;QAED,IAAI,CAAC,sBAAsB,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACjE,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,sBAAsB,CAAC;IACtC,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,EAAW,EAAE,SAAkB;QACpD,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACL,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC1B,qGAAqG;QACrG,qEAAqE;QACrE,MAAM,+BAA+B,GAAG,IAAI,CAAC,+BAA+B,CAAC;QAE7E,gJAAgJ;QAChJ,2IAA2I;QAC3I,oIAAoI;QACpI,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC;YAC/C,KAAK,EAAE,wBAAwB;YAC/B,QAAQ,EAAE,IAAI,CAAC,eAAe;SACjC,CAAC,CAAC;QAEH,IAAI,CAAC,sBAAsB,GAAG,QAAQ,CAAC,sBAAsB,CAAC;QAE9D,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,MAAM,qBAAqB,GAAG,EAAE,CAAC;QAEjC,iHAAiH;QACjH,KAAK,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,kGAAkG,EAClG;oBACI,EAAE;oBACF,SAAS;iBACZ,CACJ,CAAC;gBAEF,8EAA8E;gBAC9E,4CAA4C;gBAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;gBACrC,SAAS;YACb,CAAC;YAED,uFAAuF;YACvF,0FAA0F;YAC1F,2DAA2D;YAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;YACrF,IAAI,SAAS,EAAE,CAAC;gBACZ,qBAAqB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACJ,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;YAED,4CAA4C;YAC5C,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE;gBACxC,SAAS,EAAE,EAAE;gBACb,SAAS;gBACT,iBAAiB,EAAE,IAAI;gBACvB,iBAAiB,EAAE,KAAK;gBACxB,SAAS;aACZ,CAAC,CAAC;QACP,CAAC;QAED,8DAA8D;QAC9D,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;YAC5B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;QACzC,CAAC;QAED,KAAK,MAAM,EAAE,IAAI,qBAAqB,EAAE,CAAC;YACrC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QACxC,CAAC;QAED,sFAAsF;QACtF,MAAM,QAAQ,GAAG,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,+BAA+B;YACzC,CAAC,CAAC,wBAAwB,CAAC,oKAAoK;YAC/L,CAAC,CAAC,wBAAwB,GAAG,2BAA2B,CAAC,CAAC,+GAA+G;QAC7K,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAG,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,wHAAwH;QACxH,IAAI,+BAA+B,EAAE,CAAC;YAClC,IAAI,CAAC,+BAA+B,GAAG,KAAK,CAAC;QACjD,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,SAAiB;QAEjB,kBAAkB,EAAE,CAAC;QAErB,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAErD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,8EAA8E;YAC9E,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAEhE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,0DAA0D;YAC1D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAEzD,sIAAsI;YACtI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACnB,8EAA8E;gBAC9E,4CAA4C;gBAC5C,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;gBACnD,CAAC;gBAAC,MAAM,CAAC;oBACL,SAAS;gBACb,CAAC;gBAED,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC7B,EAAE,EAAE,SAAS;gBACb,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,QAAQ,EAAE,eAAe;gBACzB,SAAS,EAAE,eAAe,CAAC,SAAS,KAAK,IAAI;gBAC7C,aAAa,EAAE,aAAa,CAAC,OAAO,EAAE;gBACtC,SAAS,EAAE,KAAK;aACnB,CAAC,CAAC;YAEH,OAAO,eAAe,CAAC;QAC3B,CAAC;QAED,wDAAwD;QACxD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;YACvB,mIAAmI;YACnI,IAAI,WAAW,CAAC,aAAa,IAAI,WAAW,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBACtE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBAChB,CAAC;gBAED,WAAW,CAAC,aAAa,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;YACpD,CAAC;YAED,OAAO,WAAW,CAAC,QAAQ,CAAC;QAChC,CAAC;QAED,iHAAiH;QACjH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,4FAA4F;QAC5F,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAE9D,WAAW,CAAC,QAAQ,GAAG,eAAe,CAAC;QAEvC,sIAAsI;QACtI,IAAI,CAAC,eAAe,EAAE,CAAC;YACnB,8EAA8E;YAC9E,4CAA4C;YAC5C,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YACxD,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;YAED,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,eAAe,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,SAAiB;QAC/C,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;YAChG,OAAO,GAAG,CAAC,aAAa,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,6CAA6C;YAC7C,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,6CAA6C,SAAS,6DAA6D,EACnH;gBACI,GAAG;aACN,CACJ,CAAC;YAEF,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEkB,MAAM;QACrB,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACnC,IAAI,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAC5C,CAAC;IAEkB,2BAA2B;QAC1C,8FAA8F;IAClG,CAAC;IAES,KAAK,CAAC,mBAAmB;QAC/B,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACpC,IAAI,SAAwB,CAAC;QAE7B,0CAA0C;QAC1C,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5D,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACnD,CAAC;YAAC,MAAM,CAAC;gBACL,gFAAgF;YACpF,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAU,KAAK,CAAC,IAAI,CAAC,GAAG,IAA6C;QACvE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAA0B,CAAC;IACxD,CAAC;CACJ"}
|