@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.70

package/storages/request_provider.js

@@ -1,555 +0,0 @@
-import { inspect } from 'node:util';
-import { chunk, downloadListOfUrls, sleep } from '@crawlee/utils';
-import ow from 'ow';
-import { ListDictionary, LruCache } from '@apify/datastructures';
-import { cryptoRandomObjectId } from '@apify/utilities';
-import { Configuration } from '../configuration.js';
-import { log } from '../log.js';
-import { Request } from '../request.js';
-import { checkStorageAccess } from './access_checking.js';
-import { StorageManager } from './storage_manager.js';
-import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './utils.js';
-export class RequestProvider {
-    config;
-    id;
-    name;
-    timeoutSecs = 30;
-    clientKey = cryptoRandomObjectId();
-    client;
-    proxyConfiguration;
-    log;
-    internalTimeoutMillis = 5 * 60_000; // defaults to 5 minutes, will be overridden by BasicCrawler
-    requestLockSecs = 3 * 60; // defaults to 3 minutes, will be overridden by BasicCrawler
-    // We can trust these numbers only in a case that queue is used by a single client.
-    // This information is returned by getHead() under the hadMultipleClients property.
-    assumedTotalCount = 0;
-    assumedHandledCount = 0;
-    initialCount = 0;
-    queueHeadIds = new ListDictionary();
-    requestCache;
-    recentlyHandledRequestsCache;
-    queuePausedForMigration = false;
-    lastActivity = new Date();
-    isFinishedCalledWhileHeadWasNotEmpty = 0;
-    inProgressRequestBatchCount = 0;
-    constructor(options, config = Configuration.getGlobalConfig()) {
-        this.config = config;
-        this.id = options.id;
-        this.name = options.name;
-        this.client = options.client.requestQueue(this.id, {
-            clientKey: this.clientKey,
-            timeoutSecs: this.timeoutSecs,
-        });
-        this.proxyConfiguration = options.proxyConfiguration;
-        this.requestCache = new LruCache({ maxLength: options.requestCacheMaxSize });
-        this.recentlyHandledRequestsCache = new LruCache({ maxLength: options.recentlyHandledRequestsMaxSize });
-        this.log = log.child({ prefix: `${options.logPrefix}(${this.id}, ${this.name ?? 'no-name'})` });
-        const eventManager = config.getEventManager();
-        eventManager.on("migrating" /* EventType.MIGRATING */, async () => {
-            this.queuePausedForMigration = true;
-        });
-    }
-    /**
-     * Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).
-     *
-     * Survives restarts and actor migrations.
-     */
-    getTotalCount() {
-        return this.assumedTotalCount + this.initialCount;
-    }
-    /**
-     * Adds a request to the queue.
-     *
-     * If a request with the same `uniqueKey` property is already present in the queue,
-     * it will not be updated. You can find out whether this happened from the resulting
-     * {@link QueueOperationInfo} object.
-     *
-     * To add multiple requests to the queue by extracting links from a webpage,
-     * see the {@link enqueueLinks} helper function.
-     *
-     * @param requestLike {@link Request} object or vanilla object with request data.
-     * Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
-     * @param [options] Request queue operation options.
-     */
-    async addRequest(requestLike, options = {}) {
-        checkStorageAccess();
-        this.lastActivity = new Date();
-        ow(requestLike, ow.object);
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-        }));
-        const { forefront = false } = options;
-        if ('requestsFromUrl' in requestLike) {
-            const requests = await this._fetchRequestsFromUrl(requestLike);
-            const processedRequests = await this._addFetchedRequests(requestLike, requests, options);
-            return { ...processedRequests[0], forefront };
-        }
-        ow(requestLike, ow.object.partialShape({
-            url: ow.string,
-            id: ow.undefined,
-        }));
-        const request = requestLike instanceof Request ? requestLike : new Request(requestLike);
-        const cacheKey = getRequestId(request.uniqueKey);
-        const cachedInfo = this.requestCache.get(cacheKey);
-        if (cachedInfo) {
-            request.id = cachedInfo.id;
-            return {
-                wasAlreadyPresent: true,
-                // We may assume that if request is in local cache then also the information if the
-                // request was already handled is there because just one client should be using one queue.
-                wasAlreadyHandled: cachedInfo.isHandled,
-                requestId: cachedInfo.id,
-                uniqueKey: cachedInfo.uniqueKey,
-                forefront,
-            };
-        }
-        const queueOperationInfo = {
-            ...(await this.client.addRequest(request, { forefront })),
-            uniqueKey: request.uniqueKey,
-            forefront,
-        };
-        const { requestId, wasAlreadyPresent } = queueOperationInfo;
-        this._cacheRequest(cacheKey, queueOperationInfo);
-        if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
-            this.assumedTotalCount++;
-            // Performance optimization: add request straight to head if possible
-            this._maybeAddRequestToQueueHead(requestId, forefront);
-        }
-        return queueOperationInfo;
-    }
-    /**
-     * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
-     * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`
-     * if you don't want to block the processing, as those methods will only wait for the initial 1000 requests,
-     * start processing right after that happens, and continue adding more in the background.
-     *
-     * If a request passed in is already present due to its `uniqueKey` property being the same,
-     * it will not be updated. You can find out whether this happened by finding the request in the resulting
-     * {@link BatchAddRequestsResult} object.
-     *
-     * @param requestsLike {@link Request} objects or vanilla objects with request data.
-     * Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
-     * @param [options] Request queue operation options.
-     */
-    async addRequests(requestsLike, options = {}) {
-        checkStorageAccess();
-        this.lastActivity = new Date();
-        ow(requestsLike, ow.array);
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-            cache: ow.optional.boolean,
-        }));
-        const { forefront = false, cache = true } = options;
-        const uniqueKeyToCacheKey = new Map();
-        const getCachedRequestId = (uniqueKey) => {
-            const cached = uniqueKeyToCacheKey.get(uniqueKey);
-            if (cached)
-                return cached;
-            const newCacheKey = getRequestId(uniqueKey);
-            uniqueKeyToCacheKey.set(uniqueKey, newCacheKey);
-            return newCacheKey;
-        };
-        const results = {
-            processedRequests: [],
-            unprocessedRequests: [],
-        };
-        for (const requestLike of requestsLike) {
-            if ('requestsFromUrl' in requestLike) {
-                const requests = await this._fetchRequestsFromUrl(requestLike);
-                await this._addFetchedRequests(requestLike, requests, options);
-            }
-        }
-        const requests = requestsLike
-            .filter((requestLike) => !('requestsFromUrl' in requestLike))
-            .map((requestLike) => {
-            return requestLike instanceof Request ? requestLike : new Request(requestLike);
-        });
-        const requestsToAdd = new Map();
-        for (const request of requests) {
-            const cacheKey = getCachedRequestId(request.uniqueKey);
-            const cachedInfo = this.requestCache.get(cacheKey);
-            if (cachedInfo) {
-                request.id = cachedInfo.id;
-                results.processedRequests.push({
-                    wasAlreadyPresent: true,
-                    // We may assume that if request is in local cache then also the information if the
-                    // request was already handled is there because just one client should be using one queue.
-                    wasAlreadyHandled: cachedInfo.isHandled,
-                    requestId: cachedInfo.id,
-                    uniqueKey: cachedInfo.uniqueKey,
-                });
-            }
-            else if (!requestsToAdd.has(request.uniqueKey)) {
-                requestsToAdd.set(request.uniqueKey, request);
-            }
-        }
-        // Early exit if all provided requests were already added
-        if (!requestsToAdd.size) {
-            return results;
-        }
-        const apiResults = await this.client.batchAddRequests([...requestsToAdd.values()], { forefront });
-        // Report unprocessed requests
-        results.unprocessedRequests = apiResults.unprocessedRequests;
-        // Add all new requests to the requestCache
-        for (const newRequest of apiResults.processedRequests) {
-            // Add the new request to the processed list
-            results.processedRequests.push(newRequest);
-            const cacheKey = getCachedRequestId(newRequest.uniqueKey);
-            const { requestId, wasAlreadyPresent } = newRequest;
-            if (cache) {
-                this._cacheRequest(cacheKey, { ...newRequest, forefront });
-            }
-            if (!wasAlreadyPresent && !this.recentlyHandledRequestsCache.get(requestId)) {
-                this.assumedTotalCount++;
-                // Performance optimization: add request straight to head if possible
-                this._maybeAddRequestToQueueHead(requestId, forefront);
-            }
-        }
-        return results;
-    }
-    /**
-     * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
-     * adding the rest in the background. You can configure the batch size via `batchSize` option and the sleep time in between
-     * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
-     * the `waitForAllRequestsToBeAdded` promise you get in the response object.
-     *
-     * @param requests The requests to add
-     * @param options Options for the request queue
-     */
-    async addRequestsBatched(requests, options = {}) {
-        checkStorageAccess();
-        this.lastActivity = new Date();
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-            waitForAllRequestsToBeAdded: ow.optional.boolean,
-            batchSize: ow.optional.number,
-            waitBetweenBatchesMillis: ow.optional.number,
-        }));
-        // The `requests` array can be huge, and `ow` is very slow for anything more complex.
-        // This explicit iteration takes a few milliseconds, while the ow check can take tens of seconds.
-        // ow(requests, ow.array.ofType(ow.any(
-        //     ow.string,
-        //     ow.object.partialShape({ url: ow.string, id: ow.undefined }),
-        //     ow.object.partialShape({ requestsFromUrl: ow.string, regex: ow.optional.regExp }),
-        // )));
-        for (const request of requests) {
-            if (typeof request === 'string') {
-                continue;
-            }
-            if (typeof request === 'object' && request !== null) {
-                if (typeof request.url === 'string' && typeof request.id === 'undefined') {
-                    continue;
-                }
-                if (typeof request.requestsFromUrl === 'string') {
-                    continue;
-                }
-            }
-            throw new Error(`Request options are not valid, provide either a URL or an object with 'url' property (but without 'id' property), or an object with 'requestsFromUrl' property. Input: ${inspect(request)}`);
-        }
-        const { batchSize = 1000, waitBetweenBatchesMillis = 1000 } = options;
-        const sources = [];
-        for (const opts of requests) {
-            if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
-                await this.addRequest(opts, { forefront: options.forefront });
-            }
-            else {
-                sources.push(typeof opts === 'string' ? { url: opts } : opts);
-            }
-        }
-        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true) => {
-            const resultsToReturn = [];
-            const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
-            resultsToReturn.push(...apiResult.processedRequests);
-            if (apiResult.unprocessedRequests.length) {
-                await sleep(waitBetweenBatchesMillis);
-                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false)));
-            }
-            return resultsToReturn;
-        };
-        const initialChunk = sources.splice(0, batchSize);
-        // Add initial batch of `batchSize` to process them right away
-        const addedRequests = await attemptToAddToQueueAndAddAnyUnprocessed(initialChunk);
-        // If we have no more requests to add, return early
-        if (!sources.length) {
-            return {
-                addedRequests,
-                waitForAllRequestsToBeAdded: Promise.resolve([]),
-            };
-        }
-        // eslint-disable-next-line no-async-promise-executor
-        const promise = new Promise(async (resolve) => {
-            const chunks = chunk(sources, batchSize);
-            const finalAddedRequests = [];
-            for (const requestChunk of chunks) {
-                finalAddedRequests.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(requestChunk, false)));
-                await sleep(waitBetweenBatchesMillis);
-            }
-            resolve(finalAddedRequests);
-        });
-        this.inProgressRequestBatchCount += 1;
-        void promise.finally(() => {
-            this.inProgressRequestBatchCount -= 1;
-        });
-        // If the user wants to wait for all the requests to be added, we wait for the promise to resolve for them
-        if (options.waitForAllRequestsToBeAdded) {
-            addedRequests.push(...(await promise));
-        }
-        return {
-            addedRequests,
-            waitForAllRequestsToBeAdded: promise,
-        };
-    }
-    /**
-     * Gets the request from the queue specified by ID.
-     *
-     * @param id ID of the request.
-     * @returns Returns the request object, or `null` if it was not found.
-     */
-    async getRequest(id) {
-        checkStorageAccess();
-        ow(id, ow.string);
-        const requestOptions = await this.client.getRequest(id);
-        if (!requestOptions)
-            return null;
-        return new Request(requestOptions);
-    }
-    /**
-     * Marks a request that was previously returned by the
-     * {@link RequestQueue.fetchNextRequest}
-     * function as handled after successful processing.
-     * Handled requests will never again be returned by the `fetchNextRequest` function.
-     */
-    async markRequestHandled(request) {
-        checkStorageAccess();
-        this.lastActivity = new Date();
-        ow(request, ow.object.partialShape({
-            id: ow.string,
-            uniqueKey: ow.string,
-            handledAt: ow.optional.string,
-        }));
-        const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
-        const handledAt = request.handledAt ?? new Date().toISOString();
-        const queueOperationInfo = {
-            ...(await this.client.updateRequest({
-                ...request,
-                handledAt,
-            })),
-            uniqueKey: request.uniqueKey,
-            forefront,
-        };
-        request.handledAt = handledAt;
-        this.recentlyHandledRequestsCache.add(request.id, true);
-        if (!queueOperationInfo.wasAlreadyHandled) {
-            this.assumedHandledCount++;
-        }
-        this.queueHeadIds.remove(request.id);
-        this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
-        return queueOperationInfo;
-    }
-    /**
-     * Reclaims a failed request back to the queue, so that it can be returned for processing later again
-     * by another call to {@link RequestQueue.fetchNextRequest}.
-     * The request record in the queue is updated using the provided `request` parameter.
-     * For example, this lets you store the number of retries or error messages for the request.
-     */
-    async reclaimRequest(request, options = {}) {
-        checkStorageAccess();
-        this.lastActivity = new Date();
-        ow(request, ow.object.partialShape({
-            id: ow.string,
-            uniqueKey: ow.string,
-        }));
-        ow(options, ow.object.exactShape({
-            forefront: ow.optional.boolean,
-        }));
-        const { forefront = false } = options;
-        // TODO: If request hasn't been changed since the last getRequest(),
-        //   we don't need to call updateRequest() and thus improve performance.
-        const queueOperationInfo = {
-            ...(await this.client.updateRequest(request, {
-                forefront,
-            })),
-            uniqueKey: request.uniqueKey,
-            forefront,
-        };
-        this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
-        return queueOperationInfo;
-    }
-    /**
-     * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest}
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the queue is empty, there might be some pending requests currently being processed.
-     * If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
-     */
-    async isEmpty() {
-        await this.ensureHeadIsNonEmpty();
-        return this.queueHeadIds.length() === 0;
-    }
-    _reset() {
-        this.lastActivity = new Date();
-        this.queueHeadIds.clear();
-        this.recentlyHandledRequestsCache.clear();
-        this.assumedTotalCount = 0;
-        this.assumedHandledCount = 0;
-        this.requestCache.clear();
-    }
-    /**
-     * Caches information about request to beware of unneeded addRequest() calls.
-     */
-    _cacheRequest(cacheKey, queueOperationInfo) {
-        // Remove the previous entry, as otherwise our cache will never update 👀
-        this.requestCache.remove(cacheKey);
-        this.requestCache.add(cacheKey, {
-            id: queueOperationInfo.requestId,
-            isHandled: queueOperationInfo.wasAlreadyHandled,
-            uniqueKey: queueOperationInfo.uniqueKey,
-            hydrated: null,
-            lockExpiresAt: null,
-            forefront: queueOperationInfo.forefront,
-        });
-    }
-    /**
-     * Adds a request straight to the queueHeadDict, to improve performance.
-     */
-    _maybeAddRequestToQueueHead(requestId, forefront) {
-        if (forefront) {
-            this.queueHeadIds.add(requestId, requestId, true);
-        }
-        else if (this.assumedTotalCount < QUERY_HEAD_MIN_LENGTH) {
-            this.queueHeadIds.add(requestId, requestId, false);
-        }
-    }
-    /**
-     * Removes the queue either from the Apify Cloud storage or from the local database,
-     * depending on the mode of operation.
-     */
-    async drop() {
-        checkStorageAccess();
-        await this.client.delete();
-        const manager = StorageManager.getManager(this.constructor, this.config);
-        manager.closeStorage(this);
-    }
-    /**
-     * Returns the number of handled requests.
-     *
-     * This function is just a convenient shortcut for:
-     *
-     * ```javascript
-     * const { handledRequestCount } = await queue.getInfo();
-     * ```
-     */
-    async handledCount() {
-        // NOTE: We keep this function for compatibility with RequestList.handledCount()
-        const { handledRequestCount } = (await this.getInfo()) ?? {};
-        return handledRequestCount ?? 0;
-    }
-    /**
-     * Returns an object containing general information about the request queue.
-     *
-     * The function returns the same object as the Apify API Client's
-     * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
-     * function, which in turn calls the
-     * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
-     * API endpoint.
-     *
-     * **Example:**
-     * ```
-     * {
-     *   id: "WkzbQMuFYuamGv3YF",
-     *   name: "my-queue",
-     *   userId: "wRsJZtadYvn4mBZmm",
-     *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
-     *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
-     *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
-     *   totalRequestCount: 25,
-     *   handledRequestCount: 5,
-     *   pendingRequestCount: 20,
-     * }
-     * ```
-     */
-    async getInfo() {
-        checkStorageAccess();
-        return this.client.get();
-    }
-    /**
-     * Fetches URLs from requestsFromUrl and returns them in format of list of requests
-     */
-    async _fetchRequestsFromUrl(source) {
-        const { requestsFromUrl, regex, ...sharedOpts } = source;
-        // Download remote resource and parse URLs.
-        let urlsArr;
-        try {
-            urlsArr = await this._downloadListOfUrls({
-                url: requestsFromUrl,
-                urlRegExp: regex,
-                proxyUrl: await this.proxyConfiguration?.newUrl(),
-            });
-        }
-        catch (err) {
-            throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`);
-        }
-        // Skip if resource contained no URLs.
-        if (!urlsArr.length) {
-            this.log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex });
-            return [];
-        }
-        return urlsArr.map((url) => ({ url, ...sharedOpts }));
-    }
-    /**
-     * Adds all fetched requests from a URL from a remote resource.
-     */
-    async _addFetchedRequests(source, fetchedRequests, options) {
-        const { requestsFromUrl, regex } = source;
-        const { addedRequests } = await this.addRequestsBatched(fetchedRequests, options);
-        this.log.info('Fetched and loaded Requests from a remote resource.', {
-            requestsFromUrl,
-            regex,
-            fetchedCount: fetchedRequests.length,
-            importedCount: addedRequests.length,
-            duplicateCount: fetchedRequests.length - addedRequests.length,
-            sample: JSON.stringify(fetchedRequests.slice(0, 5)),
-        });
-        return addedRequests;
-    }
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    async _downloadListOfUrls(options) {
-        return downloadListOfUrls(options);
-    }
-    /**
-     * Opens a request queue and returns a promise resolving to an instance
-     * of the {@link RequestQueue} class.
-     *
-     * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
-     * The queue is used for deep crawling of websites, where you start with several URLs and then
-     * recursively follow links to other pages. The data structure supports both breadth-first
-     * and depth-first crawling orders.
-     *
-     * For more details and code examples, see the {@link RequestQueue} class.
-     *
-     * @param [queueIdOrName]
-     *   ID or name of the request queue to be opened. If `null` or `undefined`,
-     *   the function returns the default request queue associated with the crawler run.
-     * @param [options] Open Request Queue options.
-     */
-    static async open(queueIdOrName, options = {}) {
-        checkStorageAccess();
-        ow(queueIdOrName, ow.optional.any(ow.string, ow.null));
-        ow(options, ow.object.exactShape({
-            config: ow.optional.object.instanceOf(Configuration),
-            storageClient: ow.optional.object,
-            proxyConfiguration: ow.optional.object,
-        }));
-        options.config ??= Configuration.getGlobalConfig();
-        options.storageClient ??= options.config.getStorageClient();
-        await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
-        const manager = StorageManager.getManager(this, options.config);
-        const queue = await manager.openStorage(queueIdOrName, options.storageClient);
-        queue.proxyConfiguration = options.proxyConfiguration;
-        // eslint-disable-next-line dot-notation
-        queue['initialCount'] = (await queue.client.get())?.totalRequestCount ?? 0;
-        return queue;
-    }
-}
-//# sourceMappingURL=request_provider.js.map

package/storages/request_provider.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"request_provider.js","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,OAAO,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAGhC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEvF,MAAM,OAAgB,eAAe;IAkCpB;IAjCb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,WAAW,GAAG,EAAE,CAAC;IACjB,SAAS,GAAG,oBAAoB,EAAE,CAAC;IACnC,MAAM,CAAqB;IACjB,kBAAkB,CAAsB;IAElD,GAAG,CAAM;IACT,qBAAqB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,4DAA4D;IAChG,eAAe,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,4DAA4D;IAEtF,mFAAmF;IACnF,mFAAmF;IACnF,iBAAiB,GAAG,CAAC,CAAC;IACtB,mBAAmB,GAAG,CAAC,CAAC;IAEhB,YAAY,GAAG,CAAC,CAAC;IAEf,YAAY,GAAG,IAAI,cAAc,EAAU,CAAC;IAC5C,YAAY,CAA2B;IAEvC,4BAA4B,CAAoB;IAEhD,uBAAuB,GAAG,KAAK,CAAC;IAEhC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;IAE1B,oCAAoC,GAAG,CAAC,CAAC;IAEzC,2BAA2B,GAAG,CAAC,CAAC;IAE1C,YACI,OAAuC,EAC9B,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC,4BAA4B,GAAG,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,8BAA8B,EAAE,CAAC,CAAC;QACxG,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,IAAI,SAAS,GAAG,EAAE,CAAC,CAAC;QAEhG,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAE9C,YAAY,CAAC,EAAE,wCAAsB,KAAK,IAAI,EAAE;YAC5C,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACH,aAAa;QACT,OAAO,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,UAAU,CACZ,WAAmB,EACnB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC3B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;YACjF,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE3G,OAAO,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;QAClD,CAAC;QAED,EAAE,CACE,WAAW,EACX,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,GAAG,EAAE,EAAE,CAAC,MAAM;YACd,EAAE,EAAE,EAAE,CAAC,SAAS;SACnB,CAAC,CACL,CAAC;QAEF,MAAM,OAAO,GAAG,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;QAExF,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEnD,IAAI,UAAU,EAAE,CAAC;YACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,IAAI;gBACvB,mFAAmF;gBACnF,0FAA0F;gBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;gBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,SAAS;aACZ,CAAC;QACN,CAAC;QAED,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;YACzD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QAEtC,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,kBAAkB,CAAC;QAC5D,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAEjD,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEzB,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,WAAW,CACb,YAAsB,EACtB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;QAC3B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtD,MAAM,kBAAkB,GAAG,CAAC,SAAiB,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAElD,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;YAE1B,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YAC5C,mBAAmB,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAEhD,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC;QAEF,MAAM,OAAO,GAA2B;YACpC,iBAAiB,EAAE,EAAE;YACrB,mBAAmB,EAAE,EAAE;SAC1B,CAAC;QAEF,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACrC,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;gBACnC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAA6B,CAAC,CAAC;gBACjF,MAAM,IAAI,CAAC,mBAAmB,CAAC,WAA6B,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrF,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,YAAY;aACxB,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,IAAI,WAAW,CAAC,CAAC;aAC5D,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YACjB,OAAO,WAAW,YAAY,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,WAA6B,CAAC,CAAC;QACrG,CAAC,CAAC,CAAC;QAEP,MAAM,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEnD,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC;oBAC3B,iBAAiB,EAAE,IAAI;oBACvB,mFAAmF;oBACnF,0FAA0F;oBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;oBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;oBACxB,SAAS,EAAE,UAAU,CAAC,SAAS;iBAClC,CAAC,CAAC;YACP,CAAC;iBAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YAClD,CAAC;QACL,CAAC;QAED,yDAAyD;QACzD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAElG,8BAA8B;QAC9B,OAAO,CAAC,mBAAmB,GAAG,UAAU,CAAC,mBAAmB,CAAC;QAE7D,2CAA2C;QAC3C,KAAK,MAAM,UAAU,IAAI,UAAU,CAAC,iBAAiB,EAAE,CAAC;YACpD,4CAA4C;YAC5C,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAE1D,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC;YAEpD,IAAI,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/D,CAAC;YAED,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1E,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAEzB,qEAAqE;gBACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAA6B,EAC7B,UAAqC,EAAE;QAEvC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC9B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAChD,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC/C,CAAC,CACL,CAAC;QAEF,qFAAqF;QACrF,iGAAiG;QAEjG,uCAAuC;QACvC,iBAAiB;QACjB,oEAAoE;QACpE,yFAAyF;QACzF,OAAO;QAEP,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC9B,SAAS;YACb,CAAC;YAED,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBAClD,IAAI,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,OAAO,CAAC,EAAE,KAAK,WAAW,EAAE,CAAC;oBACvE,SAAS;gBACb,CAAC;gBAED,IAAI,OAAQ,OAAe,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;oBACvD,SAAS;gBACb,CAAC;YACL,CAAC;YAED,MAAM,IAAI,KAAK,CACX,0KAA0K,OAAO,CAC7K,OAAO,CACV,EAAE,CACN,CAAC;QACN,CAAC;QAED,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,wBAAwB,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QACtE,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBAChE,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;YAClE,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,IAAuB,CAAC,CAAC;YACtF,CAAC;QACL,CAAC;QAED,MAAM,uCAAuC,GAAG,KAAK,EAAE,gBAA0B,EAAE,KAAK,GAAG,IAAI,EAAE,EAAE;YAC/F,MAAM,eAAe,GAAuB,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACpG,eAAe,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAErD,IAAI,SAAS,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC;gBACvC,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;gBAEtC,eAAe,CAAC,IAAI,CAChB,GAAG,CAAC,MAAM,uCAAuC,CAC7C,gBAAgB,CAAC,MAAM,CACnB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS,CAAC,CACjF,EACD,KAAK,CACR,CAAC,CACL,CAAC;YACN,CAAC;YAED,OAAO,eAAe,CAAC;QAC3B,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAElD,8DAA8D;QAC9D,MAAM,aAAa,GAAG,MAAM,uCAAuC,CAAC,YAAY,CAAC,CAAC;QAElF,mDAAmD;QACnD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,OAAO;gBACH,aAAa;gBACb,2BAA2B,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;aACnD,CAAC;QACN,CAAC;QAED,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAqB,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACzC,MAAM,kBAAkB,GAAuB,EAAE,CAAC;YAElD,KAAK,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;gBAChC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,uCAAuC,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBAEjG,MAAM,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC1C,CAAC;YAED,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QACtC,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;YACtB,IAAI,CAAC,2BAA2B,IAAI,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,0GAA0G;QAC1G,IAAI,OAAO,CAAC,2BAA2B,EAAE,CAAC;YACtC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO;YACH,aAAa;YACb,2BAA2B,EAAE,OAAO;SACvC,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAoC,EAAU;QAC1D,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,OAAO,IAAI,OAAO,CAAC,cAA2C,CAAC,CAAC;IACpE,CAAC;IAqBD;;;;;OAKG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;YACpB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAChC,CAAC,CACL,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;QAE7F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAChE,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;gBAChC,GAAG,OAAO;gBACV,SAAS;aACZ,CAAC,CAAC;YACH,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAExD,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE,CAAC;YACxC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAErC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,UAAwC,EAAE;QAE1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnB,EAAE,EAAE,EAAE,CAAC,MAAM;YACb,SAAS,EAAE,EAAE,CAAC,MAAM;SACvB,CAAC,CACL,CAAC;QACF,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CACL,CAAC;QAEF,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,oEAAoE;QACpE,wEAAwE;QACxE,MAAM,kBAAkB,GAAG;YACvB,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE;gBACzC,SAAS;aACZ,CAAC,CAAC;YACH,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,SAAS;SACwB,CAAC;QACtC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAID;;;;;OAKG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC5C,CAAC;IAUS,MAAM;QACZ,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,4BAA4B,CAAC,KAAK,EAAE,CAAC;QAC1C,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,QAAgB,EAAE,kBAA6C;QACnF,yEAAyE;QACzE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC5B,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;YAC/C,SAAS,EAAE,kBAAkB,CAAC,SAAS;YACvC,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,kBAAkB,CAAC,SAAS;SAC1C,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACO,2BAA2B,CAAC,SAAiB,EAAE,SAAkB;QACvE,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACtD,CAAC;aAAM,IAAI,IAAI,CAAC,iBAAiB,GAAG,qBAAqB,EAAE,CAAC;YACxD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACvD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,WAAoC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY;QACd,gFAAgF;QAChF,MAAM,EAAE,mBAAmB,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7D,OAAO,mBAAmB,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBACrC,GAAG,EAAE,eAAe;gBACpB,SAAS,EAAE,KAAK;gBAChB,QAAQ,EAAE,MAAM,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE;aACpD,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0CAA0C,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YACzF,OAAO,EAAE,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAC/B,MAAsB,EACtB,eAAiC,EACjC,OAAqC;QAErC,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QAElF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY,EAAE,eAAe,CAAC,MAAM;YACpC,aAAa,EAAE,aAAa,CAAC,MAAM;YACnC,cAAc,EAAE,eAAe,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;YAC7D,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;QAEH,OAAO,aAAa,CAAC;IACzB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAA+D;QAG7F,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,IAAmC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/F,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC9E,KAAK,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAEtD,wCAAwC;QACxC,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,CAAC;QAE3E,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ"}

package/storages/request_queue_v2.d.ts

@@ -1,87 +0,0 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { Configuration } from '../configuration.js';
-import type { Request, Source } from '../request.js';
-import type { RequestProviderOptions, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_provider.js';
-import { RequestProvider } from './request_provider.js';
-/**
- * Represents a queue of URLs to crawl, which is used for deep crawling of websites
- * where you start with several URLs and then recursively
- * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
- *
- * Each URL is represented using an instance of the {@link Request} class.
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
- * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
- * To add a single URL multiple times to the queue,
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
- *
- * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
- *
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
- * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Open the default request queue associated with the crawler run
- * const queue = await RequestQueue.open();
- *
- * // Open a named request queue
- * const queueWithName = await RequestQueue.open('some-name');
- *
- * // Enqueue few requests
- * await queue.addRequest({ url: 'http://example.com/aaa' });
- * await queue.addRequest({ url: 'http://example.com/bbb' });
- * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
- * ```
- * @category Sources
- */
-export declare class RequestQueue extends RequestProvider {
-    private listHeadAndLockPromise;
-    private queueHasLockedRequests;
-    private shouldCheckForForefrontRequests;
-    private dequeuedRequestCount;
-    constructor(options: RequestProviderOptions, config?: Configuration);
-    /**
-     * Caches information about request to beware of unneeded addRequest() calls.
-     */
-    protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
-    /**
-     * @inheritDoc
-     */
-    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
-    /**
-     * @inheritDoc
-     */
-    addRequests(requestsLike: Source[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
-    /**
-     * @inheritDoc
-     */
-    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
-    /**
-     * @inheritDoc
-     */
-    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
-    /**
-     * @inheritDoc
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>): ReturnType<RequestProvider['reclaimRequest']>;
-    protected ensureHeadIsNonEmpty(): Promise<void>;
-    private giveUpLock;
-    private _listHeadAndLock;
-    private getOrHydrateRequest;
-    private _prolongRequestLock;
-    protected _reset(): void;
-    protected _maybeAddRequestToQueueHead(): void;
-    protected _clearPossibleLocks(): Promise<void>;
-    /**
-     * @inheritDoc
-     */
-    static open(...args: Parameters<typeof RequestProvider.open>): Promise<RequestQueue>;
-}
-//# sourceMappingURL=request_queue_v2.d.ts.map