apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/storages/request_queue.js

@@ -1,636 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.openRequestQueue = exports.RequestQueue = exports.getRequestId = exports.STORAGE_CONSISTENCY_DELAY_MILLIS = exports.MAX_QUERIES_FOR_CONSISTENCY = exports.API_PROCESSED_REQUESTS_DELAY_MILLIS = exports.QUERY_HEAD_BUFFER = exports.QUERY_HEAD_MIN_LENGTH = void 0;
-const tslib_1 = require("tslib");
-const crypto_1 = (0, tslib_1.__importDefault)(require("crypto"));
-const datastructures_1 = require("@apify/datastructures");
-const consts_1 = require("@apify/consts");
-const utilities_1 = require("@apify/utilities");
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const storage_manager_1 = require("./storage_manager");
-const utils_1 = require("../utils");
-const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
-/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
-const request_1 = (0, tslib_1.__importDefault)(require("../request")); // eslint-disable-line import/named,no-unused-vars
-/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
-const MAX_CACHED_REQUESTS = 1000 * 1000;
-// When requesting queue head we always fetch requestsInProgressCount * QUERY_HEAD_BUFFER number of requests.
-exports.QUERY_HEAD_MIN_LENGTH = 100;
-exports.QUERY_HEAD_BUFFER = 3;
-// If queue was modified (request added/updated/deleted) before more than API_PROCESSED_REQUESTS_DELAY_MILLIS
-// then we assume the get head operation to be consistent.
-exports.API_PROCESSED_REQUESTS_DELAY_MILLIS = 10 * 1000;
-// How many times we try to get queue head with queueModifiedAt older than API_PROCESSED_REQUESTS_DELAY_MILLIS.
-exports.MAX_QUERIES_FOR_CONSISTENCY = 6;
-// This number must be large enough so that processing of all these requests cannot be done in
-// a time lower than expected maximum latency of DynamoDB, but low enough not to waste too much memory.
-const RECENTLY_HANDLED_CACHE_SIZE = 1000;
-// Indicates how long it usually takes for the underlying storage to propagate all writes
-// to be available to subsequent reads.
-exports.STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
-/**
- * Helper function that creates ID from uniqueKey for local emulation of request queue.
- * It's also used for local cache of remote request queue.
- *
- * This function may not exactly match how requestId is created server side.
- * So we never pass requestId created by this to server and use it only for local cache.
- *
- * @ignore
- */
-const getRequestId = (uniqueKey) => {
-    const str = crypto_1.default
-        .createHash('sha256')
-        .update(uniqueKey)
-        .digest('base64')
-        .replace(/(\+|\/|=)/g, '');
-    return str.substr(0, 15);
-};
-exports.getRequestId = getRequestId;
-/**
- * A helper class that is used to report results from various
- * {@link RequestQueue} functions as well as
- * {@link utils#enqueueLinks}.
- *
- * @typedef QueueOperationInfo
- * @property {boolean} wasAlreadyPresent Indicates if request was already present in the queue.
- * @property {boolean} wasAlreadyHandled Indicates if request was already marked as handled.
- * @property {string} requestId The ID of the added request
- * @property {Request} request The original {@link Request} object passed to the `RequestQueue` function.
- */
-/**
- * Represents a queue of URLs to crawl, which is used for deep crawling of websites
- * where you start with several URLs and then recursively
- * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
- *
- * Each URL is represented using an instance of the {@link Request} class.
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
- * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
- * To add a single URL multiple times to the queue,
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
- *
- * Do not instantiate this class directly, use the
- * {@link Apify#openRequestQueue} function instead.
- *
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
- * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
- *
- * `RequestQueue` stores its data either on local disk or in the Apify Cloud,
- * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
- *
- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in
- * that directory in an SQLite database file.
- *
- * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
- * [Apify Request Queue](https://docs.apify.com/storage/request-queue)
- * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
- * option to {@link Apify#openRequestQueue} function,
- * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Open the default request queue associated with the actor run
- * const queue = await Apify.openRequestQueue();
- *
- * // Open a named request queue
- * const queueWithName = await Apify.openRequestQueue('some-name');
- *
- * // Enqueue few requests
- * await queue.addRequest({ url: 'http://example.com/aaa' });
- * await queue.addRequest({ url: 'http://example.com/bbb' });
- * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
- * ```
- * @hideconstructor
- */
-class RequestQueue {
-    /**
-     * @param {object} options
-     * @param {string} options.id
-     * @param {string} [options.name]
-     * @param {boolean} options.isLocal
-     * @param {ApifyClient|ApifyStorageLocal} options.client
-     */
-    constructor(options) {
-        this.id = options.id;
-        this.name = options.name;
-        this.isLocal = options.isLocal;
-        this.timeoutSecs = 30;
-        this.clientKey = (0, utilities_1.cryptoRandomObjectId)();
-        this.client = options.client.requestQueue(this.id, {
-            clientKey: this.clientKey,
-            timeoutSecs: this.timeoutSecs,
-        });
-        this.log = utils_log_1.default.child({ prefix: 'RequestQueue' });
-        // Contains a cached list of request IDs from the head of the queue,
-        // as obtained in the last query. Both key and value is the request ID.
-        // Need to apply a type here to the generated TS types don't try to use types-apify
-        /**
-         * @type {*}
-         * @ignore
-         */
-        this.queueHeadDict = new datastructures_1.ListDictionary();
-        this.queryQueueHeadPromise = null;
-        // A set of all request IDs that are currently being handled,
-        // i.e. which were returned by fetchNextRequest() but not markRequestHandled()
-        this.inProgress = new Set();
-        // To track whether the queue gets stuck, and we need to reset it
-        // `lastActivity` tracks the time when we either added, processed or reclaimed a request,
-        // or when we add new request to in-progress cache
-        this.lastActivity = new Date();
-        this.internalTimeoutMillis = 5 * 60e3; // defaults to 5 minutes, will be overridden by BasicCrawler
-        // Contains a list of recently handled requests. It is used to avoid inconsistencies
-        // caused by delays in the underlying DynamoDB storage.
-        // Keys are request IDs, values are true.
-        this.recentlyHandled = new datastructures_1.LruCache({ maxLength: RECENTLY_HANDLED_CACHE_SIZE });
-        // We can trust these numbers only in a case that queue is used by a single client.
-        // This information is returned by getHead() under the hadMultipleClients property.
-        this.assumedTotalCount = 0;
-        this.assumedHandledCount = 0;
-        // Caching requests to avoid redundant addRequest() calls.
-        // Key is computed using getRequestId() and value is { id, isHandled }.
-        // TODO: We could extend the caching to improve performance
-        //       of other operations such as fetchNextRequest().
-        this.requestsCache = new datastructures_1.LruCache({ maxLength: MAX_CACHED_REQUESTS });
-    }
-    /**
-     * @ignore
-     */
-    inProgressCount() {
-        return this.inProgress.size;
-    }
-    /**
-     * Adds a request to the queue.
-     *
-     * If a request with the same `uniqueKey` property is already present in the queue,
-     * it will not be updated. You can find out whether this happened from the resulting
-     * {@link QueueOperationInfo} object.
-     *
-     * To add multiple requests to the queue by extracting links from a webpage,
-     * see the {@link utils#enqueueLinks} helper function.
-     *
-     * @param {(Request|RequestOptions)} requestLike {@link Request} object or vanilla object with request data.
-     * Note that the function sets the `uniqueKey` and `id` fields to the passed Request.
-     * @param {Object} [options]
-     * @param {boolean} [options.forefront=false] If `true`, the request will be added to the foremost position in the queue.
-     * @return {Promise<QueueOperationInfo>}
-     */
-    async addRequest(requestLike, options = {}) {
-        (0, ow_1.default)(requestLike, ow_1.default.object.partialShape({
-            url: ow_1.default.string.url,
-            id: ow_1.default.undefined,
-        }));
-        (0, ow_1.default)(options, ow_1.default.object.exactShape({
-            forefront: ow_1.default.optional.boolean,
-        }));
-        this.lastActivity = new Date();
-        const { forefront = false } = options;
-        const request = requestLike instanceof request_1.default
-            ? requestLike
-            : new request_1.default(requestLike);
-        const cacheKey = (0, exports.getRequestId)(request.uniqueKey);
-        const cachedInfo = this.requestsCache.get(cacheKey);
-        if (cachedInfo) {
-            request.id = cachedInfo.id;
-            return {
-                wasAlreadyPresent: true,
-                // We may assume that if request is in local cache then also the information if the
-                // request was already handled is there because just one client should be using one queue.
-                wasAlreadyHandled: cachedInfo.isHandled,
-                requestId: cachedInfo.id,
-                request,
-            };
-        }
-        const queueOperationInfo = await this.client.addRequest(request, { forefront });
-        const { requestId, wasAlreadyPresent } = queueOperationInfo;
-        this._cacheRequest(cacheKey, queueOperationInfo);
-        if (!wasAlreadyPresent && !this.inProgress.has(requestId) && !this.recentlyHandled.get(requestId)) {
-            this.assumedTotalCount++;
-            // Performance optimization: add request straight to head if possible
-            this._maybeAddRequestToQueueHead(requestId, forefront);
-        }
-        queueOperationInfo.request = { ...request, id: requestId };
-        return queueOperationInfo;
-    }
-    /**
-     * Gets the request from the queue specified by ID.
-     *
-     * @param {string} id ID of the request.
-     * @return {Promise<(Request|null)>} Returns the request object, or `null` if it was not found.
-     */
-    async getRequest(id) {
-        (0, ow_1.default)(id, ow_1.default.string);
-        // TODO: Could we also use requestsCache here? It would be consistent with addRequest()
-        // Downside is that it wouldn't reflect changes from outside...
-        const requestOptions = await this.client.getRequest(id);
-        if (!requestOptions)
-            return null;
-        // TODO: compatibility fix for old/broken request queues with null Request props
-        const optionsWithoutNulls = Object.entries(requestOptions).reduce((opts, [key, value]) => {
-            if (value !== null) {
-                opts[key] = value;
-            }
-            return opts;
-        }, {});
-        return new request_1.default(optionsWithoutNulls);
-    }
-    /**
-     * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
-     *
-     * Once you successfully finish processing of the request, you need to call
-     * {@link RequestQueue#markRequestHandled}
-     * to mark the request as handled in the queue. If there was some error in processing the request,
-     * call {@link RequestQueue#reclaimRequest} instead,
-     * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
-     *
-     * Note that the `null` return value doesn't mean the queue processing finished,
-     * it means there are currently no pending requests.
-     * To check whether all requests in queue were finished,
-     * use {@link RequestQueue#isFinished} instead.
-     *
-     * @returns {Promise<(Request|null)>}
-     * Returns the request object or `null` if there are no more pending requests.
-     */
-    async fetchNextRequest() {
-        await this._ensureHeadIsNonEmpty();
-        const nextRequestId = this.queueHeadDict.removeFirst();
-        // We are likely done at this point.
-        if (!nextRequestId)
-            return null;
-        // This should never happen, but...
-        if (this.inProgress.has(nextRequestId) || this.recentlyHandled.get(nextRequestId)) {
-            this.log.warning('Queue head returned a request that is already in progress?!', {
-                nextRequestId,
-                inProgress: this.inProgress.has(nextRequestId),
-                recentlyHandled: !!this.recentlyHandled.get(nextRequestId),
-            });
-            return null;
-        }
-        this.inProgress.add(nextRequestId);
-        this.lastActivity = new Date();
-        let request;
-        try {
-            request = await this.getRequest(nextRequestId);
-        }
-        catch (e) {
-            // On error, remove the request from in progress, otherwise it would be there forever
-            this.inProgress.delete(nextRequestId);
-            throw e;
-        }
-        // NOTE: It can happen that the queue head index is inconsistent with the main queue table. This can occur in two situations:
-        // 1) Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
-        //    In this case, keep the request marked as in progress for a short while,
-        //    so that isFinished() doesn't return true and _ensureHeadIsNonEmpty() doesn't not load the request
-        //    into the queueHeadDict straight again. After the interval expires, fetchNextRequest()
-        //    will try to fetch this request again, until it eventually appears in the main table.
-        if (!request) {
-            this.log.debug('Cannot find a request from the beginning of queue, will be retried later', { nextRequestId });
-            setTimeout(() => {
-                this.inProgress.delete(nextRequestId);
-            }, exports.STORAGE_CONSISTENCY_DELAY_MILLIS);
-            return null;
-        }
-        // 2) Queue head index is behind the main table and the underlying request was already handled
-        //    (by some other client, since we keep the track of handled requests in recentlyHandled dictionary).
-        //    We just add the request to the recentlyHandled dictionary so that next call to _ensureHeadIsNonEmpty()
-        //    will not put the request again to queueHeadDict.
-        if (request.handledAt) {
-            this.log.debug('Request fetched from the beginning of queue was already handled', { nextRequestId });
-            this.recentlyHandled.add(nextRequestId, true);
-            return null;
-        }
-        return request;
-    }
-    /**
-     * Marks a request that was previously returned by the
-     * {@link RequestQueue#fetchNextRequest}
-     * function as handled after successful processing.
-     * Handled requests will never again be returned by the `fetchNextRequest` function.
-     *
-     * @param {Request} request
-     * @return {Promise<QueueOperationInfo | null>}
-     */
-    async markRequestHandled(request) {
-        this.lastActivity = new Date();
-        (0, ow_1.default)(request, ow_1.default.object.partialShape({
-            id: ow_1.default.string,
-            uniqueKey: ow_1.default.string,
-            handledAt: ow_1.default.optional.date,
-        }));
-        if (!this.inProgress.has(request.id)) {
-            this.log.debug(`Cannot mark request ${request.id} as handled, because it is not in progress!`, { requestId: request.id });
-            return null;
-        }
-        const handledAt = request.handledAt ?? new Date();
-        const queueOperationInfo = await this.client.updateRequest({ ...request, handledAt });
-        request.handledAt = handledAt;
-        this.inProgress.delete(request.id);
-        this.recentlyHandled.add(request.id, true);
-        if (!queueOperationInfo.wasAlreadyHandled) {
-            this.assumedHandledCount++;
-        }
-        this._cacheRequest((0, exports.getRequestId)(request.uniqueKey), queueOperationInfo);
-        queueOperationInfo.request = request;
-        return queueOperationInfo;
-    }
-    /**
-     * Reclaims a failed request back to the queue, so that it can be returned for processed later again
-     * by another call to {@link RequestQueue#fetchNextRequest}.
-     * The request record in the queue is updated using the provided `request` parameter.
-     * For example, this lets you store the number of retries or error messages for the request.
-     *
-     * @param {Request} request
-     * @param {object} [options]
-     * @param {boolean} [options.forefront=false]
-     * If `true` then the request it placed to the beginning of the queue, so that it's returned
-     * in the next call to {@link RequestQueue#fetchNextRequest}.
-     * By default, it's put to the end of the queue.
-     * @return {Promise<QueueOperationInfo | null>}
-     */
-    async reclaimRequest(request, options = {}) {
-        this.lastActivity = new Date();
-        (0, ow_1.default)(request, ow_1.default.object.partialShape({
-            id: ow_1.default.string,
-            uniqueKey: ow_1.default.string,
-        }));
-        (0, ow_1.default)(options, ow_1.default.object.exactShape({
-            forefront: ow_1.default.optional.boolean,
-        }));
-        const { forefront = false } = options;
-        if (!this.inProgress.has(request.id)) {
-            this.log.debug(`Cannot reclaim request ${request.id}, because it is not in progress!`, { requestId: request.id });
-            return null;
-        }
-        // TODO: If request hasn't been changed since the last getRequest(),
-        // we don't need to call updateRequest() and thus improve performance.
-        const queueOperationInfo = await this.client.updateRequest(request, { forefront });
-        this._cacheRequest((0, exports.getRequestId)(request.uniqueKey), queueOperationInfo);
-        queueOperationInfo.request = request;
-        // Wait a little to increase a chance that the next call to fetchNextRequest() will return the request with updated data.
-        // This is to compensate for the limitation of DynamoDB, where writes might not be immediately visible to subsequent reads.
-        setTimeout(() => {
-            if (!this.inProgress.has(request.id)) {
-                this.log.debug('The request is no longer marked as in progress in the queue?!', { requestId: request.id });
-                return;
-            }
-            this.inProgress.delete(request.id);
-            // Performance optimization: add request straight to head if possible
-            this._maybeAddRequestToQueueHead(request.id, forefront);
-        }, exports.STORAGE_CONSISTENCY_DELAY_MILLIS);
-        return queueOperationInfo;
-    }
-    /**
-     * Resolves to `true` if the next call to {@link RequestQueue#fetchNextRequest}
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the queue is empty, there might be some pending requests currently being processed.
-     * If you need to ensure that there is no activity in the queue, use {@link RequestQueue#isFinished}.
-     *
-     * @returns {Promise<boolean>}
-     */
-    async isEmpty() {
-        await this._ensureHeadIsNonEmpty();
-        return this.queueHeadDict.length() === 0;
-    }
-    /**
-     * Resolves to `true` if all requests were already handled and there are no more left.
-     * Due to the nature of distributed storage used by the queue,
-     * the function might occasionally return a false negative,
-     * but it will never return a false positive.
-     *
-     * @returns {Promise<boolean>}
-     */
-    async isFinished() {
-        if (this.inProgressCount() > 0 && (Date.now() - +this.lastActivity) > this.internalTimeoutMillis) {
-            const message = `The request queue seems to be stuck for ${this.internalTimeoutMillis / 1e3}s, resetting internal state.`;
-            this.log.warning(message, { inProgress: [...this.inProgress] });
-            this._reset();
-        }
-        if (this.queueHeadDict.length() > 0 || this.inProgressCount() > 0)
-            return false;
-        const isHeadConsistent = await this._ensureHeadIsNonEmpty(true);
-        return isHeadConsistent && this.queueHeadDict.length() === 0 && this.inProgressCount() === 0;
-    }
-    /**
-     * @internal
-     */
-    _reset() {
-        this.queueHeadDict.clear();
-        this.queryQueueHeadPromise = null;
-        this.inProgress.clear();
-        this.recentlyHandled.clear();
-        this.assumedTotalCount = 0;
-        this.assumedHandledCount = 0;
-        this.requestsCache.clear();
-        this.lastActivity = new Date();
-    }
-    /**
-     * Caches information about request to beware of unneeded addRequest() calls.
-     * @param {string} cacheKey
-     * @param {object} queueOperationInfo
-     * @param {string} queueOperationInfo.requestId
-     * @param {boolean} queueOperationInfo.wasAlreadyHandled
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _cacheRequest(cacheKey, queueOperationInfo) {
-        this.requestsCache.add(cacheKey, {
-            id: queueOperationInfo.requestId,
-            isHandled: queueOperationInfo.wasAlreadyHandled,
-        });
-    }
-    /**
-     * We always request more items than is in progress to ensure that something falls into head.
-     *
-     * @param {boolean} [ensureConsistency=false] If true then query for queue head is retried until queueModifiedAt
-     *   is older than queryStartedAt by at least API_PROCESSED_REQUESTS_DELAY_MILLIS to ensure that queue
-     *   head is consistent.
-     * @param {number} [limit] How many queue head items will be fetched.
-     * @param {number} [iteration] Used when this function is called recursively to limit the recursion.
-     * @return {Promise<boolean>} Indicates if queue head is consistent (true) or inconsistent (false).
-     * @ignore
-     * @protected
-     * @internal
-     */
-    async _ensureHeadIsNonEmpty(ensureConsistency = false, limit = Math.max(this.inProgressCount() * exports.QUERY_HEAD_BUFFER, exports.QUERY_HEAD_MIN_LENGTH), iteration = 0) {
-        // If is nonempty resolve immediately.
-        if (this.queueHeadDict.length() > 0)
-            return true;
-        if (!this.queryQueueHeadPromise) {
-            const queryStartedAt = new Date();
-            this.queryQueueHeadPromise = this.client
-                .listHead({ limit })
-                .then(({ items, queueModifiedAt, hadMultipleClients }) => {
-                items.forEach(({ id: requestId, uniqueKey }) => {
-                    // Queue head index might be behind the main table, so ensure we don't recycle requests
-                    if (this.inProgress.has(requestId) || this.recentlyHandled.get(requestId))
-                        return;
-                    this.queueHeadDict.add(requestId, requestId, false);
-                    this._cacheRequest((0, exports.getRequestId)(uniqueKey), { requestId, wasAlreadyHandled: false });
-                });
-                // This is needed so that the next call to _ensureHeadIsNonEmpty() will fetch the queue head again.
-                this.queryQueueHeadPromise = null;
-                return {
-                    wasLimitReached: items.length >= limit,
-                    prevLimit: limit,
-                    queueModifiedAt: new Date(queueModifiedAt),
-                    queryStartedAt,
-                    hadMultipleClients,
-                };
-            });
-        }
-        const { queueModifiedAt, wasLimitReached, prevLimit, queryStartedAt, hadMultipleClients } = await this.queryQueueHeadPromise;
-        // TODO: I feel this code below can be greatly simplified...
-        // If queue is still empty then one of the following holds:
-        // - the other calls waiting for this promise already consumed all the returned requests
-        // - the limit was too low and contained only requests in progress
-        // - the writes from other clients were not propagated yet
-        // - the whole queue was processed and we are done
-        // If limit was not reached in the call then there are no more requests to be returned.
-        if (prevLimit >= consts_1.REQUEST_QUEUE_HEAD_MAX_LIMIT) {
-            this.log.warning(`Reached the maximum number of requests in progress: ${consts_1.REQUEST_QUEUE_HEAD_MAX_LIMIT}.`);
-        }
-        const shouldRepeatWithHigherLimit = this.queueHeadDict.length() === 0
-            && wasLimitReached
-            && prevLimit < consts_1.REQUEST_QUEUE_HEAD_MAX_LIMIT;
-        // If ensureConsistency=true then we must ensure that either:
-        // - queueModifiedAt is older than queryStartedAt by at least API_PROCESSED_REQUESTS_DELAY_MILLIS
-        // - hadMultipleClients=false and this.assumedTotalCount<=this.assumedHandledCount
-        const isDatabaseConsistent = queryStartedAt - queueModifiedAt >= exports.API_PROCESSED_REQUESTS_DELAY_MILLIS;
-        const isLocallyConsistent = !hadMultipleClients && this.assumedTotalCount <= this.assumedHandledCount;
-        // Consistent information from one source is enough to consider request queue finished.
-        const shouldRepeatForConsistency = ensureConsistency && !isDatabaseConsistent && !isLocallyConsistent;
-        // If both are false then head is consistent and we may exit.
-        if (!shouldRepeatWithHigherLimit && !shouldRepeatForConsistency)
-            return true;
-        // If we are querying for consistency then we limit the number of queries to MAX_QUERIES_FOR_CONSISTENCY.
-        // If this is reached then we return false so that empty() and finished() returns possibly false negative.
-        if (!shouldRepeatWithHigherLimit && iteration > exports.MAX_QUERIES_FOR_CONSISTENCY)
-            return false;
-        const nextLimit = shouldRepeatWithHigherLimit
-            ? Math.round(prevLimit * 1.5)
-            : prevLimit;
-        // If we are repeating for consistency then wait required time.
-        if (shouldRepeatForConsistency) {
-            const delayMillis = exports.API_PROCESSED_REQUESTS_DELAY_MILLIS - (Date.now() - queueModifiedAt);
-            this.log.info(`Waiting for ${delayMillis}ms before considering the queue as finished to ensure that the data is consistent.`);
-            await (0, utils_1.sleep)(delayMillis);
-        }
-        return this._ensureHeadIsNonEmpty(ensureConsistency, nextLimit, iteration + 1);
-    }
-    /**
-     * Adds a request straight to the queueHeadDict, to improve performance.
-     * @private
-     */
-    _maybeAddRequestToQueueHead(requestId, forefront) {
-        if (forefront) {
-            this.queueHeadDict.add(requestId, requestId, true);
-        }
-        else if (this.assumedTotalCount < exports.QUERY_HEAD_MIN_LENGTH) {
-            this.queueHeadDict.add(requestId, requestId, false);
-        }
-    }
-    /**
-     * Removes the queue either from the Apify Cloud storage or from the local database,
-     * depending on the mode of operation.
-     *
-     * @return {Promise<void>}
-     */
-    async drop() {
-        await this.client.delete();
-        const manager = new storage_manager_1.StorageManager(RequestQueue);
-        manager.closeStorage(this);
-    }
-    /**
-     * Returns the number of handled requests.
-     *
-     * This function is just a convenient shortcut for:
-     *
-     * ```javascript
-     * const { handledRequestCount } = await queue.getInfo();
-     * ```
-     *
-     * @return {Promise<number>}
-     */
-    async handledCount() {
-        // NOTE: We keep this function for compatibility with RequestList.handledCount()
-        const { handledRequestCount } = await this.getInfo();
-        return handledRequestCount;
-    }
-    /**
-     * Returns an object containing general information about the request queue.
-     *
-     * The function returns the same object as the Apify API Client's
-     * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
-     * function, which in turn calls the
-     * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
-     * API endpoint.
-     *
-     * **Example:**
-     * ```
-     * {
-     *   id: "WkzbQMuFYuamGv3YF",
-     *   name: "my-queue",
-     *   userId: "wRsJZtadYvn4mBZmm",
-     *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
-     *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
-     *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
-     *   totalRequestCount: 25,
-     *   handledRequestCount: 5,
-     *   pendingRequestCount: 20,
-     * }
-     * ```
-     *
-     * @returns {Promise<RequestQueueInfo>}
-     */
-    async getInfo() {
-        return this.client.get();
-    }
-}
-exports.RequestQueue = RequestQueue;
-/**
- * Opens a request queue and returns a promise resolving to an instance
- * of the {@link RequestQueue} class.
- *
- * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
- * The queue is used for deep crawling of websites, where you start with several URLs and then
- * recursively follow links to other pages. The data structure supports both breadth-first
- * and depth-first crawling orders.
- *
- * For more details and code examples, see the {@link RequestQueue} class.
- *
- * @param {string} [queueIdOrName]
- *   ID or name of the request queue to be opened. If `null` or `undefined`,
- *   the function returns the default request queue associated with the actor run.
- * @param {object} [options]
- * @param {boolean} [options.forceCloud=false]
- *   If set to `true` then the function uses cloud storage usage even if the `APIFY_LOCAL_STORAGE_DIR`
- *   environment variable is set. This way it is possible to combine local and cloud storage.
- * @returns {Promise<RequestQueue>}
- * @memberof module:Apify
- * @name openRequestQueue
- * @function
- */
-const openRequestQueue = async (queueIdOrName, options = {}) => {
-    (0, ow_1.default)(queueIdOrName, ow_1.default.optional.string);
-    (0, ow_1.default)(options, ow_1.default.object.exactShape({
-        forceCloud: ow_1.default.optional.boolean,
-    }));
-    const manager = new storage_manager_1.StorageManager(RequestQueue);
-    return manager.openStorage(queueIdOrName, options);
-};
-exports.openRequestQueue = openRequestQueue;
-/**
- * @typedef RequestQueueInfo
- * @property {string} id
- * @property {string} name
- * @property {string} userId
- * @property {Date} createdAt
- * @property {Date} modifiedAt
- * @property {Date} accessedAt
- * @property {number} totalRequestCount
- * @property {number} handledRequestCount
- * @property {number} pendingRequestCount
- */
-//# sourceMappingURL=request_queue.js.map

package/build/storages/request_queue.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.js"],"names":[],"mappings":";;;;AAAA,iEAA4B;AAC5B,0DAAiE;AACjE,0CAA6D;AAC7D,gDAAwD;AACxD,yDAAoB;AACpB,uDAAmD;AACnD,oCAAiC;AACjC,0EAA+B;AAE/B,kFAAkF;AAClF,sEAAqD,CAAC,kDAAkD;AAMxG,iFAAiF;AAEjF,MAAM,mBAAmB,GAAG,IAAI,GAAG,IAAI,CAAC;AAExC,6GAA6G;AAChG,QAAA,qBAAqB,GAAG,GAAG,CAAC;AAC5B,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAEnC,6GAA6G;AAC7G,0DAA0D;AAC7C,QAAA,mCAAmC,GAAG,EAAE,GAAG,IAAI,CAAC;AAE7D,+GAA+G;AAClG,QAAA,2BAA2B,GAAG,CAAC,CAAC;AAE7C,8FAA8F;AAC9F,uGAAuG;AACvG,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC,yFAAyF;AACzF,uCAAuC;AAC1B,QAAA,gCAAgC,GAAG,IAAI,CAAC;AAErD;;;;;;;;GAQG;AACI,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,EAAE;IACtC,MAAM,GAAG,GAAG,gBAAM;SACb,UAAU,CAAC,QAAQ,CAAC;SACpB,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC,QAAQ,CAAC;SAChB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAE/B,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7B,CAAC,CAAC;AARW,QAAA,YAAY,gBAQvB;AAEF;;;;;;;;;;GAUG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,MAAa,YAAY;IACrB;;;;;;OAMG;IACH,YAAY,OAAO;QACf,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,IAAA,gCAAoB,GAAE,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE;YAC/C,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;SAChC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,GAAG,mBAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;QAEjD,oEAAoE;QACpE,uEAAuE;QACvE,mFAAmF;QACnF;;;WAGG;QACH,IAAI,CAAC,aAAa,GAAG,IAAI,+BAAc,EAAE,CAAC;QAC1C,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAElC,6DAA6D;QAC7D,8EAA8E;QAC9E,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAE5B,iEAAiE;QACjE,yFAAyF;QACzF,kDAAkD;QAClD,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,4DAA4D;QAEnG,oFAAoF;QACpF,uDAAuD;QACvD,yCAAyC;QACzC,IAAI,CAAC,eAAe,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,2BAA2B,EAAE,CAAC,CAAC;QAEhF,mFAAmF;QACnF,mFAAmF;QACnF,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAE7B,0DAA0D;QAC1D,uEAAuE;QACvE,2DAA2D;QAC3D,wDAAwD;QACxD,IAAI,CAAC,aAAa,GAAG,IAAI,yBAAQ,CAAC,EAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,eAAe;QACX,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,KAAK,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,GAAG,EAAE;QACtC,IAAA,YAAE,EAAC,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YACnC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;YAClB,EAAE,EAAE,YAAE,CAAC,SAAS;SACnB,CAAC,CAAC,CAAC;QACJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,MAAM,OAAO,GAAG,WAAW,YAAY,iBAAO;YAC1C,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,IAAI,iBAAO,CAAC,WAAW,CAAC,CAAC;QAE/B,MAAM,QAAQ,GAAG,IAAA,oBAAY,EAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEpD,IAAI,UAAU,EAAE;YACZ,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,IAAI;gBACvB,mFAAmF;gBACnF,0FAA0F;gBAC1F,iBAAiB,EAAE,UAAU,CAAC,SAAS;gBACvC,SAAS,EAAE,UAAU,CAAC,EAAE;gBACxB,OAAO;aACV,CAAC;SACL;QAED,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAChF,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,GAAG,kBAAkB,CAAC;QAC5D,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAEjD,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC/F,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAEzB,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;SAC1D;QAED,kBAAkB,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC;QAE3D,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,EAAE;QACf,IAAA,YAAE,EAAC,EAAE,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAElB,uFAAuF;QACvF,+DAA+D;QAC/D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,gFAAgF;QAChF,MAAM,mBAAmB,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACrF,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;aACrB;YACD,OAAO,IAAI,CAAC;QAChB,CAAC,EAAE,EAAE,CAAC,CAAC;QAEP,OAAO,IAAI,iBAAO,CAAC,mBAAmB,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,gBAAgB;QAClB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC;QAEvD,oCAAoC;QACpC,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAEhC,mCAAmC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE;YAC/E,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;gBAC5E,aAAa;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC9C,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,CAAC;aAC7D,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;SACf;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,IAAI,OAAO,CAAC;QACZ,IAAI;YACA,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACR,qFAAqF;YACrF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,CAAC;SACX;QAED,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE;YACV,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0EAA0E,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9G,UAAU,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC1C,CAAC,EAAE,wCAAgC,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC;SACf;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE;YACnB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC9C,OAAO,IAAI,CAAC;SACf;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAO;QAC5B,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,MAAM;YACb,SAAS,EAAE,YAAE,CAAC,MAAM;YACpB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;SAC9B,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;YAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uBAAuB,OAAO,CAAC,EAAE,6CAA6C,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1H,OAAO,IAAI,CAAC;SACf;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;QAClD,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;QAE9B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAE3C,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,EAAE;YACvC,IAAI,CAAC,mBAAmB,EAAE,CAAC;SAC9B;QAED,IAAI,CAAC,aAAa,CAAC,IAAA,oBAAY,EAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAExE,kBAAkB,CAAC,OAAO,GAAG,OAAO,CAAC;QAErC,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,EAAE;QACtC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;YAC/B,EAAE,EAAE,YAAE,CAAC,MAAM;YACb,SAAS,EAAE,YAAE,CAAC,MAAM;SACvB,CAAC,CAAC,CAAC;QACJ,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;YAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,EAAE,kCAAkC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClH,OAAO,IAAI,CAAC;SACf;QAED,oEAAoE;QACpE,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACnF,IAAI,CAAC,aAAa,CAAC,IAAA,oBAAY,EAAC,OAAO,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;QACxE,kBAAkB,CAAC,OAAO,GAAG,OAAO,CAAC;QAErC,yHAAyH;QACzH,2HAA2H;QAC3H,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;gBAClC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3G,OAAO;aACV;YAED,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAEnC,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAC5D,CAAC,EAAE,wCAAgC,CAAC,CAAC;QAErC,OAAO,kBAAkB,CAAC;IAC9B,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,qBAAqB,EAAE;YAC9F,MAAM,OAAO,GAAG,2CAA2C,IAAI,CAAC,qBAAqB,GAAG,GAAG,8BAA8B,CAAC;YAC1H,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QAED,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAEhF,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,gBAAgB,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACjG,CAAC;IAED;;OAEG;IACH,MAAM;QACF,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAClC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;IACnC,CAAC;IAED;;;;;;;;;OASG;IACH,aAAa,CAAC,QAAQ,EAAE,kBAAkB;QACtC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC7B,EAAE,EAAE,kBAAkB,CAAC,SAAS;YAChC,SAAS,EAAE,kBAAkB,CAAC,iBAAiB;SAClD,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,qBAAqB,CACvB,iBAAiB,GAAG,KAAK,EACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,yBAAiB,EAAE,6BAAqB,CAAC,EACnF,SAAS,GAAG,CAAC;QAEb,sCAAsC;QACtC,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC7B,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC;YAElC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,MAAM;iBACnC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC;iBACnB,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,EAAE;gBACrD,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE;oBAC3C,uFAAuF;oBACvF,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC;wBAAE,OAAO;oBAElF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;oBACpD,IAAI,CAAC,aAAa,CAAC,IAAA,oBAAY,EAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC;gBACzF,CAAC,CAAC,CAAC;gBAEH,mGAAmG;gBACnG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;gBAElC,OAAO;oBACH,eAAe,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK;oBACtC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI,IAAI,CAAC,eAAe,CAAC;oBAC1C,cAAc;oBACd,kBAAkB;iBACrB,CAAC;YACN,CAAC,CAAC,CAAC;SACV;QAED,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;QAE7H,4DAA4D;QAE5D,2DAA2D;QAC3D,wFAAwF;QACxF,kEAAkE;QAClE,0DAA0D;QAC1D,kDAAkD;QAElD,uFAAuF;QACvF,IAAI,SAAS,IAAI,qCAA4B,EAAE;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uDAAuD,qCAA4B,GAAG,CAAC,CAAC;SAC5G;QACD,MAAM,2BAA2B,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC;eAC9D,eAAe;eACf,SAAS,GAAG,qCAA4B,CAAC;QAEhD,6DAA6D;QAC7D,iGAAiG;QACjG,kFAAkF;QAClF,MAAM,oBAAoB,GAAG,cAAc,GAAG,eAAe,IAAI,2CAAmC,CAAC;QACrG,MAAM,mBAAmB,GAAG,CAAC,kBAAkB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,mBAAmB,CAAC;QACtG,uFAAuF;QACvF,MAAM,0BAA0B,GAAG,iBAAiB,IAAI,CAAC,oBAAoB,IAAI,CAAC,mBAAmB,CAAC;QAEtG,6DAA6D;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,0BAA0B;YAAE,OAAO,IAAI,CAAC;QAE7E,yGAAyG;QACzG,0GAA0G;QAC1G,IAAI,CAAC,2BAA2B,IAAI,SAAS,GAAG,mCAA2B;YAAE,OAAO,KAAK,CAAC;QAE1F,MAAM,SAAS,GAAG,2BAA2B;YACzC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC;QAEhB,+DAA+D;QAC/D,IAAI,0BAA0B,EAAE;YAC5B,MAAM,WAAW,GAAG,2CAAmC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,CAAC;YACzF,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,WAAW,oFAAoF,CAAC,CAAC;YAC9H,MAAM,IAAA,aAAK,EAAC,WAAW,CAAC,CAAC;SAC5B;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACnF,CAAC;IAED;;;OAGG;IACH,2BAA2B,CAAC,SAAS,EAAE,SAAS;QAC5C,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;SACtD;aAAM,IAAI,IAAI,CAAC,iBAAiB,GAAG,6BAAqB,EAAE;YACvD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;SACvD;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,YAAY,CAAC,CAAC;QACjD,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,YAAY;QACd,gFAAgF;QAChF,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrD,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;CACJ;AAviBD,oCAuiBC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACI,MAAM,gBAAgB,GAAG,KAAK,EAAE,aAAa,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IAClE,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACtC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;KAClC,CAAC,CAAC,CAAC;IACJ,MAAM,OAAO,GAAG,IAAI,gCAAc,CAAC,YAAY,CAAC,CAAC;IACjD,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;AACvD,CAAC,CAAC;AAPW,QAAA,gBAAgB,oBAO3B;AAEF;;;;;;;;;;;GAWG"}