@crawlee/core 4.0.0-beta.26 → 4.0.0-beta.261

package/storages/request_queue.d.ts

@@ -1,128 +0,0 @@
-import type { Dictionary } from '@crawlee/types';
-import { Configuration } from '../configuration.js';
-import type { Request } from '../request.js';
-import type { RequestProviderOptions, RequestQueueOperationInfo } from './request_provider.js';
-import { RequestProvider } from './request_provider.js';
-/**
- * Represents a queue of URLs to crawl, which is used for deep crawling of websites
- * where you start with several URLs and then recursively
- * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
- *
- * Each URL is represented using an instance of the {@link Request} class.
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
- * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
- * To add a single URL multiple times to the queue,
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
- *
- * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
- *
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
- * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
- *
- * `RequestQueue` stores its data either on local disk or in the Apify Cloud,
- * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
- *
- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in
- * that directory in an SQLite database file.
- *
- * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
- * [Apify Request Queue](https://docs.apify.com/storage/request-queue)
- * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
- * option to {@link RequestQueue.open} function,
- * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Open the default request queue associated with the crawler run
- * const queue = await RequestQueue.open();
- *
- * // Open a named request queue
- * const queueWithName = await RequestQueue.open('some-name');
- *
- * // Enqueue few requests
- * await queue.addRequest({ url: 'http://example.com/aaa' });
- * await queue.addRequest({ url: 'http://example.com/bbb' });
- * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
- * ```
- * @category Sources
- *
- * @deprecated RequestQueue v1 is deprecated and will be removed in the future. Please use {@link RequestQueue} instead.
- */
-declare class RequestQueue extends RequestProvider {
-    private queryQueueHeadPromise?;
-    private inProgress;
-    /**
-     * @internal
-     */
-    constructor(options: RequestProviderOptions, config?: Configuration);
-    /**
-     * @internal
-     */
-    inProgressCount(): number;
-    /**
-     * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
-     *
-     * Once you successfully finish processing of the request, you need to call
-     * {@link RequestQueue.markRequestHandled}
-     * to mark the request as handled in the queue. If there was some error in processing the request,
-     * call {@link RequestQueue.reclaimRequest} instead,
-     * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
-     *
-     * Note that the `null` return value doesn't mean the queue processing finished,
-     * it means there are currently no pending requests.
-     * To check whether all requests in queue were finished,
-     * use {@link RequestQueue.isFinished} instead.
-     *
-     * @returns
-     *   Returns the request object or `null` if there are no more pending requests.
-     */
-    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
-    protected ensureHeadIsNonEmpty(): Promise<void>;
-    /**
-     * We always request more items than is in progress to ensure that something falls into head.
-     *
-     * @param [ensureConsistency] If true then query for queue head is retried until queueModifiedAt
-     *   is older than queryStartedAt by at least API_PROCESSED_REQUESTS_DELAY_MILLIS to ensure that queue
-     *   head is consistent.
-     * @default false
-     * @param [limit] How many queue head items will be fetched.
-     * @param [iteration] Used when this function is called recursively to limit the recursion.
-     * @returns Indicates if queue head is consistent (true) or inconsistent (false).
-     */
-    protected _ensureHeadIsNonEmpty(ensureConsistency?: boolean, limit?: number, iteration?: number): Promise<boolean>;
-    isFinished(): Promise<boolean>;
-    /**
-     * Reclaims a failed request back to the queue, so that it can be returned for processing later again
-     * by another call to {@link RequestQueue.fetchNextRequest}.
-     * The request record in the queue is updated using the provided `request` parameter.
-     * For example, this lets you store the number of retries or error messages for the request.
-     */
-    reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>): Promise<RequestQueueOperationInfo | null>;
-    /**
-     * @inheritdoc
-     */
-    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
-    protected _reset(): void;
-    /**
-     * Opens a request queue and returns a promise resolving to an instance
-     * of the {@link RequestQueue} class.
-     *
-     * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
-     * The queue is used for deep crawling of websites, where you start with several URLs and then
-     * recursively follow links to other pages. The data structure supports both breadth-first
-     * and depth-first crawling orders.
-     *
-     * For more details and code examples, see the {@link RequestQueue} class.
-     *
-     * @param [queueIdOrName]
-     *   ID or name of the request queue to be opened. If `null` or `undefined`,
-     *   the function returns the default request queue associated with the crawler run.
-     * @param [options] Open Request Queue options.
-     */
-    static open(...args: Parameters<typeof RequestProvider.open>): Promise<RequestQueue>;
-}
-export { RequestQueue as RequestQueueV1 };
-//# sourceMappingURL=request_queue.d.ts.map

package/storages/request_queue.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAE7C,OAAO,KAAK,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAC/F,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAmBxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,cAAM,YAAa,SAAQ,eAAe;IACtC,OAAO,CAAC,qBAAqB,CAAC,CAMb;IAEjB,OAAO,CAAC,UAAU,CAAqB;IAEvC;;OAEG;gBACS,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAkC;IAYrF;;OAEG;IACI,eAAe,IAAI,MAAM;IAIhC;;;;;;;;;;;;;;;;OAgBG;IACY,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;cAgEvE,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAK9D;;;;;;;;;;OAUG;cACa,qBAAqB,CACjC,iBAAiB,UAAQ,EACzB,KAAK,SAA8E,EACnF,SAAS,SAAI,GACd,OAAO,CAAC,OAAO,CAAC;IAmGJ,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAqB7C;;;;;OAKG;IACY,cAAc,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;IA2BpF;;OAEG;IACY,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;cAQ3E,MAAM,IAAI,IAAI;IAMjC;;;;;;;;;;;;;;;OAeG;WACmB,IAAI,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;CAGtG;AAED,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,CAAC"}

package/storages/request_queue.js

@@ -1,320 +0,0 @@
-import { setTimeout as sleep } from 'node:timers/promises';
-import { REQUEST_QUEUE_HEAD_MAX_LIMIT } from '@apify/consts';
-import { Configuration } from '../configuration.js';
-import { checkStorageAccess } from './access_checking.js';
-import { RequestProvider } from './request_provider.js';
-import { API_PROCESSED_REQUESTS_DELAY_MILLIS, getRequestId, MAX_QUERIES_FOR_CONSISTENCY, QUERY_HEAD_BUFFER, QUERY_HEAD_MIN_LENGTH, STORAGE_CONSISTENCY_DELAY_MILLIS, } from './utils.js';
-const MAX_CACHED_REQUESTS = 1_000_000;
-/**
- * This number must be large enough so that processing of all these requests cannot be done in
- * a time lower than expected maximum latency of DynamoDB, but low enough not to waste too much memory.
- * @internal
- */
-const RECENTLY_HANDLED_CACHE_SIZE = 1000;
-/**
- * Represents a queue of URLs to crawl, which is used for deep crawling of websites
- * where you start with several URLs and then recursively
- * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
- *
- * Each URL is represented using an instance of the {@link Request} class.
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
- * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
- * To add a single URL multiple times to the queue,
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
- *
- * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
- *
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
- * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
- *
- * `RequestQueue` stores its data either on local disk or in the Apify Cloud,
- * depending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variable is set.
- *
- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the queue data is stored in
- * that directory in an SQLite database file.
- *
- * If the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the
- * [Apify Request Queue](https://docs.apify.com/storage/request-queue)
- * cloud storage. Note that you can force usage of the cloud storage also by passing the `forceCloud`
- * option to {@link RequestQueue.open} function,
- * even if the `APIFY_LOCAL_STORAGE_DIR` variable is set.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Open the default request queue associated with the crawler run
- * const queue = await RequestQueue.open();
- *
- * // Open a named request queue
- * const queueWithName = await RequestQueue.open('some-name');
- *
- * // Enqueue few requests
- * await queue.addRequest({ url: 'http://example.com/aaa' });
- * await queue.addRequest({ url: 'http://example.com/bbb' });
- * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
- * ```
- * @category Sources
- *
- * @deprecated RequestQueue v1 is deprecated and will be removed in the future. Please use {@link RequestQueue} instead.
- */
-class RequestQueue extends RequestProvider {
-    queryQueueHeadPromise = null;
-    inProgress = new Set();
-    /**
-     * @internal
-     */
-    constructor(options, config = Configuration.getGlobalConfig()) {
-        super({
-            ...options,
-            logPrefix: 'RequestQueue',
-            recentlyHandledRequestsMaxSize: RECENTLY_HANDLED_CACHE_SIZE,
-            requestCacheMaxSize: MAX_CACHED_REQUESTS,
-        }, config);
-    }
-    /**
-     * @internal
-     */
-    inProgressCount() {
-        return this.inProgress.size;
-    }
-    /**
-     * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
-     *
-     * Once you successfully finish processing of the request, you need to call
-     * {@link RequestQueue.markRequestHandled}
-     * to mark the request as handled in the queue. If there was some error in processing the request,
-     * call {@link RequestQueue.reclaimRequest} instead,
-     * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
-     *
-     * Note that the `null` return value doesn't mean the queue processing finished,
-     * it means there are currently no pending requests.
-     * To check whether all requests in queue were finished,
-     * use {@link RequestQueue.isFinished} instead.
-     *
-     * @returns
-     *   Returns the request object or `null` if there are no more pending requests.
-     */
-    async fetchNextRequest() {
-        checkStorageAccess();
-        this.lastActivity = new Date();
-        await this.ensureHeadIsNonEmpty();
-        const nextRequestId = this.queueHeadIds.removeFirst();
-        // We are likely done at this point.
-        if (!nextRequestId)
-            return null;
-        // This should never happen, but...
-        if (this.inProgress.has(nextRequestId) || this.recentlyHandledRequestsCache.get(nextRequestId)) {
-            this.log.warning('Queue head returned a request that is already in progress?!', {
-                nextRequestId,
-                inProgress: this.inProgress.has(nextRequestId),
-                recentlyHandled: !!this.recentlyHandledRequestsCache.get(nextRequestId),
-            });
-            return null;
-        }
-        this.inProgress.add(nextRequestId);
-        this.lastActivity = new Date();
-        let request;
-        try {
-            request = await this.getRequest(nextRequestId);
-        }
-        catch (e) {
-            // On error, remove the request from in progress, otherwise it would be there forever
-            this.inProgress.delete(nextRequestId);
-            throw e;
-        }
-        // NOTE: It can happen that the queue head index is inconsistent with the main queue table. This can occur in two situations:
-        // 1) Queue head index is ahead of the main table and the request is not present in the main table yet (i.e. getRequest() returned null).
-        //    In this case, keep the request marked as in progress for a short while,
-        //    so that isFinished() doesn't return true and _ensureHeadIsNonEmpty() doesn't not load the request
-        //    into the queueHeadDict straight again. After the interval expires, fetchNextRequest()
-        //    will try to fetch this request again, until it eventually appears in the main table.
-        if (!request) {
-            this.log.debug('Cannot find a request from the beginning of queue, will be retried later', {
-                nextRequestId,
-            });
-            setTimeout(() => {
-                this.inProgress.delete(nextRequestId);
-            }, STORAGE_CONSISTENCY_DELAY_MILLIS);
-            return null;
-        }
-        // 2) Queue head index is behind the main table and the underlying request was already handled
-        //    (by some other client, since we keep the track of handled requests in recentlyHandled dictionary).
-        //    We just add the request to the recentlyHandled dictionary so that next call to _ensureHeadIsNonEmpty()
-        //    will not put the request again to queueHeadDict.
-        if (request.handledAt) {
-            this.log.debug('Request fetched from the beginning of queue was already handled', { nextRequestId });
-            this.recentlyHandledRequestsCache.add(nextRequestId, true);
-            return null;
-        }
-        return request;
-    }
-    async ensureHeadIsNonEmpty() {
-        // Alias for backwards compatibility
-        await this._ensureHeadIsNonEmpty();
-    }
-    /**
-     * We always request more items than is in progress to ensure that something falls into head.
-     *
-     * @param [ensureConsistency] If true then query for queue head is retried until queueModifiedAt
-     *   is older than queryStartedAt by at least API_PROCESSED_REQUESTS_DELAY_MILLIS to ensure that queue
-     *   head is consistent.
-     * @default false
-     * @param [limit] How many queue head items will be fetched.
-     * @param [iteration] Used when this function is called recursively to limit the recursion.
-     * @returns Indicates if queue head is consistent (true) or inconsistent (false).
-     */
-    async _ensureHeadIsNonEmpty(ensureConsistency = false, limit = Math.max(this.inProgressCount() * QUERY_HEAD_BUFFER, QUERY_HEAD_MIN_LENGTH), iteration = 0) {
-        // If we are paused for migration, resolve immediately.
-        if (this.queuePausedForMigration) {
-            return true;
-        }
-        // If is nonempty resolve immediately.
-        if (this.queueHeadIds.length() > 0) {
-            return true;
-        }
-        if (!this.queryQueueHeadPromise) {
-            const queryStartedAt = new Date();
-            this.queryQueueHeadPromise = this.client
-                .listHead({ limit })
-                .then(({ items, queueModifiedAt, hadMultipleClients }) => {
-                items.forEach(({ id: requestId, uniqueKey }) => {
-                    // Queue head index might be behind the main table, so ensure we don't recycle requests
-                    if (!requestId ||
-                        !uniqueKey ||
-                        this.inProgress.has(requestId) ||
-                        this.recentlyHandledRequestsCache.get(requestId))
-                        return;
-                    this.queueHeadIds.add(requestId, requestId, false);
-                    const forefront = this.requestCache.get(getRequestId(uniqueKey))?.forefront ?? false;
-                    this._cacheRequest(getRequestId(uniqueKey), {
-                        requestId,
-                        wasAlreadyHandled: false,
-                        wasAlreadyPresent: true,
-                        uniqueKey,
-                        forefront,
-                    });
-                });
-                // This is needed so that the next call to _ensureHeadIsNonEmpty() will fetch the queue head again.
-                this.queryQueueHeadPromise = null;
-                return {
-                    wasLimitReached: items.length >= limit,
-                    prevLimit: limit,
-                    queueModifiedAt: new Date(queueModifiedAt),
-                    queryStartedAt,
-                    hadMultipleClients,
-                };
-            });
-        }
-        const { queueModifiedAt, wasLimitReached, prevLimit, queryStartedAt, hadMultipleClients } = await this.queryQueueHeadPromise;
-        // TODO: I feel this code below can be greatly simplified...
-        // If queue is still empty then one of the following holds:
-        // - the other calls waiting for this promise already consumed all the returned requests
-        // - the limit was too low and contained only requests in progress
-        // - the writes from other clients were not propagated yet
-        // - the whole queue was processed and we are done
-        // If limit was not reached in the call then there are no more requests to be returned.
-        if (prevLimit >= REQUEST_QUEUE_HEAD_MAX_LIMIT) {
-            this.log.warning(`Reached the maximum number of requests in progress: ${REQUEST_QUEUE_HEAD_MAX_LIMIT}.`);
-        }
-        const shouldRepeatWithHigherLimit = this.queueHeadIds.length() === 0 && wasLimitReached && prevLimit < REQUEST_QUEUE_HEAD_MAX_LIMIT;
-        // If ensureConsistency=true then we must ensure that either:
-        // - queueModifiedAt is older than queryStartedAt by at least API_PROCESSED_REQUESTS_DELAY_MILLIS
-        // - hadMultipleClients=false and this.assumedTotalCount<=this.assumedHandledCount
-        const isDatabaseConsistent = +queryStartedAt - +queueModifiedAt >= API_PROCESSED_REQUESTS_DELAY_MILLIS;
-        const isLocallyConsistent = !hadMultipleClients && this.assumedTotalCount <= this.assumedHandledCount;
-        // Consistent information from one source is enough to consider request queue finished.
-        const shouldRepeatForConsistency = ensureConsistency && !isDatabaseConsistent && !isLocallyConsistent;
-        // If both are false then head is consistent and we may exit.
-        if (!shouldRepeatWithHigherLimit && !shouldRepeatForConsistency)
-            return true;
-        // If we are querying for consistency then we limit the number of queries to MAX_QUERIES_FOR_CONSISTENCY.
-        // If this is reached then we return false so that empty() and finished() returns possibly false negative.
-        if (!shouldRepeatWithHigherLimit && iteration > MAX_QUERIES_FOR_CONSISTENCY)
-            return false;
-        const nextLimit = shouldRepeatWithHigherLimit ? Math.round(prevLimit * 1.5) : prevLimit;
-        // If we are repeating for consistency then wait required time.
-        if (shouldRepeatForConsistency) {
-            const delayMillis = API_PROCESSED_REQUESTS_DELAY_MILLIS - (Date.now() - +queueModifiedAt);
-            this.log.info(`Waiting for ${delayMillis}ms before considering the queue as finished to ensure that the data is consistent.`);
-            await sleep(delayMillis);
-        }
-        return this._ensureHeadIsNonEmpty(ensureConsistency, nextLimit, iteration + 1);
-    }
-    // RequestQueue v1 behavior overrides below
-    async isFinished() {
-        checkStorageAccess();
-        if (Date.now() - +this.lastActivity > this.internalTimeoutMillis) {
-            const message = `The request queue seems to be stuck for ${this.internalTimeoutMillis / 1e3}s, resetting internal state.`;
-            this.log.warning(message, { inProgress: [...this.inProgress] });
-            this._reset();
-        }
-        if (this.inProgressRequestBatchCount > 0) {
-            return false;
-        }
-        if (this.queueHeadIds.length() > 0 || this.inProgressCount() > 0)
-            return false;
-        const isHeadConsistent = await this._ensureHeadIsNonEmpty(true);
-        return isHeadConsistent && this.queueHeadIds.length() === 0 && this.inProgressCount() === 0;
-    }
-    /**
-     * Reclaims a failed request back to the queue, so that it can be returned for processing later again
-     * by another call to {@link RequestQueue.fetchNextRequest}.
-     * The request record in the queue is updated using the provided `request` parameter.
-     * For example, this lets you store the number of retries or error messages for the request.
-     */
-    async reclaimRequest(...args) {
-        checkStorageAccess();
-        const [request, options] = args;
-        const forefront = options?.forefront ?? false;
-        const result = await super.reclaimRequest(...args);
-        // Wait a little to increase a chance that the next call to fetchNextRequest() will return the request with updated data.
-        // This is to compensate for the limitation of DynamoDB, where writes might not be immediately visible to subsequent reads.
-        setTimeout(() => {
-            if (!this.inProgress.has(request.id)) {
-                this.log.debug('The request is no longer marked as in progress in the queue?!', {
-                    requestId: request.id,
-                });
-                return;
-            }
-            this.inProgress.delete(request.id);
-            // Performance optimization: add request straight to head if possible
-            this._maybeAddRequestToQueueHead(request.id, forefront);
-        }, STORAGE_CONSISTENCY_DELAY_MILLIS);
-        return result;
-    }
-    /**
-     * @inheritdoc
-     */
-    async markRequestHandled(request) {
-        const res = await super.markRequestHandled(request);
-        this.inProgress.delete(request.id);
-        return res;
-    }
-    _reset() {
-        super._reset();
-        this.inProgress.clear();
-    }
-    /**
-     * Opens a request queue and returns a promise resolving to an instance
-     * of the {@link RequestQueue} class.
-     *
-     * {@link RequestQueue} represents a queue of URLs to crawl, which is stored either on local filesystem or in the cloud.
-     * The queue is used for deep crawling of websites, where you start with several URLs and then
-     * recursively follow links to other pages. The data structure supports both breadth-first
-     * and depth-first crawling orders.
-     *
-     * For more details and code examples, see the {@link RequestQueue} class.
-     *
-     * @param [queueIdOrName]
-     *   ID or name of the request queue to be opened. If `null` or `undefined`,
-     *   the function returns the default request queue associated with the crawler run.
-     * @param [options] Open Request Queue options.
-     */
-    static async open(...args) {
-        return super.open(...args);
-    }
-}
-export { RequestQueue as RequestQueueV1 };
-//# sourceMappingURL=request_queue.js.map

package/storages/request_queue.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC;AAE7D,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EACH,mCAAmC,EACnC,YAAY,EACZ,2BAA2B,EAC3B,iBAAiB,EACjB,qBAAqB,EACrB,gCAAgC,GACnC,MAAM,YAAY,CAAC;AAEpB,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAM,YAAa,SAAQ,eAAe;IAC9B,qBAAqB,GAMjB,IAAI,CAAC;IAET,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAEvC;;OAEG;IACH,YAAY,OAA+B,EAAE,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE;QACjF,KAAK,CACD;YACI,GAAG,OAAO;YACV,SAAS,EAAE,cAAc;YACzB,8BAA8B,EAAE,2BAA2B;YAC3D,mBAAmB,EAAE,mBAAmB;SAC3C,EACD,MAAM,CACT,CAAC;IACN,CAAC;IAED;;OAEG;IACI,eAAe;QAClB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACM,KAAK,CAAC,gBAAgB;QAC3B,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAEtD,oCAAoC;QACpC,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAEhC,mCAAmC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;YAC7F,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;gBAC5E,aAAa;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC9C,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC;aAC1E,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,IAAI,OAAuB,CAAC;QAC5B,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,qFAAqF;YACrF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,CAAC;QACZ,CAAC;QAED,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0EAA0E,EAAE;gBACvF,aAAa;aAChB,CAAC,CAAC;YACH,UAAU,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC1C,CAAC,EAAE,gCAAgC,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC3D,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,oCAAoC;QACpC,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACvC,CAAC;IAED;;;;;;;;;;OAUG;IACO,KAAK,CAAC,qBAAqB,CACjC,iBAAiB,GAAG,KAAK,EACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,iBAAiB,EAAE,qBAAqB,CAAC,EACnF,SAAS,GAAG,CAAC;QAEb,uDAAuD;QACvD,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,sCAAsC;QACtC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC9B,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC;YAElC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,MAAM;iBACnC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC;iBACnB,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,EAAE;gBACrD,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE;oBAC3C,uFAAuF;oBACvF,IACI,CAAC,SAAS;wBACV,CAAC,SAAS;wBACV,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;wBAC9B,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAU,CAAC;wBAEjD,OAAO;oBAEX,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;oBACnD,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;oBACrF,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE;wBACxC,SAAS;wBACT,iBAAiB,EAAE,KAAK;wBACxB,iBAAiB,EAAE,IAAI;wBACvB,SAAS;wBACT,SAAS;qBACZ,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEH,mGAAmG;gBACnG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;gBAElC,OAAO;oBACH,eAAe,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK;oBACtC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI,IAAI,CAAC,eAAe,CAAC;oBAC1C,cAAc;oBACd,kBAAkB;iBACrB,CAAC;YACN,CAAC,CAAC,CAAC;QACX,CAAC;QAED,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,GACrF,MAAM,IAAI,CAAC,qBAAqB,CAAC;QAErC,4DAA4D;QAE5D,2DAA2D;QAC3D,wFAAwF;QACxF,kEAAkE;QAClE,0DAA0D;QAC1D,kDAAkD;QAElD,uFAAuF;QACvF,IAAI,SAAS,IAAI,4BAA4B,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uDAAuD,4BAA4B,GAAG,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,2BAA2B,GAC7B,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,eAAe,IAAI,SAAS,GAAG,4BAA4B,CAAC;QAEpG,6DAA6D;QAC7D,iGAAiG;QACjG,kFAAkF;QAClF,MAAM,oBAAoB,GAAG,CAAC,cAAc,GAAG,CAAC,eAAe,IAAI,mCAAmC,CAAC;QACvG,MAAM,mBAAmB,GAAG,CAAC,kBAAkB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,mBAAmB,CAAC;QACtG,uFAAuF;QACvF,MAAM,0BAA0B,GAAG,iBAAiB,IAAI,CAAC,oBAAoB,IAAI,CAAC,mBAAmB,CAAC;QAEtG,6DAA6D;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,0BAA0B;YAAE,OAAO,IAAI,CAAC;QAE7E,yGAAyG;QACzG,0GAA0G;QAC1G,IAAI,CAAC,2BAA2B,IAAI,SAAS,GAAG,2BAA2B;YAAE,OAAO,KAAK,CAAC;QAE1F,MAAM,SAAS,GAAG,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAExF,+DAA+D;QAC/D,IAAI,0BAA0B,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,mCAAmC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC1F,IAAI,CAAC,GAAG,CAAC,IAAI,CACT,eAAe,WAAW,oFAAoF,CACjH,CAAC;YACF,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,2CAA2C;IAClC,KAAK,CAAC,UAAU;QACrB,kBAAkB,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC/D,MAAM,OAAO,GAAG,2CACZ,IAAI,CAAC,qBAAqB,GAAG,GACjC,8BAA8B,CAAC;YAC/B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,CAAC,2BAA2B,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/E,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAChG,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAmD;QAChF,kBAAkB,EAAE,CAAC;QAErB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;QAChC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,KAAK,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;QAEnD,yHAAyH;QACzH,2HAA2H;QAC3H,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAG,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE;oBAC5E,SAAS,EAAE,OAAO,CAAC,EAAE;iBACxB,CAAC,CAAC;gBACH,OAAO;YACX,CAAC;YAED,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAG,CAAC,CAAC;YAEpC,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,EAAG,EAAE,SAAS,CAAC,CAAC;QAC7D,CAAC,EAAE,gCAAgC,CAAC,CAAC;QAErC,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QAC9C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEpD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAG,CAAC,CAAC;QAEpC,OAAO,GAAG,CAAC;IACf,CAAC;IAEkB,MAAM;QACrB,KAAK,CAAC,MAAM,EAAE,CAAC;QAEf,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAU,KAAK,CAAC,IAAI,CAAC,GAAG,IAA6C;QACvE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAA0B,CAAC;IACxD,CAAC;CACJ;AAED,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,CAAC"}

package/storages/request_queue_v2.d.ts

@@ -1,87 +0,0 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { Configuration } from '../configuration.js';
-import type { Request, Source } from '../request.js';
-import type { RequestProviderOptions, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike } from './request_provider.js';
-import { RequestProvider } from './request_provider.js';
-/**
- * Represents a queue of URLs to crawl, which is used for deep crawling of websites
- * where you start with several URLs and then recursively
- * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
- *
- * Each URL is represented using an instance of the {@link Request} class.
- * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
- * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
- * To add a single URL multiple times to the queue,
- * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
- *
- * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
- *
- * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
- * and {@link PlaywrightCrawler} as a source of URLs to crawl.
- * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
- * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Open the default request queue associated with the crawler run
- * const queue = await RequestQueue.open();
- *
- * // Open a named request queue
- * const queueWithName = await RequestQueue.open('some-name');
- *
- * // Enqueue few requests
- * await queue.addRequest({ url: 'http://example.com/aaa' });
- * await queue.addRequest({ url: 'http://example.com/bbb' });
- * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
- * ```
- * @category Sources
- */
-export declare class RequestQueue extends RequestProvider {
-    private listHeadAndLockPromise;
-    private queueHasLockedRequests;
-    private shouldCheckForForefrontRequests;
-    private dequeuedRequestCount;
-    constructor(options: RequestProviderOptions, config?: Configuration);
-    /**
-     * Caches information about request to beware of unneeded addRequest() calls.
-     */
-    protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
-    /**
-     * @inheritDoc
-     */
-    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
-    /**
-     * @inheritDoc
-     */
-    addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
-    /**
-     * @inheritDoc
-     */
-    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
-    /**
-     * @inheritDoc
-     */
-    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
-    /**
-     * @inheritDoc
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>): ReturnType<RequestProvider['reclaimRequest']>;
-    protected ensureHeadIsNonEmpty(): Promise<void>;
-    private giveUpLock;
-    private _listHeadAndLock;
-    private getOrHydrateRequest;
-    private _prolongRequestLock;
-    protected _reset(): void;
-    protected _maybeAddRequestToQueueHead(): void;
-    protected _clearPossibleLocks(): Promise<void>;
-    /**
-     * @inheritDoc
-     */
-    static open(...args: Parameters<typeof RequestProvider.open>): Promise<RequestQueue>;
-}
-//# sourceMappingURL=request_queue_v2.d.ts.map

package/storages/request_queue_v2.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"request_queue_v2.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue_v2.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,KAAK,EACR,sBAAsB,EACtB,yBAAyB,EACzB,4BAA4B,EAC5B,YAAY,EACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAiBxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,YAAa,SAAQ,eAAe;IAC7C,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,OAAO,CAAC,sBAAsB,CAAkC;IAChE,OAAO,CAAC,+BAA+B,CAAS;IAChD,OAAO,CAAC,oBAAoB,CAAK;gBAErB,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAkC;IAsBrF;;OAEG;cACgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAevG;;OAEG;IACY,UAAU,CACrB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAQrC;;OAEG;IACY,WAAW,CACtB,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAWlC;;OAEG;IACY,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAiDhG;;OAEG;IACY,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAK9F;;OAEG;IACY,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IA+E7C;;OAEG;IACY,cAAc,CACzB,GAAG,IAAI,EAAE,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC,GACvD,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;cAqBhC,oBAAoB;YAqBtB,UAAU;YAYV,gBAAgB;YAmFhB,mBAAmB;YAuFnB,mBAAmB;cAiBd,MAAM;cAMN,2BAA2B;cAI9B,mBAAmB;IAcnC;;OAEG;WACmB,IAAI,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;CAGtG"}