@crawlee/core 4.0.0-beta.5 → 4.0.0-beta.51

package/storages/request_manager_tandem.d.ts

@@ -0,0 +1,68 @@
+import type { Dictionary } from '@crawlee/types';
+import type { Request, Source } from '../request.js';
+import type { IRequestList } from './request_list.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, IRequestManager, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike } from './request_provider.js';
+/**
+ * A request manager that combines a RequestList and a RequestQueue.
+ * It first reads requests from the RequestList and then, when needed,
+ * transfers them in batches to the RequestQueue.
+ */
+export declare class RequestManagerTandem implements IRequestManager {
+    private log;
+    private requestList;
+    private requestQueue;
+    constructor(requestList: IRequestList, requestQueue: IRequestManager);
+    /**
+     * Transfers a batch of requests from the RequestList to the RequestQueue.
+     * Handles both successful transfers and failures appropriately.
+     * @private
+     */
+    private transferNextBatchToQueue;
+    /**
+     * Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
+     * is not finished, it will transfer a batch of requests from the list to the queue first.
+     * @inheritdoc
+     */
+    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+    /**
+     * @inheritdoc
+     */
+    isFinished(): Promise<boolean>;
+    /**
+     * @inheritdoc
+     */
+    isEmpty(): Promise<boolean>;
+    /**
+     * @inheritdoc
+     */
+    handledCount(): Promise<number>;
+    /**
+     * @inheritdoc
+     */
+    getTotalCount(): number;
+    /**
+     * @inheritdoc
+     */
+    getPendingCount(): number;
+    /**
+     * @inheritdoc
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
+    /**
+     * @inheritdoc
+     */
+    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    /**
+     * @inheritdoc
+     */
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    /**
+     * @inheritdoc
+     */
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /**
+     * @inheritdoc
+     */
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+}
+//# sourceMappingURL=request_manager_tandem.d.ts.map

package/storages/request_manager_tandem.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request_manager_tandem.d.ts","sourceRoot":"","sources":["../../src/storages/request_manager_tandem.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,eAAe,EACf,yBAAyB,EACzB,4BAA4B,EAC5B,YAAY,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;;GAIG;AACH,qBAAa,oBAAqB,YAAW,eAAe;IACxD,OAAO,CAAC,GAAG,CAAgB;IAC3B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,YAAY,CAAkB;gBAE1B,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe;IAMpE;;;;OAIG;YACW,wBAAwB;IAuBtC;;;;OAIG;IACG,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAevF;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAKpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAKrC;;OAEG;IACH,aAAa,IAAI,MAAM;IAIvB;;OAEG;IACH,eAAe,IAAI,MAAM;IAIzB;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC;IAI5F;;OAEG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,CAAC,EAAE,4BAA4B,GACvC,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAI5C;;OAEG;IACG,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAIjH;;OAEG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,EACtB,OAAO,CAAC,EAAE,yBAAyB,GACpC,OAAO,CAAC,wBAAwB,CAAC;CAGvC"}

package/storages/request_manager_tandem.js

@@ -0,0 +1,124 @@
+import { serviceLocator } from '../service_locator.js';
+/**
+ * A request manager that combines a RequestList and a RequestQueue.
+ * It first reads requests from the RequestList and then, when needed,
+ * transfers them in batches to the RequestQueue.
+ */
+export class RequestManagerTandem {
+    log;
+    requestList;
+    requestQueue;
+    constructor(requestList, requestQueue) {
+        this.log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
+        this.requestList = requestList;
+        this.requestQueue = requestQueue;
+    }
+    /**
+     * Transfers a batch of requests from the RequestList to the RequestQueue.
+     * Handles both successful transfers and failures appropriately.
+     * @private
+     */
+    async transferNextBatchToQueue() {
+        const request = await this.requestList.fetchNextRequest();
+        if (request === null) {
+            return;
+        }
+        try {
+            await this.requestQueue.addRequest(request, { forefront: true });
+        }
+        catch (error) {
+            // If requestQueue.addRequest() fails here then we must reclaim it back to
+            // the RequestList because probably it's not yet in the queue!
+            this.log.error('Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.', { request });
+            await this.requestList.reclaimRequest(request);
+            return;
+        }
+        await this.requestList.markRequestHandled(request);
+    }
+    /**
+     * Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
+     * is not finished, it will transfer a batch of requests from the list to the queue first.
+     * @inheritdoc
+     */
+    async fetchNextRequest() {
+        // First, try to transfer a request from the requestList
+        const [listEmpty, listFinished] = await Promise.all([
+            this.requestList.isEmpty(),
+            this.requestList.isFinished(),
+        ]);
+        if (!listEmpty && !listFinished) {
+            await this.transferNextBatchToQueue();
+        }
+        // Try to fetch from queue after potential transfer
+        return this.requestQueue.fetchNextRequest();
+    }
+    /**
+     * @inheritdoc
+     */
+    async isFinished() {
+        const storagesFinished = await Promise.all([this.requestList.isFinished(), this.requestQueue.isFinished()]);
+        return storagesFinished.every(Boolean);
+    }
+    /**
+     * @inheritdoc
+     */
+    async isEmpty() {
+        const storagesEmpty = await Promise.all([this.requestList.isEmpty(), this.requestQueue.isEmpty()]);
+        return storagesEmpty.every(Boolean);
+    }
+    /**
+     * @inheritdoc
+     */
+    async handledCount() {
+        // Since one of the stores needs to have priority when both are present, we query the request queue - the request list will first be dumped into the queue and then left empty.
+        return await this.requestQueue.handledCount();
+    }
+    /**
+     * @inheritdoc
+     */
+    getTotalCount() {
+        return this.requestQueue.getTotalCount();
+    }
+    /**
+     * @inheritdoc
+     */
+    getPendingCount() {
+        return this.requestQueue.getPendingCount() + this.requestList.length() - this.requestList.handledCount();
+    }
+    /**
+     * @inheritdoc
+     */
+    async *[Symbol.asyncIterator]() {
+        while (true) {
+            const req = await this.fetchNextRequest();
+            if (!req)
+                break;
+            yield req;
+        }
+    }
+    /**
+     * @inheritdoc
+     */
+    async markRequestHandled(request) {
+        return this.requestQueue.markRequestHandled(request);
+    }
+    /**
+     * @inheritdoc
+     */
+    async reclaimRequest(request, options) {
+        return await this.requestQueue.reclaimRequest(request, options);
+    }
+    /**
+     * @inheritdoc
+     */
+    async addRequest(requestLike, options) {
+        return await this.requestQueue.addRequest(requestLike, options);
+    }
+    /**
+     * @inheritdoc
+     */
+    async addRequestsBatched(requests, options) {
+        return await this.requestQueue.addRequestsBatched(requests, options);
+    }
+}
+//# sourceMappingURL=request_manager_tandem.js.map

package/storages/request_manager_tandem.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request_manager_tandem.js","sourceRoot":"","sources":["../../src/storages/request_manager_tandem.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAWvD;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IACrB,GAAG,CAAgB;IACnB,WAAW,CAAe;IAC1B,YAAY,CAAkB;IAEtC,YAAY,WAAyB,EAAE,YAA6B;QAChE,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,wBAAwB;QAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,yGAAyG,EACzG,EAAE,OAAO,EAAE,CACd,CAAC;YACF,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QAClB,wDAAwD;QACxD,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChD,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE;YAC1B,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAC1C,CAAC;QAED,mDAAmD;QACnD,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAK,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC5G,OAAO,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnG,OAAO,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,+KAA+K;QAC/K,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,aAAa;QACT,OAAO,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,eAAe;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;IAC7G,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,OAAsC;QAEtC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,OAAsC;QACxE,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAAsB,EACtB,OAAmC;QAEnC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;CACJ"}

package/storages/request_provider.d.ts

@@ -1,25 +1,79 @@
-import type { BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueClient, RequestQueueInfo, StorageClient } from '@crawlee/types';
+import type { BaseHttpClient, BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueClient, RequestQueueInfo } from '@crawlee/types';
+import type { ReadonlyDeep } from 'type-fest';
 import { ListDictionary, LruCache } from '@apify/datastructures';
-import type { Log } from '@apify/log';
 import { Configuration } from '../configuration.js';
+import type { EventManager } from '../events/event_manager.js';
+import type { CrawleeLogger } from '../log.js';
 import type { ProxyConfiguration } from '../proxy_configuration.js';
 import type { InternalSource, RequestOptions, Source } from '../request.js';
 import { Request } from '../request.js';
-import type { IStorage, StorageManagerOptions } from './storage_manager.js';
-export declare abstract class RequestProvider implements IStorage {
-    readonly config: Configuration;
+import type { IStorage, StorageIdentifier, StorageManagerOptions } from './storage_manager.js';
+export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
+/**
+ * Represents a provider of requests/URLs to crawl.
+ */
+export interface IRequestManager {
+    /**
+     * Returns `true` if all requests were already handled and there are no more left.
+     */
+    isFinished(): Promise<boolean>;
+    /**
+     * Resolves to `true` if the next call to {@link IRequestManager.fetchNextRequest} function
+     * would return `null`, otherwise it resolves to `false`.
+     * Note that even if the provider is empty, there might be some pending requests currently being processed.
+     */
+    isEmpty(): Promise<boolean>;
+    /**
+     * Returns number of handled requests.
+     */
+    handledCount(): Promise<number>;
+    /**
+     * Get the total number of requests known to the request manager.
+     */
+    getTotalCount(): number;
+    /**
+     * Get an offline approximation of the number of pending requests.
+     */
+    getPendingCount(): number;
+    /**
+     * Gets the next {@link Request} to process.
+     *
+     * The function's `Promise` resolves to `null` if there are no more
+     * requests to process.
+     */
+    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+    /**
+     * Can be used to iterate over the `RequestManager` instance in a `for await .. of` loop.
+     * Provides an alternative for the repeated use of `fetchNextRequest`.
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Request>;
+    /**
+     * Marks request as handled after successful processing.
+     */
+    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    /**
+     * Reclaims request to the provider if its processing failed.
+     * The request will become available in the next `fetchNextRequest()`.
+     */
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+}
+export declare abstract class RequestProvider implements IStorage, IRequestManager {
+    protected readonly config: Configuration;
     id: string;
     name?: string;
     timeoutSecs: number;
     clientKey: string;
     client: RequestQueueClient;
     protected proxyConfiguration?: ProxyConfiguration;
-    log: Log;
+    log: CrawleeLogger;
     internalTimeoutMillis: number;
     requestLockSecs: number;
     assumedTotalCount: number;
     assumedHandledCount: number;
     private initialCount;
+    private initialHandledCount;
     protected queueHeadIds: ListDictionary<string>;
     protected requestCache: LruCache<RequestLruItem>;
     protected recentlyHandledRequestsCache: LruCache<boolean>;
@@ -27,13 +81,21 @@ export declare abstract class RequestProvider implements IStorage {
     protected lastActivity: Date;
     protected isFinishedCalledWhileHeadWasNotEmpty: number;
     protected inProgressRequestBatchCount: number;
-    constructor(options: InternalRequestProviderOptions, config?: Configuration);
+    protected httpClient?: BaseHttpClient;
+    protected readonly events: EventManager;
+    constructor(options: InternalRequestProviderOptions, config: Configuration);
     /**
      * Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).
      *
      * Survives restarts and actor migrations.
      */
     getTotalCount(): number;
+    /**
+     * Returns an offline approximation of the total number of pending requests in the queue.
+     *
+     * Survives restarts and Actor migrations.
+     */
+    getPendingCount(): number;
     /**
      * Adds a request to the queue.
      *
@@ -63,7 +125,7 @@ export declare abstract class RequestProvider implements IStorage {
      * Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
      * @param [options] Request queue operation options.
      */
-    addRequests(requestsLike: Source[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
+    addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in the background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -73,7 +135,7 @@ export declare abstract class RequestProvider implements IStorage {
      * @param requests The requests to add
      * @param options Options for the request queue
      */
-    addRequestsBatched(requests: (string | Source)[], options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
     /**
      * Gets the request from the queue specified by ID.
      *
@@ -98,7 +160,7 @@ export declare abstract class RequestProvider implements IStorage {
      * @returns
      *   Returns the request object or `null` if there are no more pending requests.
      */
-    abstract fetchNextRequest<T extends Dictionary = Dictionary>(options?: RequestOptions): Promise<Request<T> | null>;
+    abstract fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
     /**
      * Marks a request that was previously returned by the
      * {@link RequestQueue.fetchNextRequest}
@@ -142,6 +204,10 @@ export declare abstract class RequestProvider implements IStorage {
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * @inheritdoc
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
     /**
      * Returns the number of handled requests.
      *
@@ -150,23 +216,17 @@ export declare abstract class RequestProvider implements IStorage {
      * ```javascript
      * const { handledRequestCount } = await queue.getInfo();
      * ```
+     * @inheritdoc
      */
     handledCount(): Promise<number>;
     /**
      * Returns an object containing general information about the request queue.
      *
-     * The function returns the same object as the Apify API Client's
-     * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
-     * function, which in turn calls the
-     * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
-     * API endpoint.
-     *
      * **Example:**
      * ```
      * {
      *   id: "WkzbQMuFYuamGv3YF",
      *   name: "my-queue",
-     *   userId: "wRsJZtadYvn4mBZmm",
      *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
      *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
      *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
@@ -175,8 +235,10 @@ export declare abstract class RequestProvider implements IStorage {
      *   pendingRequestCount: 20,
      * }
      * ```
+     *
+     * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
-    getInfo(): Promise<RequestQueueInfo | undefined>;
+    getInfo(): Promise<RequestQueueInfo>;
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
@@ -200,12 +262,13 @@ export declare abstract class RequestProvider implements IStorage {
      *
      * For more details and code examples, see the {@link RequestQueue} class.
      *
-     * @param [queueIdOrName]
-     *   ID or name of the request queue to be opened. If `null` or `undefined`,
-     *   the function returns the default request queue associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the request queue to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default request queue associated with the crawler run.
      * @param [options] Open Request Queue options.
      */
-    static open(queueIdOrName?: string | null, options?: StorageManagerOptions): Promise<RequestProvider>;
+    static open(identifier?: string | StorageIdentifier | null, options?: StorageManagerOptions): Promise<RequestProvider>;
 }
 interface RequestLruItem {
     uniqueKey: string;
@@ -218,7 +281,7 @@ interface RequestLruItem {
 export interface RequestProviderOptions {
     id: string;
     name?: string;
-    client: StorageClient;
+    client: RequestQueueClient;
     /**
      * Used to pass the proxy configuration for the `requestsFromUrl` objects.
      * Takes advantage of the internal address rotation and authentication process.

package/storages/request_provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAChB,MAAM,gBAAgB,CAAC;AAIxB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAI5E,8BAAsB,eAAgB,YAAW,QAAQ;IAkCjD,QAAQ,CAAC,MAAM;IAjCnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,GAAG,CAAC;IACT,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IAEzB,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;gBAGtC,OAAO,EAAE,8BAA8B,EAC9B,MAAM,gBAAkC;IAsBrD;;;;OAIG;IACH,aAAa;IAIb;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,MAAM,EAAE,EACtB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAqGlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,EAC7B,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IAyHpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAElH;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;;;;;;;OAQG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAMtD;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAMjC;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC;CA2BlH;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IAEtB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D"}
+{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,cAAc,EACd,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EACnB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGjE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAIxC,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAI/F,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE5G;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhC;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAElF;;;OAGG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAElD;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAEvF;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC;IAEpH,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE5G,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtH;AAED,8BAAsB,eAAgB,YAAW,QAAQ,EAAE,eAAe;IAuClE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa;IAtC5C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,aAAa,CAAC;IACnB,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,mBAAmB,CAAK;IAEhC,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;IAE1C,SAAS,CAAC,UAAU,CAAC,EAAE,cAAc,CAAC;IAEtC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;gBAGpC,OAAO,EAAE,8BAA8B,EACpB,MAAM,EAAE,aAAa;IAoB5C;;;;OAIG;IACH,aAAa;IAIb;;;;OAIG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IA4GlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,CAAC,YAAY,CAAC,EACpC,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IAmIpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAE1F;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;;;;;;;;OASG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;OAkBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAM1C;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAWjC;;;;;;;;;;;;;;;;OAgBG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,qBAA0B,GACpC,OAAO,CAAC,eAAe,CAAC;CAwC9B;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,kBAAkB,CAAC;IAE3B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D"}