@crawlee/core 4.0.0-beta.6 → 4.0.0-beta.61

package/storages/request_provider.d.ts

@@ -1,25 +1,81 @@
-import type { BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueClient, RequestQueueInfo, StorageClient } from '@crawlee/types';
+import type { BaseHttpClient, BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueClient, RequestQueueInfo } from '@crawlee/types';
+import type { ReadonlyDeep } from 'type-fest';
 import { ListDictionary, LruCache } from '@apify/datastructures';
-import type { Log } from '@apify/log';
 import { Configuration } from '../configuration.js';
+import type { EventManager } from '../events/event_manager.js';
+import type { CrawleeLogger } from '../log.js';
 import type { ProxyConfiguration } from '../proxy_configuration.js';
 import type { InternalSource, RequestOptions, Source } from '../request.js';
 import { Request } from '../request.js';
-import type { IStorage, StorageManagerOptions } from './storage_manager.js';
-export declare abstract class RequestProvider implements IStorage {
-    readonly config: Configuration;
+import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
+import type { StorageOpenOptions } from './utils.js';
+export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
+/**
+ * Represents a provider of requests/URLs to crawl.
+ */
+export interface IRequestManager {
+    /**
+     * Returns `true` if all requests were already handled and there are no more left.
+     */
+    isFinished(): Promise<boolean>;
+    /**
+     * Resolves to `true` if the next call to {@link IRequestManager.fetchNextRequest} function
+     * would return `null`, otherwise it resolves to `false`.
+     * Note that even if the provider is empty, there might be some pending requests currently being processed.
+     */
+    isEmpty(): Promise<boolean>;
+    /**
+     * Returns number of handled requests.
+     */
+    handledCount(): Promise<number>;
+    /**
+     * Get the total number of requests known to the request manager.
+     */
+    getTotalCount(): number;
+    /**
+     * Get an offline approximation of the number of pending requests.
+     */
+    getPendingCount(): number;
+    /**
+     * Gets the next {@link Request} to process.
+     *
+     * The function's `Promise` resolves to `null` if there are no more
+     * requests to process.
+     */
+    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+    /**
+     * Can be used to iterate over the `RequestManager` instance in a `for await .. of` loop.
+     * Provides an alternative for the repeated use of `fetchNextRequest`.
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Request>;
+    /**
+     * Marks request as handled after successful processing.
+     */
+    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    /**
+     * Reclaims request to the provider if its processing failed.
+     * The request will become available in the next `fetchNextRequest()`.
+     */
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+}
+export declare abstract class RequestProvider implements IStorage, IRequestManager {
+    protected readonly config: Configuration;
     id: string;
     name?: string;
     timeoutSecs: number;
     clientKey: string;
     client: RequestQueueClient;
     protected proxyConfiguration?: ProxyConfiguration;
-    log: Log;
+    log: CrawleeLogger;
     internalTimeoutMillis: number;
     requestLockSecs: number;
     assumedTotalCount: number;
     assumedHandledCount: number;
     private initialCount;
+    private initialHandledCount;
+    private isInitialized;
     protected queueHeadIds: ListDictionary<string>;
     protected requestCache: LruCache<RequestLruItem>;
     protected recentlyHandledRequestsCache: LruCache<boolean>;
@@ -27,6 +83,8 @@ export declare abstract class RequestProvider implements IStorage {
     protected lastActivity: Date;
     protected isFinishedCalledWhileHeadWasNotEmpty: number;
     protected inProgressRequestBatchCount: number;
+    protected httpClient?: BaseHttpClient;
+    protected readonly events: EventManager;
     constructor(options: InternalRequestProviderOptions, config?: Configuration);
     /**
      * Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).
@@ -34,6 +92,12 @@ export declare abstract class RequestProvider implements IStorage {
      * Survives restarts and actor migrations.
      */
     getTotalCount(): number;
+    /**
+     * Returns an offline approximation of the total number of pending requests in the queue.
+     *
+     * Survives restarts and Actor migrations.
+     */
+    getPendingCount(): number;
     /**
      * Adds a request to the queue.
      *
@@ -63,7 +127,7 @@ export declare abstract class RequestProvider implements IStorage {
      * Note that the function sets the `uniqueKey` and `id` fields to the passed requests if missing.
      * @param [options] Request queue operation options.
      */
-    addRequests(requestsLike: Source[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
+    addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in the background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -73,7 +137,7 @@ export declare abstract class RequestProvider implements IStorage {
      * @param requests The requests to add
      * @param options Options for the request queue
      */
-    addRequestsBatched(requests: (string | Source)[], options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
     /**
      * Gets the request from the queue specified by ID.
      *
@@ -98,7 +162,7 @@ export declare abstract class RequestProvider implements IStorage {
      * @returns
      *   Returns the request object or `null` if there are no more pending requests.
      */
-    abstract fetchNextRequest<T extends Dictionary = Dictionary>(options?: RequestOptions): Promise<Request<T> | null>;
+    abstract fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
     /**
      * Marks a request that was previously returned by the
      * {@link RequestQueue.fetchNextRequest}
@@ -142,6 +206,10 @@ export declare abstract class RequestProvider implements IStorage {
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * @inheritdoc
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
     /**
      * Returns the number of handled requests.
      *
@@ -150,23 +218,17 @@ export declare abstract class RequestProvider implements IStorage {
      * ```javascript
      * const { handledRequestCount } = await queue.getInfo();
      * ```
+     * @inheritdoc
      */
     handledCount(): Promise<number>;
     /**
      * Returns an object containing general information about the request queue.
      *
-     * The function returns the same object as the Apify API Client's
-     * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
-     * function, which in turn calls the
-     * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
-     * API endpoint.
-     *
      * **Example:**
      * ```
      * {
      *   id: "WkzbQMuFYuamGv3YF",
      *   name: "my-queue",
-     *   userId: "wRsJZtadYvn4mBZmm",
      *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
      *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
      *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
@@ -175,8 +237,10 @@ export declare abstract class RequestProvider implements IStorage {
      *   pendingRequestCount: 20,
      * }
      * ```
+     *
+     * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
-    getInfo(): Promise<RequestQueueInfo | undefined>;
+    getInfo(): Promise<RequestQueueInfo>;
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
@@ -200,12 +264,13 @@ export declare abstract class RequestProvider implements IStorage {
      *
      * For more details and code examples, see the {@link RequestQueue} class.
      *
-     * @param [queueIdOrName]
-     *   ID or name of the request queue to be opened. If `null` or `undefined`,
-     *   the function returns the default request queue associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the request queue to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default request queue associated with the crawler run.
      * @param [options] Open Request Queue options.
      */
-    static open(queueIdOrName?: string | null, options?: StorageManagerOptions): Promise<RequestProvider>;
+    static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestProvider>;
 }
 interface RequestLruItem {
     uniqueKey: string;
@@ -218,7 +283,7 @@ interface RequestLruItem {
 export interface RequestProviderOptions {
     id: string;
     name?: string;
-    client: StorageClient;
+    client: RequestQueueClient;
     /**
      * Used to pass the proxy configuration for the `requestsFromUrl` objects.
      * Takes advantage of the internal address rotation and authentication process.

package/storages/request_provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAChB,MAAM,gBAAgB,CAAC;AAIxB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAI5E,8BAAsB,eAAgB,YAAW,QAAQ;IAkCjD,QAAQ,CAAC,MAAM;IAjCnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,GAAG,CAAC;IACT,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IAEzB,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;gBAGtC,OAAO,EAAE,8BAA8B,EAC9B,MAAM,gBAAkC;IAsBrD;;;;OAIG;IACH,aAAa;IAIb;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,MAAM,EAAE,EACtB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAqGlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,EAC7B,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IAyHpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAElH;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;;;;;;;OAQG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAMtD;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAMjC;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,CAAC;CA2BlH;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IAEtB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D"}
+{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,cAAc,EACd,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EACnB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGjE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAIrD,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE5G;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhC;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAElF;;;OAGG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAElD;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAEvF;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC;IAEpH,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE5G,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtH;AAED,8BAAsB,eAAgB,YAAW,QAAQ,EAAE,eAAe;IAwClE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa;IAvC5C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,aAAa,CAAC;IACnB,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,aAAa,CAAS;IAE9B,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;IAE1C,SAAS,CAAC,UAAU,CAAC,EAAE,cAAc,CAAC;IAEtC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;gBAGpC,OAAO,EAAE,8BAA8B,EACpB,MAAM,GAAE,aAA+C;IAoB9E;;;;OAIG;IACH,aAAa;IAIb;;;;OAIG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IA4GlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,CAAC,YAAY,CAAC,EACpC,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IAmIpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAE1F;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;;;;;;;;OASG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;OAkBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAM1C;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAWjC;;;;;;;;;;;;;;;;OAgBG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,eAAe,CAAC;CAmD9B;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,kBAAkB,CAAC;IAE3B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D"}

package/storages/request_provider.js

@@ -1,13 +1,13 @@
 import { inspect } from 'node:util';
-import { chunk, downloadListOfUrls, sleep } from '@crawlee/utils';
+import { chunkedAsyncIterable, downloadListOfUrls, getObjectType, isAsyncIterable, isIterable, peekableAsyncIterable, sleep, } from '@crawlee/utils';
 import ow from 'ow';
 import { ListDictionary, LruCache } from '@apify/datastructures';
 import { cryptoRandomObjectId } from '@apify/utilities';
 import { Configuration } from '../configuration.js';
-import { log } from '../log.js';
 import { Request } from '../request.js';
+import { serviceLocator } from '../service_locator.js';
 import { checkStorageAccess } from './access_checking.js';
-import { StorageManager } from './storage_manager.js';
+import { resolveStorageIdentifier } from './storage_instance_manager.js';
 import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './utils.js';
 export class RequestProvider {
     config;
@@ -25,6 +25,8 @@ export class RequestProvider {
     assumedTotalCount = 0;
     assumedHandledCount = 0;
     initialCount = 0;
+    initialHandledCount = 0; // We track this separately from `assumedHandledCount` which is used non-trivially by RequestQueueV1
+    isInitialized = false;
     queueHeadIds = new ListDictionary();
     requestCache;
     recentlyHandledRequestsCache;
@@ -32,20 +34,21 @@ export class RequestProvider {
     lastActivity = new Date();
     isFinishedCalledWhileHeadWasNotEmpty = 0;
     inProgressRequestBatchCount = 0;
+    httpClient;
+    events;
     constructor(options, config = Configuration.getGlobalConfig()) {
         this.config = config;
         this.id = options.id;
         this.name = options.name;
-        this.client = options.client.requestQueue(this.id, {
-            clientKey: this.clientKey,
-            timeoutSecs: this.timeoutSecs,
-        });
+        this.events = serviceLocator.getEventManager();
+        this.client = options.client;
         this.proxyConfiguration = options.proxyConfiguration;
         this.requestCache = new LruCache({ maxLength: options.requestCacheMaxSize });
         this.recentlyHandledRequestsCache = new LruCache({ maxLength: options.recentlyHandledRequestsMaxSize });
-        this.log = log.child({ prefix: `${options.logPrefix}(${this.id}, ${this.name ?? 'no-name'})` });
-        const eventManager = config.getEventManager();
-        eventManager.on("migrating" /* EventType.MIGRATING */, async () => {
+        this.log = serviceLocator
+            .getLogger()
+            .child({ prefix: `${options.logPrefix}(${this.id}, ${this.name ?? 'no-name'})` });
+        this.events.on("migrating" /* EventType.MIGRATING */, async () => {
             this.queuePausedForMigration = true;
         });
     }
@@ -57,6 +60,14 @@ export class RequestProvider {
     getTotalCount() {
         return this.assumedTotalCount + this.initialCount;
     }
+    /**
+     * Returns an offline approximation of the total number of pending requests in the queue.
+     *
+     * Survives restarts and Actor migrations.
+     */
+    getPendingCount() {
+        return this.getTotalCount() - this.initialHandledCount - this.assumedHandledCount;
+    }
     /**
      * Adds a request to the queue.
      *
@@ -134,7 +145,9 @@ export class RequestProvider {
     async addRequests(requestsLike, options = {}) {
         checkStorageAccess();
         this.lastActivity = new Date();
-        ow(requestsLike, ow.array);
+        ow(requestsLike, ow.object
+            .is((value) => isIterable(value) || isAsyncIterable(value))
+            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
         ow(options, ow.object.exactShape({
             forefront: ow.optional.boolean,
             cache: ow.optional.boolean,
@@ -153,17 +166,19 @@ export class RequestProvider {
             processedRequests: [],
             unprocessedRequests: [],
         };
-        for (const requestLike of requestsLike) {
-            if ('requestsFromUrl' in requestLike) {
-                const requests = await this._fetchRequestsFromUrl(requestLike);
-                await this._addFetchedRequests(requestLike, requests, options);
+        const requests = [];
+        for await (const requestLike of requestsLike) {
+            if (typeof requestLike === 'string') {
+                requests.push(new Request({ url: requestLike }));
+            }
+            else if ('requestsFromUrl' in requestLike) {
+                const fetchedRequests = await this._fetchRequestsFromUrl(requestLike);
+                await this._addFetchedRequests(requestLike, fetchedRequests, options);
+            }
+            else {
+                requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike));
             }
         }
-        const requests = requestsLike
-            .filter((requestLike) => !('requestsFromUrl' in requestLike))
-            .map((requestLike) => {
-            return requestLike instanceof Request ? requestLike : new Request(requestLike);
-        });
         const requestsToAdd = new Map();
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
@@ -219,43 +234,44 @@ export class RequestProvider {
     async addRequestsBatched(requests, options = {}) {
         checkStorageAccess();
         this.lastActivity = new Date();
+        ow(requests, ow.object
+            .is((value) => isIterable(value) || isAsyncIterable(value))
+            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
         ow(options, ow.object.exactShape({
             forefront: ow.optional.boolean,
             waitForAllRequestsToBeAdded: ow.optional.boolean,
             batchSize: ow.optional.number,
             waitBetweenBatchesMillis: ow.optional.number,
         }));
-        // The `requests` array can be huge, and `ow` is very slow for anything more complex.
-        // This explicit iteration takes a few milliseconds, while the ow check can take tens of seconds.
-        // ow(requests, ow.array.ofType(ow.any(
-        //     ow.string,
-        //     ow.object.partialShape({ url: ow.string, id: ow.undefined }),
-        //     ow.object.partialShape({ requestsFromUrl: ow.string, regex: ow.optional.regExp }),
-        // )));
-        for (const request of requests) {
-            if (typeof request === 'string') {
-                continue;
-            }
-            if (typeof request === 'object' && request !== null) {
-                if (typeof request.url === 'string' && typeof request.id === 'undefined') {
-                    continue;
+        const addRequest = this.addRequest.bind(this);
+        async function* generateRequests() {
+            for await (const opts of requests) {
+                // Validate the input
+                if (typeof opts === 'object' && opts !== null) {
+                    if (opts.url !== undefined && typeof opts.url !== 'string') {
+                        throw new Error(`Request options are not valid, the 'url' property is not a string. Input: ${inspect(opts)}`);
+                    }
+                    if (opts.id !== undefined) {
+                        throw new Error(`Request options are not valid, the 'id' property must not be present. Input: ${inspect(opts)}`);
+                    }
+                    if (opts.requestsFromUrl !== undefined &&
+                        typeof opts.requestsFromUrl !== 'string') {
+                        throw new Error(`Request options are not valid, the 'requestsFromUrl' property is not a string. Input: ${inspect(opts)}`);
+                    }
                 }
-                if (typeof request.requestsFromUrl === 'string') {
-                    continue;
+                if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
+                    // Handle URL lists right away
+                    await addRequest(opts, { forefront: options.forefront });
+                }
+                else {
+                    // Yield valid requests
+                    yield typeof opts === 'string' ? { url: opts } : opts;
                 }
             }
-            throw new Error(`Request options are not valid, provide either a URL or an object with 'url' property (but without 'id' property), or an object with 'requestsFromUrl' property. Input: ${inspect(request)}`);
         }
         const { batchSize = 1000, waitBetweenBatchesMillis = 1000 } = options;
-        const sources = [];
-        for (const opts of requests) {
-            if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) {
-                await this.addRequest(opts, { forefront: options.forefront });
-            }
-            else {
-                sources.push(typeof opts === 'string' ? { url: opts } : opts);
-            }
-        }
+        const chunks = peekableAsyncIterable(chunkedAsyncIterable(generateRequests(), batchSize));
+        const chunksIterator = chunks[Symbol.asyncIterator]();
         const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true) => {
             const resultsToReturn = [];
             const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
@@ -266,11 +282,15 @@ export class RequestProvider {
             }
             return resultsToReturn;
         };
-        const initialChunk = sources.splice(0, batchSize);
         // Add initial batch of `batchSize` to process them right away
+        const initialChunk = await chunksIterator.peek();
+        if (initialChunk === undefined) {
+            return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]) };
+        }
         const addedRequests = await attemptToAddToQueueAndAddAnyUnprocessed(initialChunk);
-        // If we have no more requests to add, return early
-        if (!sources.length) {
+        await chunksIterator.next();
+        // If we have no more requests to add, return immediately
+        if ((await chunksIterator.peek()) === undefined) {
             return {
                 addedRequests,
                 waitForAllRequestsToBeAdded: Promise.resolve([]),
@@ -278,9 +298,8 @@ export class RequestProvider {
         }
         // eslint-disable-next-line no-async-promise-executor
         const promise = new Promise(async (resolve) => {
-            const chunks = chunk(sources, batchSize);
             const finalAddedRequests = [];
-            for (const requestChunk of chunks) {
+            for await (const requestChunk of chunks) {
                 finalAddedRequests.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(requestChunk, false)));
                 await sleep(waitBetweenBatchesMillis);
             }
@@ -425,9 +444,19 @@ export class RequestProvider {
      */
     async drop() {
         checkStorageAccess();
-        await this.client.delete();
-        const manager = StorageManager.getManager(this.constructor, this.config);
-        manager.closeStorage(this);
+        await this.client.drop();
+        serviceLocator.getStorageInstanceManager().removeFromCache(this);
+    }
+    /**
+     * @inheritdoc
+     */
+    async *[Symbol.asyncIterator]() {
+        while (true) {
+            const req = await this.fetchNextRequest();
+            if (!req)
+                break;
+            yield req;
+        }
     }
     /**
      * Returns the number of handled requests.
@@ -437,27 +466,21 @@ export class RequestProvider {
      * ```javascript
      * const { handledRequestCount } = await queue.getInfo();
      * ```
+     * @inheritdoc
      */
     async handledCount() {
         // NOTE: We keep this function for compatibility with RequestList.handledCount()
-        const { handledRequestCount } = (await this.getInfo()) ?? {};
-        return handledRequestCount ?? 0;
+        const { handledRequestCount } = await this.getInfo();
+        return handledRequestCount;
     }
     /**
      * Returns an object containing general information about the request queue.
      *
-     * The function returns the same object as the Apify API Client's
-     * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
-     * function, which in turn calls the
-     * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
-     * API endpoint.
-     *
      * **Example:**
      * ```
      * {
      *   id: "WkzbQMuFYuamGv3YF",
      *   name: "my-queue",
-     *   userId: "wRsJZtadYvn4mBZmm",
      *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
      *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
      *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
@@ -466,10 +489,12 @@ export class RequestProvider {
      *   pendingRequestCount: 20,
      * }
      * ```
+     *
+     * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
         checkStorageAccess();
-        return this.client.get();
+        return this.client.getMetadata();
     }
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
@@ -515,7 +540,10 @@ export class RequestProvider {
      * @internal wraps public utility for mocking purposes
      */
     async _downloadListOfUrls(options) {
-        return downloadListOfUrls(options);
+        return downloadListOfUrls({
+            ...options,
+            httpClient: this.httpClient,
+        });
     }
     /**
      * Opens a request queue and returns a promise resolving to an instance
@@ -528,27 +556,49 @@ export class RequestProvider {
      *
      * For more details and code examples, see the {@link RequestQueue} class.
      *
-     * @param [queueIdOrName]
-     *   ID or name of the request queue to be opened. If `null` or `undefined`,
-     *   the function returns the default request queue associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the request queue to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default request queue associated with the crawler run.
      * @param [options] Open Request Queue options.
      */
-    static async open(queueIdOrName, options = {}) {
+    static async open(identifier, options = {}) {
         checkStorageAccess();
-        ow(queueIdOrName, ow.optional.any(ow.string, ow.null));
         ow(options, ow.object.exactShape({
             config: ow.optional.object.instanceOf(Configuration),
             storageClient: ow.optional.object,
             proxyConfiguration: ow.optional.object,
+            httpClient: ow.optional.object,
         }));
-        options.config ??= Configuration.getGlobalConfig();
-        options.storageClient ??= options.config.getStorageClient();
-        await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
-        const manager = StorageManager.getManager(this, options.config);
-        const queue = await manager.openStorage(queueIdOrName, options.storageClient);
+        const client = options.storageClient ?? serviceLocator.getStorageClient();
+        const config = options.config ?? serviceLocator.getConfiguration();
+        await purgeDefaultStorages({ onlyPurgeOnce: true, client, config });
+        const resolved = await resolveStorageIdentifier(identifier, client, 'RequestQueue');
+        const queue = await serviceLocator
+            .getStorageInstanceManager()
+            .openStorage(this, {
+            ...resolved,
+            clientOpener: () => client.createRequestQueueClient(resolved),
+            clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
+        });
         queue.proxyConfiguration = options.proxyConfiguration;
-        // eslint-disable-next-line dot-notation
-        queue['initialCount'] = (await queue.client.get())?.totalRequestCount ?? 0;
+        queue.httpClient = options.httpClient;
+        if (!queue.isInitialized) {
+            // Re-create the request queue client with clientKey and timeoutSecs so that
+            // request locking works correctly for API-backed implementations.
+            // TODO: clientKey/timeoutSecs are Apify-platform concerns and should eventually be pushed
+            // down into the Apify SDK's client implementation, aligning with crawlee-python's approach
+            // where locking is handled internally by the client (see crawlee-python PR #1194).
+            queue.client = await client.createRequestQueueClient({
+                id: queue.id,
+                clientKey: queue.clientKey,
+                timeoutSecs: queue.timeoutSecs,
+            });
+            const queueInfo = await queue.client.getMetadata();
+            queue.initialCount = queueInfo.totalRequestCount;
+            queue.initialHandledCount = queueInfo.handledRequestCount;
+            queue.isInitialized = true;
+        }
         return queue;
     }
 }