@crawlee/core 4.0.0-beta.28 → 4.0.0-beta.280

package/storages/{request_provider.d.ts → request-queue.d.ts}

@@ -1,101 +1,76 @@
-import type { BaseHttpClient, BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueClient, RequestQueueInfo, StorageClient } from '@crawlee/types';
+import type { BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueBackend, RequestQueueInfo } from '@crawlee/types';
 import type { ReadonlyDeep } from 'type-fest';
-import { ListDictionary, LruCache } from '@apify/datastructures';
-import type { Log } from '@apify/log';
-import { Configuration } from '../configuration.js';
-import type { EventManager } from '../events/event_manager.js';
-import type { ProxyConfiguration } from '../proxy_configuration.js';
-import type { InternalSource, RequestOptions, Source } from '../request.js';
+import type { CrawleeLogger } from '../log.js';
+import type { IProxyConfiguration } from '../proxy-configuration.js';
+import type { Source } from '../request.js';
 import { Request } from '../request.js';
-import type { IStorage, StorageManagerOptions } from './storage_manager.js';
-export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
+import type { JournalEntry } from './transaction.js';
+import type { RequestLoaderStatus } from './request-loader.js';
+import type { IRequestManager, PacingSignal, RequestsLike } from './request-manager.js';
+import type { RequestQueueStats } from './storage-stats.js';
+import type { IStorage, StorageIdentifier } from './storage-instance-manager.js';
+import type { StorageOpenOptions } from './utils.js';
 /**
- * Represents a provider of requests/URLs to crawl.
+ * Represents a queue of URLs to crawl, which is used for deep crawling of websites
+ * where you start with several URLs and then recursively
+ * follow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.
+ *
+ * Each URL is represented using an instance of the {@link Request} class.
+ * The queue can only contain unique URLs. More precisely, it can only contain {@link Request} instances
+ * with distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.
+ * To add a single URL multiple times to the queue,
+ * corresponding {@link Request} objects will need to have different `uniqueKey` properties.
+ *
+ * Do not instantiate this class directly, use the {@link RequestQueue.open} function instead.
+ *
+ * `RequestQueue` is used by {@link BasicCrawler}, {@link CheerioCrawler}, {@link PuppeteerCrawler}
+ * and {@link PlaywrightCrawler} as a source of URLs to crawl.
+ * Unlike {@link RequestList}, `RequestQueue` supports dynamic adding and removing of requests.
+ * On the other hand, the queue is not optimized for operations that add or remove a large number of URLs in a batch.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * // Open the default request queue associated with the crawler run
+ * const queue = await RequestQueue.open();
+ *
+ * // Open a named request queue
+ * const queueWithName = await RequestQueue.open('some-name');
+ *
+ * // Enqueue few requests
+ * await queue.addRequest({ url: 'http://example.com/aaa' });
+ * await queue.addRequest({ url: 'http://example.com/bbb' });
+ * await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
+ * ```
+ * @category Sources
  */
-export interface IRequestManager {
+export declare class RequestQueue implements IStorage, IRequestManager {
+    #private;
+    readonly id: string;
+    readonly name?: string;
+    readonly backend: RequestQueueBackend;
+    readonly log: CrawleeLogger;
     /**
-     * Returns `true` if all requests were already handled and there are no more left.
+     * Backend-independent usage counters tracked for this request queue (write operations and
+     * queue-head reads issued to the underlying storage backend). Counted per backend call.
      */
-    isFinished(): Promise<boolean>;
+    get stats(): RequestQueueStats;
     /**
-     * Resolves to `true` if the next call to {@link IRequestManager.fetchNextRequest} function
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the provider is empty, there might be some pending requests currently being processed.
-     */
-    isEmpty(): Promise<boolean>;
-    /**
-     * Returns number of handled requests.
-     */
-    handledCount(): Promise<number>;
-    /**
-     * Get the total number of requests known to the request manager.
-     */
-    getTotalCount(): number;
-    /**
-     * Get an offline approximation of the number of pending requests.
-     */
-    getPendingCount(): number;
-    /**
-     * Gets the next {@link Request} to process.
-     *
-     * The function's `Promise` resolves to `null` if there are no more
-     * requests to process.
-     */
-    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
-    /**
-     * Can be used to iterate over the `RequestManager` instance in a `for await .. of` loop.
-     * Provides an alternative for the repeated use of `fetchNextRequest`.
-     */
-    [Symbol.asyncIterator](): AsyncGenerator<Request>;
-    /**
-     * Marks request as handled after successful processing.
-     */
-    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
-    /**
-     * Reclaims request to the provider if its processing failed.
-     * The request will become available in the next `fetchNextRequest()`.
+     * @internal
      */
-    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
-    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
-    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
-}
-export declare abstract class RequestProvider implements IStorage, IRequestManager {
-    protected readonly config: Configuration;
-    id: string;
-    name?: string;
-    timeoutSecs: number;
-    clientKey: string;
-    client: RequestQueueClient;
-    protected proxyConfiguration?: ProxyConfiguration;
-    log: Log;
-    internalTimeoutMillis: number;
-    requestLockSecs: number;
-    assumedTotalCount: number;
-    assumedHandledCount: number;
-    private initialCount;
-    private initialHandledCount;
-    protected queueHeadIds: ListDictionary<string>;
-    protected requestCache: LruCache<RequestLruItem>;
-    protected recentlyHandledRequestsCache: LruCache<boolean>;
-    protected queuePausedForMigration: boolean;
-    protected lastActivity: Date;
-    protected isFinishedCalledWhileHeadWasNotEmpty: number;
-    protected inProgressRequestBatchCount: number;
-    protected httpClient?: BaseHttpClient;
-    protected readonly events: EventManager;
-    constructor(options: InternalRequestProviderOptions, config: Configuration);
+    constructor(options: RequestQueueOptions);
     /**
-     * Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).
+     * Returns the total number of requests in the queue (i.e. pending + handled).
      *
      * Survives restarts and actor migrations.
      */
-    getTotalCount(): number;
+    getTotalCount(): Promise<number>;
     /**
-     * Returns an offline approximation of the total number of pending requests in the queue.
+     * Returns the total number of pending requests in the queue.
      *
      * Survives restarts and Actor migrations.
      */
-    getPendingCount(): number;
+    getPendingCount(): Promise<number>;
     /**
      * Adds a request to the queue.
      *
@@ -111,6 +86,8 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
      * @param [options] Request queue operation options.
      */
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
      * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`
@@ -137,17 +114,17 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
      */
     addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
     /**
-     * Gets the request from the queue specified by ID.
+     * Gets the request from the queue specified by its `uniqueKey`.
      *
-     * @param id ID of the request.
+     * @param uniqueKey Unique key of the request.
      * @returns Returns the request object, or `null` if it was not found.
      */
-    getRequest<T extends Dictionary = Dictionary>(id: string): Promise<Request<T> | null>;
+    getRequest<T extends Dictionary = Dictionary>(uniqueKey: string): Promise<Request<T> | null>;
     /**
      * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
      *
      * Once you successfully finish processing of the request, you need to call
-     * {@link RequestQueue.markRequestHandled}
+     * {@link RequestQueue.markRequestAsHandled}
      * to mark the request as handled in the queue. If there was some error in processing the request,
      * call {@link RequestQueue.reclaimRequest} instead,
      * so that the queue will give the request to some other consumer in another call to the `fetchNextRequest` function.
@@ -155,19 +132,19 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
      * Note that the `null` return value doesn't mean the queue processing finished,
      * it means there are currently no pending requests.
      * To check whether all requests in queue were finished,
-     * use {@link RequestQueue.isFinished} instead.
+     * use {@link RequestQueue.checkReadiness} instead.
      *
      * @returns
      *   Returns the request object or `null` if there are no more pending requests.
      */
-    abstract fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
     /**
      * Marks a request that was previously returned by the
      * {@link RequestQueue.fetchNextRequest}
      * function as handled after successful processing.
      * Handled requests will never again be returned by the `fetchNextRequest` function.
      */
-    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | null>;
     /**
      * Reclaims a failed request back to the queue, so that it can be returned for processing later again
      * by another call to {@link RequestQueue.fetchNextRequest}.
@@ -175,35 +152,49 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
      * For example, this lets you store the number of retries or error messages for the request.
      */
     reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
-    protected abstract ensureHeadIsNonEmpty(): Promise<void>;
     /**
-     * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest}
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the queue is empty, there might be some pending requests currently being processed.
-     * If you need to ensure that there is no activity in the queue, use {@link RequestQueue.isFinished}.
+     * A queue hands requests out as fast as they are asked for; pacing is a job for a manager wrapped around it,
+     * such as {@link ThrottlingRequestManager}.
+     * @inheritdoc
      */
-    isEmpty(): Promise<boolean>;
+    recordPacingSignal(_signal: PacingSignal): boolean;
     /**
-     * Resolves to `true` if all requests were already handled and there are no more left.
-     * Due to the nature of distributed storage used by the queue,
-     * the function may occasionally return a false negative,
-     * but it shall never return a false positive.
+     * Reports whether the queue has a request to hand over, is waiting on one, or is done.
+     *
+     * `waiting` means requests are in progress (fetched but not yet handled or reclaimed, possibly by another
+     * client sharing the queue) or a background add is still landing; neither has a clock, so no `readyAt`.
+     *
+     * Due to the nature of distributed storage used by the queue, `finished` may occasionally arrive a probe or
+     * two late, but it is never reported early.
      */
-    abstract isFinished(): Promise<boolean>;
-    protected _reset(): void;
+    checkReadiness(): Promise<RequestLoaderStatus>;
     /**
-     * Caches information about request to beware of unneeded addRequest() calls.
+     * Tells the queue how long a consumer expects to hold a fetched request before marking it handled
+     * or reclaiming it (typically the request-handler timeout plus padding), so that a storage backend
+     * that reserves requests via locking does not hand the same request out again while it is still
+     * being processed.
+     *
+     * Several consumers may share one queue (and therefore one client) in a single process, so we only
+     * ever raise the reservation duration, never lower it — otherwise a short-lived consumer could cut
+     * short the reservation of a long-lived one and have its in-flight request stolen.
      */
-    protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
+    setExpectedRequestProcessingTime(processingTimeSecs: number): Promise<void>;
     /**
-     * Adds a request straight to the queueHeadDict, to improve performance.
+     * @inheritdoc
+     * Unlike {@link RequestQueue.setExpectedRequestProcessingTime}, which sizes every future lock,
+     * this only touches the one request it is given.
      */
-    protected _maybeAddRequestToQueueHead(requestId: string, forefront: boolean): void;
+    extendRequestProcessingTime(request: Request, extensionSecs: number): Promise<boolean>;
     /**
      * Removes the queue either from the Apify Cloud storage or from the local database,
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * Remove all requests from the queue but keep the queue itself, resetting it
+     * so it can be reused (e.g. across multiple `crawler.run()` calls).
+     */
+    purge(): Promise<void>;
     /**
      * @inheritdoc
      */
@@ -218,22 +209,15 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
      * ```
      * @inheritdoc
      */
-    handledCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
     /**
      * Returns an object containing general information about the request queue.
      *
-     * The function returns the same object as the Apify API Client's
-     * [getQueue](https://docs.apify.com/api/apify-client-js/latest#ApifyClient-requestQueues)
-     * function, which in turn calls the
-     * [Get request queue](https://apify.com/docs/api/v2#/reference/request-queues/queue/get-request-queue)
-     * API endpoint.
-     *
      * **Example:**
      * ```
      * {
      *   id: "WkzbQMuFYuamGv3YF",
      *   name: "my-queue",
-     *   userId: "wRsJZtadYvn4mBZmm",
      *   createdAt: new Date("2015-12-12T07:34:14.202Z"),
      *   modifiedAt: new Date("2015-12-13T08:36:13.202Z"),
      *   accessedAt: new Date("2015-12-14T08:36:13.202Z"),
@@ -242,20 +226,10 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
      *   pendingRequestCount: 20,
      * }
      * ```
+     *
+     * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
-    getInfo(): Promise<RequestQueueInfo | undefined>;
-    /**
-     * Fetches URLs from requestsFromUrl and returns them in format of list of requests
-     */
-    protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;
-    /**
-     * Adds all fetched requests from a URL from a remote resource.
-     */
-    protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[], options: RequestQueueOperationOptions): Promise<ProcessedRequest[]>;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private _downloadListOfUrls;
+    getInfo(): Promise<RequestQueueInfo>;
     /**
      * Opens a request queue and returns a promise resolving to an instance
      * of the {@link RequestQueue} class.
@@ -267,44 +241,25 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
      *
      * For more details and code examples, see the {@link RequestQueue} class.
      *
-     * @param [queueIdOrName]
-     *   ID or name of the request queue to be opened. If `null` or `undefined`,
-     *   the function returns the default request queue associated with the crawler run.
+     * @param [identifier]
+     *   ID or name of the request queue to be opened. If a string is provided, it will first be
+     *   looked up as an ID; if no such storage exists, it will be treated as a name.
+     *   If `null` or `undefined`, the function returns the default request queue associated with the crawler run.
      * @param [options] Open Request Queue options.
      */
-    static open(queueIdOrName?: string | null, options?: StorageManagerOptions): Promise<RequestProvider>;
-}
-interface RequestLruItem {
-    uniqueKey: string;
-    isHandled: boolean;
-    id: string;
-    hydrated: Request | null;
-    lockExpiresAt: number | null;
-    forefront: boolean;
+    static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;
 }
-export interface RequestProviderOptions {
-    id: string;
-    name?: string;
-    client: StorageClient;
+/** @internal */
+export interface RequestQueueOptions {
+    /** Resolved metadata for the request queue, as returned by the backend's `getMetadata()`. */
+    metadata: RequestQueueInfo;
+    backend: RequestQueueBackend;
     /**
      * Used to pass the proxy configuration for the `requestsFromUrl` objects.
      * Takes advantage of the internal address rotation and authentication process.
      * If undefined, the `requestsFromUrl` requests will be made without proxy.
      */
-    proxyConfiguration?: ProxyConfiguration;
-}
-/**
- * @deprecated Use {@link RequestProviderOptions} instead.
- */
-export interface RequestQueueOptions extends RequestProviderOptions {
-}
-/**
- * @internal
- */
-export interface InternalRequestProviderOptions extends RequestProviderOptions {
-    logPrefix: string;
-    requestCacheMaxSize: number;
-    recentlyHandledRequestsMaxSize: number;
+    proxyConfiguration?: IProxyConfiguration;
 }
 export interface RequestQueueOperationOptions {
     /**
@@ -328,9 +283,6 @@ export interface RequestQueueOperationOptions {
      */
     cache?: boolean;
 }
-/**
- * @internal
- */
 export interface RequestQueueOperationInfo extends QueueOperationInfo {
     uniqueKey: string;
     forefront: boolean;
@@ -349,6 +301,15 @@ export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions
      * @default 1000
      */
     waitBetweenBatchesMillis?: number;
+    /**
+     * If set, only this many *actually new* requests (i.e. not already present in the queue) will be added.
+     * Once the budget is reached, remaining requests from the iterable are left unconsumed, or collected in
+     * {@link AddRequestsBatchedResult.requestsOverLimit|`requestsOverLimit`} if
+     * {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} is set.
+     *
+     * This is useful in combination with `maxRequestsPerCrawl` to avoid duplicate URLs consuming the budget.
+     */
+    maxNewRequests?: number;
 }
 export interface AddRequestsBatchedResult {
     addedRequests: ProcessedRequest[];
@@ -369,6 +330,11 @@ export interface AddRequestsBatchedResult {
      * ```
      */
     waitForAllRequestsToBeAdded: Promise<ProcessedRequest[]>;
+    /**
+     * Requests from the input that were not added to the queue because the
+     * {@link AddRequestsBatchedOptions.maxNewRequests|`maxNewRequests`} budget was reached.
+     * `undefined` unless both `maxNewRequests` and
+     * {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} are set.
+     */
+    requestsOverLimit?: Source[];
 }
-export {};
-//# sourceMappingURL=request_provider.d.ts.map