@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/storages/request_queue.d.ts

@@ -4,7 +4,9 @@ import type { CrawleeLogger } from '../log.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 import type { Source } from '../request.js';
 import { Request } from '../request.js';
-import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { JournalEntry } from './transaction.js';
+import type { RequestLoaderStatus } from './request_loader.js';
+import type { IRequestManager, PacingSignal, RequestsLike } from './request_manager.js';
 import type { RequestQueueStats } from './storage_stats.js';
 import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
 import type { StorageOpenOptions } from './utils.js';
@@ -43,29 +45,11 @@ import type { StorageOpenOptions } from './utils.js';
  * @category Sources
  */
 export declare class RequestQueue implements IStorage, IRequestManager {
+    #private;
     readonly id: string;
     readonly name?: string;
     readonly backend: RequestQueueBackend;
-    private proxyConfiguration?;
     readonly log: CrawleeLogger;
-    private requestCache;
-    /**
-     * Remembers the `requestId` of every request already submitted to the client — including background
-     * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
-     * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
-     */
-    private requestSeenCache;
-    private queuePausedForMigration;
-    private inProgressRequestBatchCount;
-    /**
-     * The largest expected request-processing time (in seconds) seen so far via
-     * {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never
-     * lowered, before being forwarded to the storage backend.
-     */
-    private expectedRequestProcessingSecs;
-    private httpClient?;
-    private readonly events;
-    private readonly statsTracker;
     /**
      * Backend-independent usage counters tracked for this request queue (write operations and
      * queue-head reads issued to the underlying storage backend). Counted per backend call.
@@ -102,6 +86,8 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * @param [options] Request queue operation options.
      */
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
      * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`
@@ -146,7 +132,7 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * Note that the `null` return value doesn't mean the queue processing finished,
      * it means there are currently no pending requests.
      * To check whether all requests in queue were finished,
-     * use {@link RequestQueue.isFinished} instead.
+     * use {@link RequestQueue.checkReadiness} instead.
      *
      * @returns
      *   Returns the request object or `null` if there are no more pending requests.
@@ -167,25 +153,21 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      */
     reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
     /**
-     * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest} would return
-     * `null`, i.e. there are no pending requests to fetch right now. Otherwise it resolves to `false`.
-     *
-     * Note that even if the queue is empty, there might be some requests currently being processed
-     * (fetched but not yet handled or reclaimed). An empty queue therefore does not mean crawling is
-     * finished — those in-progress requests may still be reclaimed, and background tasks may still be
-     * adding more requests. To check whether all activity in the queue has finished, use
-     * {@link RequestQueue.isFinished}.
+     * A queue hands requests out as fast as they are asked for; pacing is a job for a manager wrapped around it,
+     * such as {@link ThrottlingRequestManager}.
+     * @inheritdoc
      */
-    isEmpty(): Promise<boolean>;
+    recordPacingSignal(_signal: PacingSignal): boolean;
     /**
-     * Resolves to `true` if all requests were already handled and there are no more left — including no
-     * requests currently in progress (fetched but not yet handled or reclaimed, including requests
-     * locked by other clients sharing the same queue) and no background add operations still in flight.
+     * Reports whether the queue has a request to hand over, is waiting on one, or is done.
+     *
+     * `waiting` means requests are in progress (fetched but not yet handled or reclaimed, possibly by another
+     * client sharing the queue) or a background add is still landing; neither has a clock, so no `readyAt`.
      *
-     * Due to the nature of distributed storage used by the queue, the function may occasionally return
-     * a false negative, but it shall never return a false positive.
+     * Due to the nature of distributed storage used by the queue, `finished` may occasionally arrive a probe or
+     * two late, but it is never reported early.
      */
-    isFinished(): Promise<boolean>;
+    checkReadiness(): Promise<RequestLoaderStatus>;
     /**
      * Tells the queue how long a consumer expects to hold a fetched request before marking it handled
      * or reclaiming it (typically the request-handler timeout plus padding), so that a storage backend
@@ -198,9 +180,11 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      */
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
     /**
-     * Caches information about request to beware of unneeded addRequest() calls.
+     * @inheritdoc
+     * Unlike {@link RequestQueue.setExpectedRequestProcessingTimeSecs}, which sizes every future lock,
+     * this only touches the one request it is given.
      */
-    private cacheRequest;
+    extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
     /**
      * Removes the queue either from the Apify Cloud storage or from the local database,
      * depending on the mode of operation.
@@ -246,18 +230,6 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     getInfo(): Promise<RequestQueueInfo>;
-    /**
-     * Fetches URLs from requestsFromUrl and returns them in format of list of requests
-     */
-    private fetchRequestsFromUrl;
-    /**
-     * Adds all fetched requests from a URL from a remote resource.
-     */
-    private addFetchedRequests;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private _downloadListOfUrls;
     /**
      * Opens a request queue and returns a promise resolving to an instance
      * of the {@link RequestQueue} class.
@@ -277,6 +249,7 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      */
     static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;
 }
+/** @internal */
 export interface RequestQueueOptions {
     /** Resolved metadata for the request queue, as returned by the backend's `getMetadata()`. */
     metadata: RequestQueueInfo;