@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/storages/request_queue.d.ts

@@ -4,6 +4,7 @@ import type { CrawleeLogger } from '../log.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 import type { Source } from '../request.js';
 import { Request } from '../request.js';
+import type { JournalEntry } from './transaction.js';
 import type { IRequestManager, RequestsLike } from './request_manager.js';
 import type { RequestQueueStats } from './storage_stats.js';
 import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
@@ -43,29 +44,13 @@ import type { StorageOpenOptions } from './utils.js';
  * @category Sources
  */
 export declare class RequestQueue implements IStorage, IRequestManager {
+    #private;
     readonly id: string;
     readonly name?: string;
     readonly backend: RequestQueueBackend;
-    private proxyConfiguration?;
     readonly log: CrawleeLogger;
     private requestCache;
-    /**
-     * Remembers the `requestId` of every request already submitted to the client — including background
-     * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
-     * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
-     */
-    private requestSeenCache;
-    private queuePausedForMigration;
     private inProgressRequestBatchCount;
-    /**
-     * The largest expected request-processing time (in seconds) seen so far via
-     * {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never
-     * lowered, before being forwarded to the storage backend.
-     */
-    private expectedRequestProcessingSecs;
-    private httpClient?;
-    private readonly events;
-    private readonly statsTracker;
     /**
      * Backend-independent usage counters tracked for this request queue (write operations and
      * queue-head reads issued to the underlying storage backend). Counted per backend call.
@@ -102,6 +87,24 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * @param [options] Request queue operation options.
      */
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    /**
+     * Journals an addition for introspection only; these entries are never replayed. A no-op unless the
+     * transaction is open, so detached and outliving writers stay out of the journal.
+     */
+    private recordRequestJournalEntry;
+    /**
+     * The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup
+     * index derived from the transaction journal.
+     */
+    private bufferedRequests;
+    /**
+     * Adds a request under the `deferred` policy: journaled now, really added by the commit replay.
+     * A new request's `requestId` is the local `uniqueKey` hash and is **provisional** — never write it
+     * to `request.id` or the dedup caches. Dedup is cheapest-first: buffer, caches, then a backend probe.
+     */
+    private addRequestDeferred;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Adds requests to the queue in batches of 25. This method will wait till all the requests are added
      * to the queue before resolving. You should prefer using `queue.addRequestsBatched()` or `crawler.addRequests()`
@@ -257,7 +260,7 @@ export declare class RequestQueue implements IStorage, IRequestManager {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    private _downloadListOfUrls;
+    private downloadListOfUrls;
     /**
      * Opens a request queue and returns a promise resolving to an instance
      * of the {@link RequestQueue} class.