@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/storages/request_list.js

@@ -1,8 +1,10 @@
 import { downloadListOfUrls } from '@crawlee/utils';
-import ow, { ArgumentError } from 'ow';
+import { z } from 'zod';
+import { EventType } from '../events/event_manager.js';
 import { Request } from '../request.js';
 import { createDeserialize, serializeArray } from '../serialization.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { KeyValueStore } from './key_value_store.js';
 import { purgeDefaultStorages } from './utils.js';
 /** @internal */
@@ -10,6 +12,24 @@ export const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';
 /** @internal */
 export const REQUESTS_PERSISTENCE_KEY = 'REQUEST_LIST_REQUESTS';
 const CONTENT_TYPE_BINARY = 'application/octet-stream';
+const requestListOptionsSchema = z.strictObject({
+    sources: schemas.anyArray.optional(), // check only for array and not subtypes to avoid iteration over the whole thing
+    sourcesFunction: schemas.anyFunction.optional(),
+    persistStateKey: z.string().optional(),
+    persistRequestsKey: z.string().optional(),
+    state: z
+        .strictObject({
+        nextIndex: schemas.anyNumber,
+        nextUniqueKey: z.string(),
+        inProgress: schemas.anyObject, // persisted as an array of unique keys
+    })
+        .optional(),
+    keepDuplicateUrls: z.boolean().default(false),
+    proxyConfiguration: validators.proxyConfiguration.optional(),
+    httpClient: schemas.httpClient.optional(),
+});
+const listNameSchema = z.string().nullish();
+const openOptionsSchema = z.looseObject({});
 /**
  * Represents a static list of URLs to crawl.
  * The URLs can be provided either in code or parsed from a text file hosted on the web.
@@ -72,7 +92,7 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
  * @category Sources
  */
 export class RequestList {
-    log = serviceLocator.getLogger().child({ prefix: 'RequestList' });
+    #log = serviceLocator.getLogger().child({ prefix: 'RequestList' });
     /**
      * Array of all requests from all sources, in the order as they appeared in sources.
      * All requests in the array have distinct uniqueKey!
@@ -80,9 +100,9 @@ export class RequestList {
      */
     requests = [];
     /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
-    nextIndex = 0;
+    #nextIndex = 0;
     /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
-    uniqueKeyToIndex = {};
+    #uniqueKeyToIndex = {};
     /**
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
@@ -93,7 +113,7 @@ export class RequestList {
      * re-crawled after a restart. They are served before advancing through the rest of the sources.
      * @internal
      */
-    requestsToRetry = [];
+    #requestsToRetry = [];
     /**
      * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
      * @internal
@@ -104,52 +124,39 @@ export class RequestList {
      * @internal
      */
     areRequestsPersisted = false;
-    isLoading = false;
-    isInitialized = false;
-    persistStateKey;
-    persistRequestsKey;
-    initialState;
-    store;
-    keepDuplicateUrls;
+    #isLoading = false;
+    #isInitialized = false;
+    #persistStateKey;
+    #persistRequestsKey;
+    #initialState;
+    #store;
+    #keepDuplicateUrls;
+    // kept as TS-private: request_list tests read this field directly
     sources;
-    sourcesFunction;
-    proxyConfiguration;
-    httpClient;
+    #sourcesFunction;
+    #proxyConfiguration;
+    #httpClient;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
      * @internal
      */
     constructor(options = {}) {
-        const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, proxyConfiguration, keepDuplicateUrls = false, httpClient, } = options;
+        const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, proxyConfiguration, keepDuplicateUrls, httpClient, } = parseArgument(options, requestListOptionsSchema);
         if (!(sources || sourcesFunction)) {
-            throw new ArgumentError('At least one of "sources" or "sourcesFunction" must be provided.', this.constructor);
-        }
-        ow(options, ow.object.exactShape({
-            sources: ow.optional.array, // check only for array and not subtypes to avoid iteration over the whole thing
-            sourcesFunction: ow.optional.function,
-            persistStateKey: ow.optional.string,
-            persistRequestsKey: ow.optional.string,
-            state: ow.optional.object.exactShape({
-                nextIndex: ow.number,
-                nextUniqueKey: ow.string,
-                inProgress: ow.object,
-            }),
-            keepDuplicateUrls: ow.optional.boolean,
-            proxyConfiguration: ow.optional.object,
-            httpClient: ow.optional.object,
-        }));
-        this.persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;
-        this.persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;
-        this.initialState = state;
-        this.httpClient = httpClient;
+            throw new Error('At least one of "sources" or "sourcesFunction" must be provided.');
+        }
+        this.#persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;
+        this.#persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;
+        this.#initialState = state;
+        this.#httpClient = httpClient;
         // If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.
-        this.keepDuplicateUrls = keepDuplicateUrls;
+        this.#keepDuplicateUrls = keepDuplicateUrls;
         // Will be empty after initialization to save memory.
         this.sources = sources ? [...sources] : [];
-        this.sourcesFunction = sourcesFunction;
+        this.#sourcesFunction = sourcesFunction;
         // The proxy configuration used for `requestsFromUrl` requests.
-        this.proxyConfiguration = proxyConfiguration;
+        this.#proxyConfiguration = proxyConfiguration;
         this.persistState = this.persistState.bind(this);
     }
     /**
@@ -157,10 +164,10 @@ export class RequestList {
      * This function must be called before you can start using the instance in a meaningful way.
      */
     async initialize() {
-        if (this.isLoading) {
+        if (this.#isLoading) {
             throw new Error('RequestList sources are already loading or were loaded.');
         }
-        this.isLoading = true;
+        this.#isLoading = true;
         await purgeDefaultStorages({ onlyPurgeOnce: true });
         const [state, persistedRequests] = await this.loadStateAndPersistedRequests();
         // Add persisted requests / new sources in a memory efficient way because with very
@@ -172,11 +179,11 @@ export class RequestList {
             await this.addRequestsFromSources();
         }
         this.restoreState(state);
-        this.isInitialized = true;
-        if (this.persistRequestsKey && !this.areRequestsPersisted)
+        this.#isInitialized = true;
+        if (this.#persistRequestsKey && !this.areRequestsPersisted)
             await this.persistRequests();
-        if (this.persistStateKey) {
-            serviceLocator.getEventManager().on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        if (this.#persistStateKey) {
+            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.persistState);
         }
         return this;
     }
@@ -224,9 +231,9 @@ export class RequestList {
         }
         // Drop the original array full of empty indexes.
         this.sources = [];
-        if (this.sourcesFunction) {
+        if (this.#sourcesFunction) {
             try {
-                const sourcesFromFunction = await this.sourcesFunction();
+                const sourcesFromFunction = await this.#sourcesFunction();
                 const sourcesFromFunctionCount = sourcesFromFunction.length;
                 for (let i = 0; i < sourcesFromFunctionCount; i++) {
                     const source = sourcesFromFunction[i];
@@ -246,19 +253,19 @@ export class RequestList {
      * @inheritDoc
      */
     async persistState() {
-        if (!this.persistStateKey) {
+        if (!this.#persistStateKey) {
             throw new Error('Cannot persist state. options.persistStateKey is not set.');
         }
         if (this.isStatePersisted)
             return;
         try {
-            this.store ??= await KeyValueStore.open();
-            await this.store.setValue(this.persistStateKey, this.getState());
+            this.#store ??= await KeyValueStore.open();
+            await this.#store.setValue(this.#persistStateKey, this.getState());
             this.isStatePersisted = true;
         }
         catch (e) {
             const err = e;
-            this.log.exception(err, 'Attempted to persist state, but failed.');
+            this.#log.exception(err, 'Attempted to persist state, but failed.');
         }
     }
     /**
@@ -267,8 +274,8 @@ export class RequestList {
      * leaking the listener (and the requests it retains) on the shared event manager.
      */
     async teardown() {
-        serviceLocator.getEventManager().off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
-        if (this.persistStateKey) {
+        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.persistState);
+        if (this.#persistStateKey) {
             await this.persistState();
         }
     }
@@ -279,8 +286,8 @@ export class RequestList {
      */
     async persistRequests() {
         const serializedRequests = await serializeArray(this.requests);
-        this.store ??= await KeyValueStore.open();
-        await this.store.setValue(this.persistRequestsKey, serializedRequests, { contentType: CONTENT_TYPE_BINARY });
+        this.#store ??= await KeyValueStore.open();
+        await this.#store.setValue(this.#persistRequestsKey, serializedRequests, { contentType: CONTENT_TYPE_BINARY });
         this.areRequestsPersisted = true;
     }
     /**
@@ -303,7 +310,7 @@ export class RequestList {
         }
         const deleteFromInProgress = [];
         state.inProgress.forEach((uniqueKey) => {
-            const index = this.uniqueKeyToIndex[uniqueKey];
+            const index = this.#uniqueKeyToIndex[uniqueKey];
             if (typeof index !== 'number') {
                 throw new Error('The state object is not consistent with RequestList. Unknown uniqueKey is present in the state.');
             }
@@ -311,7 +318,7 @@ export class RequestList {
                 deleteFromInProgress.push(uniqueKey);
             }
         });
-        this.nextIndex = state.nextIndex;
+        this.#nextIndex = state.nextIndex;
         this.inProgress = new Set(state.inProgress);
         // WORKAROUND:
         // It happened to some users that state object contained something like:
@@ -327,7 +334,7 @@ export class RequestList {
         // As a workaround, we just remove all inProgress requests whose index >= nextIndex,
         // since they will be crawled again.
         if (deleteFromInProgress.length) {
-            this.log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries", {
+            this.#log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries", {
                 deleteFromInProgress,
             });
             for (const uniqueKey of deleteFromInProgress) {
@@ -335,7 +342,7 @@ export class RequestList {
             }
         }
         // All in-progress requests were interrupted and need to be re-crawled.
-        this.requestsToRetry = [...this.inProgress];
+        this.#requestsToRetry = [...this.inProgress];
     }
     /**
      * Attempts to load state and requests using the `RequestList` configuration
@@ -344,19 +351,19 @@ export class RequestList {
     async loadStateAndPersistedRequests() {
         let state;
         let persistedRequests;
-        if (this.initialState) {
-            state = this.initialState;
-            this.log.debug('Loaded state from options.state argument.');
+        if (this.#initialState) {
+            state = this.#initialState;
+            this.#log.debug('Loaded state from options.state argument.');
         }
-        else if (this.persistStateKey) {
-            state = await this.getPersistedState(this.persistStateKey);
+        else if (this.#persistStateKey) {
+            state = await this.getPersistedState(this.#persistStateKey);
             if (state)
-                this.log.debug('Loaded state from key value store using the persistStateKey.');
+                this.#log.debug('Loaded state from key value store using the persistStateKey.');
         }
-        if (this.persistRequestsKey) {
-            persistedRequests = await this.getPersistedState(this.persistRequestsKey);
+        if (this.#persistRequestsKey) {
+            persistedRequests = await this.getPersistedState(this.#persistRequestsKey);
             if (persistedRequests)
-                this.log.debug('Loaded requests from key value store using the persistRequestsKey.');
+                this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');
         }
         return [state, persistedRequests];
     }
@@ -367,8 +374,8 @@ export class RequestList {
     getState() {
         this.ensureIsInitialized();
         return {
-            nextIndex: this.nextIndex,
-            nextUniqueKey: this.nextIndex < this.requests.length ? this.requests[this.nextIndex].uniqueKey : null,
+            nextIndex: this.#nextIndex,
+            nextUniqueKey: this.#nextIndex < this.requests.length ? this.requests[this.#nextIndex].uniqueKey : null,
             inProgress: [...this.inProgress],
         };
     }
@@ -377,14 +384,14 @@ export class RequestList {
      */
     async isEmpty() {
         this.ensureIsInitialized();
-        return this.requestsToRetry.length === 0 && this.nextIndex >= this.requests.length;
+        return this.#requestsToRetry.length === 0 && this.#nextIndex >= this.requests.length;
     }
     /**
      * @inheritDoc
      */
     async isFinished() {
         this.ensureIsInitialized();
-        return this.inProgress.size === 0 && this.nextIndex >= this.requests.length;
+        return this.inProgress.size === 0 && this.#nextIndex >= this.requests.length;
     }
     /**
      * @inheritDoc
@@ -392,17 +399,17 @@ export class RequestList {
     async fetchNextRequest() {
         this.ensureIsInitialized();
         // First re-serve any requests that were interrupted before the last state persist.
-        const uniqueKey = this.requestsToRetry.shift();
+        const uniqueKey = this.#requestsToRetry.shift();
         if (uniqueKey) {
-            const index = this.uniqueKeyToIndex[uniqueKey];
+            const index = this.#uniqueKeyToIndex[uniqueKey];
             return this.ensureRequest(this.requests[index], index);
         }
         // Otherwise return next request.
-        if (this.nextIndex < this.requests.length) {
-            const index = this.nextIndex;
+        if (this.#nextIndex < this.requests.length) {
+            const index = this.#nextIndex;
             const request = this.requests[index];
             this.inProgress.add(request.uniqueKey);
-            this.nextIndex++;
+            this.#nextIndex++;
             this.isStatePersisted = false;
             return this.ensureRequest(request, index);
         }
@@ -446,7 +453,7 @@ export class RequestList {
         fetchedRequests.forEach((request) => this.addRequest(request));
         const fetchedCount = fetchedRequests.length;
         const importedCount = this.requests.length - originalLength;
-        this.log.info('Fetched and loaded Requests from a remote resource.', {
+        this.#log.info('Fetched and loaded Requests from a remote resource.', {
             requestsFromUrl,
             regex,
             fetchedCount,
@@ -456,8 +463,8 @@ export class RequestList {
         });
     }
     async getPersistedState(key) {
-        this.store ??= await KeyValueStore.open();
-        const state = await this.store.getValue(key);
+        this.#store ??= await KeyValueStore.open();
+        const state = await this.#store.getValue(key);
         return state;
     }
     /**
@@ -468,10 +475,10 @@ export class RequestList {
         // Download remote resource and parse URLs.
         let urlsArr;
         try {
-            urlsArr = await this._downloadListOfUrls({
+            urlsArr = await this.downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
-                proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,
+                proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,
             });
         }
         catch (err) {
@@ -479,7 +486,7 @@ export class RequestList {
         }
         // Skip if resource contained no URLs.
         if (!urlsArr.length) {
-            this.log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex });
+            this.#log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex });
             return [];
         }
         return urlsArr.map((url) => ({ url, ...sharedOpts }));
@@ -507,18 +514,18 @@ export class RequestList {
         const hasUniqueKey = Reflect.has(Object(source), 'uniqueKey');
         request.uniqueKey ??= Request.computeUniqueKey(request);
         // Add index to uniqueKey if duplicates are to be kept
-        if (this.keepDuplicateUrls && !hasUniqueKey) {
+        if (this.#keepDuplicateUrls && !hasUniqueKey) {
             request.uniqueKey += `-${this.requests.length}`;
         }
         const { uniqueKey } = request;
         this.ensureUniqueKeyValid(uniqueKey);
         // Skip requests with duplicate uniqueKey
-        if (!Object.hasOwn(this.uniqueKeyToIndex, uniqueKey)) {
-            this.uniqueKeyToIndex[uniqueKey] = this.requests.length;
+        if (!Object.hasOwn(this.#uniqueKeyToIndex, uniqueKey)) {
+            this.#uniqueKeyToIndex[uniqueKey] = this.requests.length;
             this.requests.push(request);
         }
-        else if (this.keepDuplicateUrls) {
-            this.log.warning(`Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`);
+        else if (this.#keepDuplicateUrls) {
+            this.#log.warning(`Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`);
         }
     }
     /**
@@ -542,7 +549,7 @@ export class RequestList {
      * Throws an error if request list wasn't initialized.
      */
     ensureIsInitialized() {
-        if (!this.isInitialized) {
+        if (!this.#isInitialized) {
             throw new Error('RequestList is not initialized; you must call "await requestList.initialize()" before using it!');
         }
     }
@@ -558,7 +565,7 @@ export class RequestList {
      */
     async getPendingCount() {
         this.ensureIsInitialized();
-        return this.requests.length - (this.nextIndex - this.inProgress.size);
+        return this.requests.length - (this.#nextIndex - this.inProgress.size);
     }
     /**
      * Combines this list with a request manager (a {@link RequestQueue} by default) into a
@@ -576,7 +583,7 @@ export class RequestList {
      */
     async getHandledCount() {
         this.ensureIsInitialized();
-        return this.nextIndex - this.inProgress.size;
+        return this.#nextIndex - this.inProgress.size;
     }
     /**
      * Opens a request list and returns a promise resolving to an instance
@@ -639,9 +646,9 @@ export class RequestList {
             return rl;
         }
         const listName = listNameOrOptions;
-        ow(listName, ow.optional.any(ow.string, ow.null));
-        ow(sources, ow.array);
-        ow(options, ow.object.is((v) => !Array.isArray(v)));
+        parseArgument(listName, listNameSchema);
+        parseArgument(sources, schemas.anyArray);
+        parseArgument(options, openOptionsSchema);
         const rl = new RequestList({
             ...options,
             persistStateKey: listName ? `${listName}-${STATE_PERSISTENCE_KEY}` : options.persistStateKey,
@@ -654,10 +661,10 @@ export class RequestList {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    async _downloadListOfUrls(options) {
+    async downloadListOfUrls(options) {
         return downloadListOfUrls({
             ...options,
-            httpClient: this.httpClient,
+            httpClient: this.#httpClient,
         });
     }
 }

package/storages/request_loader.d.ts

@@ -54,6 +54,11 @@ export interface IRequestLoader {
      * Resolves to `true` if the next call to {@link IRequestLoader.fetchNextRequest} function
      * would return `null`, otherwise it resolves to `false`.
      * Note that even if the loader is empty, there might be some pending requests currently being processed.
+     *
+     * This is a statement about what the *next fetch* would return, not about how much work is left, so it
+     * may report `true` while {@link IRequestLoader.getPendingCount} is non-zero - a loader that withholds
+     * requests for a while (as {@link ThrottlingRequestManager} does for a rate-limited domain) is empty
+     * for as long as it will not hand anything over. Use `isFinished()` to ask whether the work is done.
      */
     isEmpty(): Promise<boolean>;
     /**

package/storages/request_manager_tandem.d.ts

@@ -9,16 +9,7 @@ import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueO
  * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
  */
 export declare class RequestManagerTandem implements IRequestManager {
-    private log;
-    private requestLoader;
-    private requestManagerPromise?;
-    private resolvedRequestManager?;
-    private requestManagerFactory;
-    /**
-     * The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.
-     * Remembered so it can be applied to the writable manager once it is lazily resolved.
-     */
-    private expectedRequestProcessingSecs?;
+    #private;
     /**
      * @param requestLoader The read-only loader to read requests from first.
      * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a

package/storages/request_manager_tandem.js

@@ -5,16 +5,16 @@ import { serviceLocator } from '../service_locator.js';
  * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
  */
 export class RequestManagerTandem {
-    log;
-    requestLoader;
-    requestManagerPromise;
-    resolvedRequestManager;
-    requestManagerFactory;
+    #log;
+    #requestLoader;
+    #requestManagerPromise;
+    #resolvedRequestManager;
+    #requestManagerFactory;
     /**
      * The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.
      * Remembered so it can be applied to the writable manager once it is lazily resolved.
      */
-    expectedRequestProcessingSecs;
+    #expectedRequestProcessingSecs;
     /**
      * @param requestLoader The read-only loader to read requests from first.
      * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
@@ -22,24 +22,24 @@ export class RequestManagerTandem {
      *  (e.g. a lazily-opened default {@link RequestQueue}).
      */
     constructor(requestLoader, requestManager) {
-        this.log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
-        this.requestLoader = requestLoader;
-        this.requestManagerFactory = typeof requestManager === 'function' ? requestManager : () => requestManager;
+        this.#log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
+        this.#requestLoader = requestLoader;
+        this.#requestManagerFactory = typeof requestManager === 'function' ? requestManager : () => requestManager;
     }
     /**
      * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
      * @private
      */
     async getRequestManager() {
-        if (this.resolvedRequestManager === undefined) {
-            this.requestManagerPromise ??= Promise.resolve(this.requestManagerFactory());
-            this.resolvedRequestManager = await this.requestManagerPromise;
+        if (this.#resolvedRequestManager === undefined) {
+            this.#requestManagerPromise ??= Promise.resolve(this.#requestManagerFactory());
+            this.#resolvedRequestManager = await this.#requestManagerPromise;
             // Apply any hint received before the manager was resolved.
-            if (this.expectedRequestProcessingSecs !== undefined) {
-                await this.resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(this.expectedRequestProcessingSecs);
+            if (this.#expectedRequestProcessingSecs !== undefined) {
+                await this.#resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(this.#expectedRequestProcessingSecs);
             }
         }
-        return this.resolvedRequestManager;
+        return this.#resolvedRequestManager;
     }
     /**
      * Transfers a single request from the read-only loader to the writable manager.
@@ -50,7 +50,7 @@ export class RequestManagerTandem {
      * @private
      */
     async transferNextRequestToQueue() {
-        const request = await this.requestLoader.fetchNextRequest();
+        const request = await this.#requestLoader.fetchNextRequest();
         if (request === null) {
             return true;
         }
@@ -60,12 +60,12 @@ export class RequestManagerTandem {
             return true;
         }
         catch (error) {
-            this.log.exception(error, 'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.', { url: request.url, uniqueKey: request.uniqueKey });
+            this.#log.exception(error, 'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.', { url: request.url, uniqueKey: request.uniqueKey });
             return false;
         }
         finally {
             // Mark it as handled so that the request doesn't get stuck in the `inProgress` state in the loader.
-            await this.requestLoader.markRequestAsHandled(request);
+            await this.#requestLoader.markRequestAsHandled(request);
         }
     }
     /**
@@ -76,8 +76,8 @@ export class RequestManagerTandem {
     async fetchNextRequest() {
         // First, try to transfer a request from the requestList
         const [listEmpty, listFinished] = await Promise.all([
-            this.requestLoader.isEmpty(),
-            this.requestLoader.isFinished(),
+            this.#requestLoader.isEmpty(),
+            this.#requestLoader.isFinished(),
         ]);
         if (!listEmpty && !listFinished) {
             // If the transfer failed, the request was dropped; don't fetch from the manager this round (matching
@@ -94,7 +94,7 @@ export class RequestManagerTandem {
      */
     async isFinished() {
         const requestManager = await this.getRequestManager();
-        const storagesFinished = await Promise.all([this.requestLoader.isFinished(), requestManager.isFinished()]);
+        const storagesFinished = await Promise.all([this.#requestLoader.isFinished(), requestManager.isFinished()]);
         return storagesFinished.every(Boolean);
     }
     /**
@@ -102,7 +102,7 @@ export class RequestManagerTandem {
      */
     async isEmpty() {
         const requestManager = await this.getRequestManager();
-        const storagesEmpty = await Promise.all([this.requestLoader.isEmpty(), requestManager.isEmpty()]);
+        const storagesEmpty = await Promise.all([this.#requestLoader.isEmpty(), requestManager.isEmpty()]);
         return storagesEmpty.every(Boolean);
     }
     /**
@@ -120,7 +120,7 @@ export class RequestManagerTandem {
         const [managerTotal, loaderTotal] = await Promise.all([
             requestManager.getTotalCount(),
             // count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager
-            this.requestLoader.getPendingCount(),
+            this.#requestLoader.getPendingCount(),
         ]);
         return managerTotal + loaderTotal;
     }
@@ -131,7 +131,7 @@ export class RequestManagerTandem {
         const requestManager = await this.getRequestManager();
         const [managerPending, loaderPending] = await Promise.all([
             requestManager.getPendingCount(),
-            this.requestLoader.getPendingCount(),
+            this.#requestLoader.getPendingCount(),
         ]);
         return managerPending + loaderPending;
     }
@@ -175,7 +175,7 @@ export class RequestManagerTandem {
      * @inheritdoc
      */
     async persistState() {
-        await this.requestLoader.persistState?.();
+        await this.#requestLoader.persistState?.();
     }
     /**
      * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
@@ -191,7 +191,7 @@ export class RequestManagerTandem {
      * @inheritdoc
      */
     async setExpectedRequestProcessingTimeSecs(secs) {
-        this.expectedRequestProcessingSecs = secs;
-        await this.resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
+        this.#expectedRequestProcessingSecs = secs;
+        await this.#resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
     }
 }