@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/storages/request_list.js

@@ -1,6 +1,6 @@
 import { downloadListOfUrls } from '@crawlee/utils';
 import { z } from 'zod';
-import { EventType } from '../events/event_manager.js';
+import { convertStateSync, RecoverableState } from '../recoverable_state.js';
 import { Request } from '../request.js';
 import { createDeserialize, serializeArray } from '../serialization.js';
 import { serviceLocator } from '../service_locator.js';
@@ -12,24 +12,87 @@ export const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';
 /** @internal */
 export const REQUESTS_PERSISTENCE_KEY = 'REQUEST_LIST_REQUESTS';
 const CONTENT_TYPE_BINARY = 'application/octet-stream';
+const persistedRequestListState = z.object({
+    nextIndex: z.number().int().nonnegative(),
+    nextUniqueKey: z.string().nullable(),
+    inProgress: z.array(z.string()),
+});
+const requestListInternalState = z.object({
+    /** Position of the next request to be processed. */
+    nextIndex: z.number().int().nonnegative(),
+    /** `uniqueKey`s of the requests being processed at the moment. */
+    inProgress: z.set(z.string()),
+});
 const requestListOptionsSchema = z.strictObject({
     sources: schemas.anyArray.optional(), // check only for array and not subtypes to avoid iteration over the whole thing
     sourcesFunction: schemas.anyFunction.optional(),
     persistStateKey: z.string().optional(),
     persistRequestsKey: z.string().optional(),
-    state: z
-        .strictObject({
-        nextIndex: schemas.anyNumber,
-        nextUniqueKey: z.string(),
-        inProgress: schemas.anyObject, // persisted as an array of unique keys
-    })
-        .optional(),
+    state: persistedRequestListState.optional(),
     keepDuplicateUrls: z.boolean().default(false),
     proxyConfiguration: validators.proxyConfiguration.optional(),
     httpClient: schemas.httpClient.optional(),
 });
 const listNameSchema = z.string().nullish();
 const openOptionsSchema = z.looseObject({});
+/**
+ * The conversion between the crawling position and its persisted record. Decoding also checks the record against
+ * the loaded requests, which is why the codec is built per list.
+ */
+function buildRequestListStateCodec(list) {
+    return z.codec(persistedRequestListState, requestListInternalState, {
+        decode: (record, ctx) => {
+            const { requests } = list;
+            const { nextIndex, nextUniqueKey } = record;
+            const inconsistent = (message) => {
+                ctx.issues.push({ code: 'custom', message, input: record });
+                return z.NEVER;
+            };
+            if (nextIndex > requests.length) {
+                return inconsistent('The state object is not consistent with RequestList, too few requests loaded.');
+            }
+            if (nextIndex < requests.length && requests[nextIndex].uniqueKey !== nextUniqueKey) {
+                return inconsistent('The state object is not consistent with RequestList the order of URLs seems to have changed.');
+            }
+            const inProgress = new Set(record.inProgress);
+            const deleteFromInProgress = [];
+            for (const uniqueKey of inProgress) {
+                const index = list.indexOf(uniqueKey);
+                if (index === undefined) {
+                    return inconsistent('The state object is not consistent with RequestList. Unknown uniqueKey is present in the state.');
+                }
+                if (index >= nextIndex) {
+                    deleteFromInProgress.push(uniqueKey);
+                }
+            }
+            // WORKAROUND:
+            // It happened to some users that state object contained something like:
+            // {
+            //   "nextIndex": 11308,
+            //   "nextUniqueKey": "https://www.anychart.com",
+            //   "inProgress": {
+            //      "https://www.ams360.com": true,
+            //      ...
+            //        "https://www.anychart.com": true,
+            // }
+            // Which then caused error "The request is not being processed (uniqueKey: https://www.anychart.com)"
+            // As a workaround, we just remove all inProgress requests whose index >= nextIndex,
+            // since they will be crawled again.
+            if (deleteFromInProgress.length) {
+                list.log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries", { deleteFromInProgress });
+                for (const uniqueKey of deleteFromInProgress) {
+                    inProgress.delete(uniqueKey);
+                }
+            }
+            return { nextIndex, inProgress };
+        },
+        encode: ({ nextIndex, inProgress }) => ({
+            nextIndex,
+            nextUniqueKey: nextIndex < list.requests.length ? list.requests[nextIndex].uniqueKey : null,
+            inProgress: [...inProgress],
+        }),
+    });
+}
 /**
  * Represents a static list of URLs to crawl.
  * The URLs can be provided either in code or parsed from a text file hosted on the web.
@@ -99,26 +162,27 @@ export class RequestList {
      * @internal
      */
     requests = [];
-    /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
-    #nextIndex = 0;
     /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
     #uniqueKeyToIndex = {};
+    /**
+     * The crawling position, kept in a {@link RecoverableState} that handles the loading, validation and
+     * periodic persistence of the record under `persistStateKey`.
+     */
+    #state;
+    #stateCodec;
     /**
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
-    inProgress = new Set();
+    get inProgress() {
+        return this.#state.currentValue.inProgress;
+    }
     /**
      * `uniqueKey`s of requests that were in progress when the state was last persisted and thus need to be
      * re-crawled after a restart. They are served before advancing through the rest of the sources.
      * @internal
      */
     #requestsToRetry = [];
-    /**
-     * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
-     * @internal
-     */
-    isStatePersisted = true;
     /**
      * Starts as false because we don't know yet and sources might change in the meantime (eg. download from live list).
      * @internal
@@ -128,11 +192,9 @@ export class RequestList {
     #isInitialized = false;
     #persistStateKey;
     #persistRequestsKey;
-    #initialState;
     #store;
     #keepDuplicateUrls;
-    // kept as TS-private: request_list tests read this field directly
-    sources;
+    #sources;
     #sourcesFunction;
     #proxyConfiguration;
     #httpClient;
@@ -148,43 +210,69 @@ export class RequestList {
         }
         this.#persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;
         this.#persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;
-        this.#initialState = state;
         this.#httpClient = httpClient;
+        this.#stateCodec = buildRequestListStateCodec({
+            requests: this.requests,
+            indexOf: (uniqueKey) => {
+                const index = this.#uniqueKeyToIndex[uniqueKey];
+                return typeof index === 'number' ? index : undefined;
+            },
+            log: this.#log,
+        });
+        const stateKey = this.#persistStateKey ?? STATE_PERSISTENCE_KEY;
+        this.#state = new RecoverableState({
+            // The `state` option is where the list starts when there is no persisted record to restore. The factory
+            // runs on `initialize()`, once the requests it is validated against are loaded.
+            defaultState: () => state
+                ? convertStateSync(this.#stateCodec, state, stateKey)
+                : { nextIndex: 0, inProgress: new Set() },
+            persistStateKey: stateKey,
+            persistenceEnabled: !!this.#persistStateKey,
+            logger: this.#log,
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: this.#stateCodec,
+            serialize: (internalState) => this.#stateCodec.encode(internalState),
+        });
         // If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.
         this.#keepDuplicateUrls = keepDuplicateUrls;
         // Will be empty after initialization to save memory.
-        this.sources = sources ? [...sources] : [];
+        this.#sources = sources ? [...sources] : [];
         this.#sourcesFunction = sourcesFunction;
         // The proxy configuration used for `requestsFromUrl` requests.
         this.#proxyConfiguration = proxyConfiguration;
-        this.persistState = this.persistState.bind(this);
     }
     /**
      * Loads all remote sources of URLs and potentially starts periodic state persistence.
      * This function must be called before you can start using the instance in a meaningful way.
      */
-    async initialize() {
+    async #initialize() {
         if (this.#isLoading) {
             throw new Error('RequestList sources are already loading or were loaded.');
         }
         this.#isLoading = true;
         await purgeDefaultStorages({ onlyPurgeOnce: true });
-        const [state, persistedRequests] = await this.loadStateAndPersistedRequests();
+        let persistedRequests;
+        if (this.#persistRequestsKey) {
+            persistedRequests = await this.#getPersistedState(this.#persistRequestsKey);
+            if (persistedRequests)
+                this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');
+        }
         // Add persisted requests / new sources in a memory efficient way because with very
         // large lists, we were running out of memory.
         if (persistedRequests) {
-            await this.addPersistedRequests(persistedRequests);
+            await this.#addPersistedRequests(persistedRequests);
         }
         else {
-            await this.addRequestsFromSources();
+            await this.#addRequestsFromSources();
         }
-        this.restoreState(state);
+        // Restores the crawling position - the requests have to be loaded first, as the record is validated
+        // against them. Also starts the periodic state persistence when `persistStateKey` is set.
+        await this.#state.initialize();
+        // All in-progress requests were interrupted and need to be re-crawled.
+        this.#requestsToRetry = [...this.inProgress];
         this.#isInitialized = true;
         if (this.#persistRequestsKey && !this.areRequestsPersisted)
-            await this.persistRequests();
-        if (this.#persistStateKey) {
-            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.persistState);
-        }
+            await this.#persistRequests();
         return this;
     }
     /**
@@ -192,18 +280,18 @@ export class RequestList {
      * This needs to be done in a memory efficient way. We should update the input
      * to a Stream once apify-client supports streams.
      */
-    async addPersistedRequests(persistedRequests) {
+    async #addPersistedRequests(persistedRequests) {
         // We don't need the sources so we purge them to
         // prevent them from hanging in memory.
-        for (let i = 0; i < this.sources.length; i++) {
+        for (let i = 0; i < this.#sources.length; i++) {
             // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
-            delete this.sources[i];
+            delete this.#sources[i];
         }
-        this.sources = [];
+        this.#sources = [];
         this.areRequestsPersisted = true;
         const requestStream = createDeserialize(persistedRequests);
         for await (const request of requestStream) {
-            this.addRequest(request);
+            this.#addRequest(request);
         }
     }
     /**
@@ -212,25 +300,25 @@ export class RequestList {
      * We need to avoid keeping both sources and requests in memory
      * to reduce memory footprint with very large sources.
      */
-    async addRequestsFromSources() {
+    async #addRequestsFromSources() {
         // We'll load all sources in sequence to ensure that they get loaded in the right order.
-        const sourcesCount = this.sources.length;
+        const sourcesCount = this.#sources.length;
         for (let i = 0; i < sourcesCount; i++) {
-            const source = this.sources[i];
+            const source = this.#sources[i];
             // Using delete here to drop the original object ASAP to free memory
             // .pop would reverse the array and .shift is SLOW.
             // oxlint-disable-next-line typescript/no-array-delete
-            delete this.sources[i];
+            delete this.#sources[i];
             if (typeof source === 'object' && source.requestsFromUrl) {
-                const fetchedRequests = await this.fetchRequestsFromUrl(source);
-                await this.addFetchedRequests(source, fetchedRequests);
+                const fetchedRequests = await this.#fetchRequestsFromUrl(source);
+                await this.#addFetchedRequests(source, fetchedRequests);
             }
             else {
-                this.addRequest(source);
+                this.#addRequest(source);
             }
         }
         // Drop the original array full of empty indexes.
-        this.sources = [];
+        this.#sources = [];
         if (this.#sourcesFunction) {
             try {
                 const sourcesFromFunction = await this.#sourcesFunction();
@@ -239,7 +327,7 @@ export class RequestList {
                     const source = sourcesFromFunction[i];
                     // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
                     delete sourcesFromFunction[i];
-                    this.addRequest(source);
+                    this.#addRequest(source);
                 }
                 sourcesFromFunction.length = 0;
             }
@@ -256,12 +344,8 @@ export class RequestList {
         if (!this.#persistStateKey) {
             throw new Error('Cannot persist state. options.persistStateKey is not set.');
         }
-        if (this.isStatePersisted)
-            return;
         try {
-            this.#store ??= await KeyValueStore.open();
-            await this.#store.setValue(this.#persistStateKey, this.getState());
-            this.isStatePersisted = true;
+            await this.#state.persistState();
         }
         catch (e) {
             const err = e;
@@ -274,144 +358,57 @@ export class RequestList {
      * leaking the listener (and the requests it retains) on the shared event manager.
      */
     async teardown() {
-        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.persistState);
-        if (this.#persistStateKey) {
-            await this.persistState();
-        }
+        await this.#state.teardown();
     }
     /**
      * Unlike persistState(), this is used only internally, since the sources
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
      * but there's no reason to persist it again afterwards, because RequestList is immutable.
      */
-    async persistRequests() {
+    async #persistRequests() {
         const serializedRequests = await serializeArray(this.requests);
         this.#store ??= await KeyValueStore.open();
         await this.#store.setValue(this.#persistRequestsKey, serializedRequests, { contentType: CONTENT_TYPE_BINARY });
         this.areRequestsPersisted = true;
     }
-    /**
-     * Restores RequestList state from a state object.
-     */
-    restoreState(state) {
-        // If there's no state it means we've not persisted any (yet).
-        if (!state)
-            return;
-        // Restore previous state.
-        if (typeof state.nextIndex !== 'number' || state.nextIndex < 0) {
-            throw new Error('The state object is invalid: nextIndex must be a non-negative number.');
-        }
-        if (state.nextIndex > this.requests.length) {
-            throw new Error('The state object is not consistent with RequestList, too few requests loaded.');
-        }
-        if (state.nextIndex < this.requests.length &&
-            this.requests[state.nextIndex].uniqueKey !== state.nextUniqueKey) {
-            throw new Error('The state object is not consistent with RequestList the order of URLs seems to have changed.');
-        }
-        const deleteFromInProgress = [];
-        state.inProgress.forEach((uniqueKey) => {
-            const index = this.#uniqueKeyToIndex[uniqueKey];
-            if (typeof index !== 'number') {
-                throw new Error('The state object is not consistent with RequestList. Unknown uniqueKey is present in the state.');
-            }
-            if (index >= state.nextIndex) {
-                deleteFromInProgress.push(uniqueKey);
-            }
-        });
-        this.#nextIndex = state.nextIndex;
-        this.inProgress = new Set(state.inProgress);
-        // WORKAROUND:
-        // It happened to some users that state object contained something like:
-        // {
-        //   "nextIndex": 11308,
-        //   "nextUniqueKey": "https://www.anychart.com",
-        //   "inProgress": {
-        //      "https://www.ams360.com": true,
-        //      ...
-        //        "https://www.anychart.com": true,
-        // }
-        // Which then caused error "The request is not being processed (uniqueKey: https://www.anychart.com)"
-        // As a workaround, we just remove all inProgress requests whose index >= nextIndex,
-        // since they will be crawled again.
-        if (deleteFromInProgress.length) {
-            this.#log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries", {
-                deleteFromInProgress,
-            });
-            for (const uniqueKey of deleteFromInProgress) {
-                this.inProgress.delete(uniqueKey);
-            }
-        }
-        // All in-progress requests were interrupted and need to be re-crawled.
-        this.#requestsToRetry = [...this.inProgress];
-    }
-    /**
-     * Attempts to load state and requests using the `RequestList` configuration
-     * and returns a tuple of [state, requests] where each may be null if not loaded.
-     */
-    async loadStateAndPersistedRequests() {
-        let state;
-        let persistedRequests;
-        if (this.#initialState) {
-            state = this.#initialState;
-            this.#log.debug('Loaded state from options.state argument.');
-        }
-        else if (this.#persistStateKey) {
-            state = await this.getPersistedState(this.#persistStateKey);
-            if (state)
-                this.#log.debug('Loaded state from key value store using the persistStateKey.');
-        }
-        if (this.#persistRequestsKey) {
-            persistedRequests = await this.getPersistedState(this.#persistRequestsKey);
-            if (persistedRequests)
-                this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');
-        }
-        return [state, persistedRequests];
-    }
     /**
      * Returns an object representing the internal state of the `RequestList` instance.
      * Note that the object's fields can change in future releases.
      */
     getState() {
-        this.ensureIsInitialized();
-        return {
-            nextIndex: this.#nextIndex,
-            nextUniqueKey: this.#nextIndex < this.requests.length ? this.requests[this.#nextIndex].uniqueKey : null,
-            inProgress: [...this.inProgress],
-        };
+        this.#ensureIsInitialized();
+        return this.#stateCodec.encode(this.#state.currentValue);
     }
     /**
      * @inheritDoc
      */
-    async isEmpty() {
-        this.ensureIsInitialized();
-        return this.#requestsToRetry.length === 0 && this.#nextIndex >= this.requests.length;
-    }
-    /**
-     * @inheritDoc
-     */
-    async isFinished() {
-        this.ensureIsInitialized();
-        return this.inProgress.size === 0 && this.#nextIndex >= this.requests.length;
+    async checkReadiness() {
+        this.#ensureIsInitialized();
+        if (this.#requestsToRetry.length > 0 || this.#state.currentValue.nextIndex < this.requests.length) {
+            return { status: 'ready' };
+        }
+        // `#requestsToRetry` is a subset of `inProgress`, so nothing in progress means nothing left to re-serve.
+        return this.inProgress.size === 0 ? { status: 'finished' } : { status: 'waiting' };
     }
     /**
      * @inheritDoc
      */
     async fetchNextRequest() {
-        this.ensureIsInitialized();
+        this.#ensureIsInitialized();
         // First re-serve any requests that were interrupted before the last state persist.
         const uniqueKey = this.#requestsToRetry.shift();
         if (uniqueKey) {
             const index = this.#uniqueKeyToIndex[uniqueKey];
-            return this.ensureRequest(this.requests[index], index);
+            return this.#ensureRequest(this.requests[index], index);
         }
         // Otherwise return next request.
-        if (this.#nextIndex < this.requests.length) {
-            const index = this.#nextIndex;
+        const state = this.#state.currentValue;
+        if (state.nextIndex < this.requests.length) {
+            const index = state.nextIndex;
             const request = this.requests[index];
-            this.inProgress.add(request.uniqueKey);
-            this.#nextIndex++;
-            this.isStatePersisted = false;
-            return this.ensureRequest(request, index);
+            state.inProgress.add(request.uniqueKey);
+            state.nextIndex++;
+            return this.#ensureRequest(request, index);
         }
         return null;
     }
@@ -426,7 +423,7 @@ export class RequestList {
             yield req;
         }
     }
-    ensureRequest(requestLike, index) {
+    #ensureRequest(requestLike, index) {
         if (requestLike instanceof Request) {
             return requestLike;
         }
@@ -438,19 +435,18 @@ export class RequestList {
      */
     async markRequestAsHandled(request) {
         const { uniqueKey } = request;
-        this.ensureUniqueKeyValid(uniqueKey);
-        this.ensureInProgress(uniqueKey);
-        this.ensureIsInitialized();
+        this.#ensureUniqueKeyValid(uniqueKey);
+        this.#ensureInProgress(uniqueKey);
+        this.#ensureIsInitialized();
         this.inProgress.delete(uniqueKey);
-        this.isStatePersisted = false;
     }
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    async addFetchedRequests(source, fetchedRequests) {
+    async #addFetchedRequests(source, fetchedRequests) {
         const { requestsFromUrl, regex } = source;
         const originalLength = this.requests.length;
-        fetchedRequests.forEach((request) => this.addRequest(request));
+        fetchedRequests.forEach((request) => this.#addRequest(request));
         const fetchedCount = fetchedRequests.length;
         const importedCount = this.requests.length - originalLength;
         this.#log.info('Fetched and loaded Requests from a remote resource.', {
@@ -462,7 +458,7 @@ export class RequestList {
             sample: JSON.stringify(fetchedRequests.slice(0, 5)),
         });
     }
-    async getPersistedState(key) {
+    async #getPersistedState(key) {
         this.#store ??= await KeyValueStore.open();
         const state = await this.#store.getValue(key);
         return state;
@@ -470,12 +466,12 @@ export class RequestList {
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    async fetchRequestsFromUrl(source) {
+    async #fetchRequestsFromUrl(source) {
         const { requestsFromUrl, regex, ...sharedOpts } = source;
         // Download remote resource and parse URLs.
         let urlsArr;
         try {
-            urlsArr = await this.downloadListOfUrls({
+            urlsArr = await this.#downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
                 proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,
@@ -496,7 +492,7 @@ export class RequestList {
      * If the `source` parameter is a string or plain object and not an instance
      * of a `Request`, then the function creates a `Request` instance.
      */
-    addRequest(source) {
+    #addRequest(source) {
         let request;
         const type = typeof source;
         if (type === 'string') {
@@ -518,7 +514,7 @@ export class RequestList {
             request.uniqueKey += `-${this.requests.length}`;
         }
         const { uniqueKey } = request;
-        this.ensureUniqueKeyValid(uniqueKey);
+        this.#ensureUniqueKeyValid(uniqueKey);
         // Skip requests with duplicate uniqueKey
         if (!Object.hasOwn(this.#uniqueKeyToIndex, uniqueKey)) {
             this.#uniqueKeyToIndex[uniqueKey] = this.requests.length;
@@ -532,7 +528,7 @@ export class RequestList {
      * Helper function that validates unique key.
      * Throws an error if uniqueKey is not a non-empty string.
      */
-    ensureUniqueKeyValid(uniqueKey) {
+    #ensureUniqueKeyValid(uniqueKey) {
         if (typeof uniqueKey !== 'string' || !uniqueKey) {
             throw new Error("Request object's uniqueKey must be a non-empty string");
         }
@@ -540,7 +536,7 @@ export class RequestList {
     /**
      * Checks that a request is currently being processed and throws an error if not.
      */
-    ensureInProgress(uniqueKey) {
+    #ensureInProgress(uniqueKey) {
         if (!this.inProgress.has(uniqueKey)) {
             throw new Error(`The request is not being processed (uniqueKey: ${uniqueKey})`);
         }
@@ -548,7 +544,7 @@ export class RequestList {
     /**
      * Throws an error if request list wasn't initialized.
      */
-    ensureIsInitialized() {
+    #ensureIsInitialized() {
         if (!this.#isInitialized) {
             throw new Error('RequestList is not initialized; you must call "await requestList.initialize()" before using it!');
         }
@@ -557,15 +553,15 @@ export class RequestList {
      * Returns the total number of unique requests present in the `RequestList`.
      */
     async getTotalCount() {
-        this.ensureIsInitialized();
+        this.#ensureIsInitialized();
         return this.requests.length;
     }
     /**
      * Returns the number of pending requests in the `RequestList`.
      */
     async getPendingCount() {
-        this.ensureIsInitialized();
-        return this.requests.length - (this.#nextIndex - this.inProgress.size);
+        this.#ensureIsInitialized();
+        return this.requests.length - (this.#state.currentValue.nextIndex - this.inProgress.size);
     }
     /**
      * Combines this list with a request manager (a {@link RequestQueue} by default) into a
@@ -582,8 +578,8 @@ export class RequestList {
      * @inheritDoc
      */
     async getHandledCount() {
-        this.ensureIsInitialized();
-        return this.#nextIndex - this.inProgress.size;
+        this.#ensureIsInitialized();
+        return this.#state.currentValue.nextIndex - this.inProgress.size;
     }
     /**
      * Opens a request list and returns a promise resolving to an instance
@@ -642,7 +638,7 @@ export class RequestList {
         if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {
             options = { ...listNameOrOptions, ...options };
             const rl = new RequestList(options);
-            await rl.initialize();
+            await rl.#initialize();
             return rl;
         }
         const listName = listNameOrOptions;
@@ -655,13 +651,13 @@ export class RequestList {
             persistRequestsKey: listName ? `${listName}-${REQUESTS_PERSISTENCE_KEY}` : options.persistRequestsKey,
             sources: sources ?? options.sources,
         });
-        await rl.initialize();
+        await rl.#initialize();
         return rl;
     }
     /**
      * @internal wraps public utility for mocking purposes
      */
-    async downloadListOfUrls(options) {
+    async #downloadListOfUrls(options) {
         return downloadListOfUrls({
             ...options,
             httpClient: this.#httpClient,