@crawlee/core 4.0.0-beta.15 → 4.0.0-beta.150

package/storages/request_list.js

@@ -1,9 +1,10 @@
 import { downloadListOfUrls } from '@crawlee/utils';
-import ow, { ArgumentError } from 'ow';
-import { Configuration } from '../configuration.js';
-import { log } from '../log.js';
+import { z } from 'zod';
+import { EventType } from '../events/event_manager.js';
 import { Request } from '../request.js';
 import { createDeserialize, serializeArray } from '../serialization.js';
+import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { KeyValueStore } from './key_value_store.js';
 import { purgeDefaultStorages } from './utils.js';
 /** @internal */
@@ -11,6 +12,24 @@ export const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';
 /** @internal */
 export const REQUESTS_PERSISTENCE_KEY = 'REQUEST_LIST_REQUESTS';
 const CONTENT_TYPE_BINARY = 'application/octet-stream';
+const requestListOptionsSchema = z.strictObject({
+    sources: schemas.anyArray.optional(), // check only for array and not subtypes to avoid iteration over the whole thing
+    sourcesFunction: schemas.anyFunction.optional(),
+    persistStateKey: z.string().optional(),
+    persistRequestsKey: z.string().optional(),
+    state: z
+        .strictObject({
+        nextIndex: schemas.anyNumber,
+        nextUniqueKey: z.string(),
+        inProgress: schemas.anyObject, // persisted as an array of unique keys
+    })
+        .optional(),
+    keepDuplicateUrls: z.boolean().default(false),
+    proxyConfiguration: validators.proxyConfiguration.optional(),
+    httpClient: schemas.httpClient.optional(),
+});
+const listNameSchema = z.string().nullish();
+const openOptionsSchema = z.looseObject({});
 /**
  * Represents a static list of URLs to crawl.
  * The URLs can be provided either in code or parsed from a text file hosted on the web.
@@ -33,8 +52,8 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
  * > In practical terms, such a combination can be useful when there is a large number of initial URLs,
  * > but more URLs would be added dynamically by the crawler.
  *
- * `RequestList` has an internal state where it stores information about which requests were already handled,
- * which are in progress and which were reclaimed. The state may be automatically persisted to the default
+ * `RequestList` has an internal state where it stores information about which requests were already handled
+ * and which are in progress. The state may be automatically persisted to the default
  * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
  * event that is periodically emitted by {@link EventManager}.
@@ -73,7 +92,7 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
  * @category Sources
  */
 export class RequestList {
-    log = log.child({ prefix: 'RequestList' });
+    #log = serviceLocator.getLogger().child({ prefix: 'RequestList' });
     /**
      * Array of all requests from all sources, in the order as they appeared in sources.
      * All requests in the array have distinct uniqueKey!
@@ -81,19 +100,20 @@ export class RequestList {
      */
     requests = [];
     /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
-    nextIndex = 0;
+    #nextIndex = 0;
     /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
-    uniqueKeyToIndex = {};
+    #uniqueKeyToIndex = {};
     /**
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
     inProgress = new Set();
     /**
-     * Set of `uniqueKey`s of requests for which reclaimRequest() was called.
+     * `uniqueKey`s of requests that were in progress when the state was last persisted and thus need to be
+     * re-crawled after a restart. They are served before advancing through the rest of the sources.
      * @internal
      */
-    reclaimed = new Set();
+    #requestsToRetry = [];
     /**
      * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
      * @internal
@@ -104,77 +124,65 @@ export class RequestList {
      * @internal
      */
     areRequestsPersisted = false;
-    isLoading = false;
-    isInitialized = false;
-    persistStateKey;
-    persistRequestsKey;
-    initialState;
-    store;
-    keepDuplicateUrls;
-    sources;
-    sourcesFunction;
-    proxyConfiguration;
-    events;
+    #isLoading = false;
+    #isInitialized = false;
+    #persistStateKey;
+    #persistRequestsKey;
+    #initialState;
+    #store;
+    #keepDuplicateUrls;
+    #sources;
+    #sourcesFunction;
+    #proxyConfiguration;
+    #httpClient;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
      * @internal
      */
     constructor(options = {}) {
-        const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, proxyConfiguration, keepDuplicateUrls = false, config = Configuration.getGlobalConfig(), } = options;
+        const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, proxyConfiguration, keepDuplicateUrls, httpClient, } = parseArgument(options, requestListOptionsSchema);
         if (!(sources || sourcesFunction)) {
-            throw new ArgumentError('At least one of "sources" or "sourcesFunction" must be provided.', this.constructor);
-        }
-        ow(options, ow.object.exactShape({
-            sources: ow.optional.array, // check only for array and not subtypes to avoid iteration over the whole thing
-            sourcesFunction: ow.optional.function,
-            persistStateKey: ow.optional.string,
-            persistRequestsKey: ow.optional.string,
-            state: ow.optional.object.exactShape({
-                nextIndex: ow.number,
-                nextUniqueKey: ow.string,
-                inProgress: ow.object,
-            }),
-            keepDuplicateUrls: ow.optional.boolean,
-            proxyConfiguration: ow.optional.object,
-        }));
-        this.persistStateKey = persistStateKey ? `SDK_${persistStateKey}` : persistStateKey;
-        this.persistRequestsKey = persistRequestsKey ? `SDK_${persistRequestsKey}` : persistRequestsKey;
-        this.initialState = state;
-        this.events = config.getEventManager();
+            throw new Error('At least one of "sources" or "sourcesFunction" must be provided.');
+        }
+        this.#persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;
+        this.#persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;
+        this.#initialState = state;
+        this.#httpClient = httpClient;
         // If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.
-        this.keepDuplicateUrls = keepDuplicateUrls;
+        this.#keepDuplicateUrls = keepDuplicateUrls;
         // Will be empty after initialization to save memory.
-        this.sources = sources ? [...sources] : [];
-        this.sourcesFunction = sourcesFunction;
+        this.#sources = sources ? [...sources] : [];
+        this.#sourcesFunction = sourcesFunction;
         // The proxy configuration used for `requestsFromUrl` requests.
-        this.proxyConfiguration = proxyConfiguration;
+        this.#proxyConfiguration = proxyConfiguration;
+        this.persistState = this.persistState.bind(this);
     }
     /**
      * Loads all remote sources of URLs and potentially starts periodic state persistence.
      * This function must be called before you can start using the instance in a meaningful way.
      */
     async initialize() {
-        if (this.isLoading) {
+        if (this.#isLoading) {
             throw new Error('RequestList sources are already loading or were loaded.');
         }
-        this.isLoading = true;
+        this.#isLoading = true;
         await purgeDefaultStorages({ onlyPurgeOnce: true });
-        const [state, persistedRequests] = await this._loadStateAndPersistedRequests();
+        const [state, persistedRequests] = await this.loadStateAndPersistedRequests();
         // Add persisted requests / new sources in a memory efficient way because with very
         // large lists, we were running out of memory.
         if (persistedRequests) {
-            await this._addPersistedRequests(persistedRequests);
+            await this.addPersistedRequests(persistedRequests);
         }
         else {
-            await this._addRequestsFromSources();
+            await this.addRequestsFromSources();
         }
-        this._restoreState(state);
-        this.isInitialized = true;
-        if (this.persistRequestsKey && !this.areRequestsPersisted)
-            await this._persistRequests();
-        if (this.persistStateKey) {
-            this.events.on("persistState" /* EventType.PERSIST_STATE */, this.persistState.bind(this));
+        this.restoreState(state);
+        this.#isInitialized = true;
+        if (this.#persistRequestsKey && !this.areRequestsPersisted)
+            await this.persistRequests();
+        if (this.#persistStateKey) {
+            serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.persistState);
         }
         return this;
     }
@@ -183,17 +191,18 @@ export class RequestList {
      * This needs to be done in a memory efficient way. We should update the input
      * to a Stream once apify-client supports streams.
      */
-    async _addPersistedRequests(persistedRequests) {
+    async addPersistedRequests(persistedRequests) {
         // We don't need the sources so we purge them to
         // prevent them from hanging in memory.
-        for (let i = 0; i < this.sources.length; i++) {
-            delete this.sources[i];
+        for (let i = 0; i < this.#sources.length; i++) {
+            // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
+            delete this.#sources[i];
         }
-        this.sources = [];
+        this.#sources = [];
         this.areRequestsPersisted = true;
         const requestStream = createDeserialize(persistedRequests);
         for await (const request of requestStream) {
-            this._addRequest(request);
+            this.addRequest(request);
         }
     }
     /**
@@ -202,32 +211,36 @@ export class RequestList {
      * We need to avoid keeping both sources and requests in memory
      * to reduce memory footprint with very large sources.
      */
-    async _addRequestsFromSources() {
+    async addRequestsFromSources() {
         // We'll load all sources in sequence to ensure that they get loaded in the right order.
-        const sourcesCount = this.sources.length;
+        const sourcesCount = this.#sources.length;
         for (let i = 0; i < sourcesCount; i++) {
-            const source = this.sources[i];
+            const source = this.#sources[i];
             // Using delete here to drop the original object ASAP to free memory
             // .pop would reverse the array and .shift is SLOW.
-            delete this.sources[i];
+            // oxlint-disable-next-line typescript/no-array-delete
+            delete this.#sources[i];
             if (typeof source === 'object' && source.requestsFromUrl) {
-                const fetchedRequests = await this._fetchRequestsFromUrl(source);
-                await this._addFetchedRequests(source, fetchedRequests);
+                const fetchedRequests = await this.fetchRequestsFromUrl(source);
+                await this.addFetchedRequests(source, fetchedRequests);
             }
             else {
-                this._addRequest(source);
+                this.addRequest(source);
             }
         }
         // Drop the original array full of empty indexes.
-        this.sources = [];
-        if (this.sourcesFunction) {
+        this.#sources = [];
+        if (this.#sourcesFunction) {
             try {
-                const sourcesFromFunction = await this.sourcesFunction();
+                const sourcesFromFunction = await this.#sourcesFunction();
                 const sourcesFromFunctionCount = sourcesFromFunction.length;
                 for (let i = 0; i < sourcesFromFunctionCount; i++) {
-                    const source = sourcesFromFunction.shift();
-                    this._addRequest(source);
+                    const source = sourcesFromFunction[i];
+                    // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
+                    delete sourcesFromFunction[i];
+                    this.addRequest(source);
                 }
+                sourcesFromFunction.length = 0;
             }
             catch (e) {
                 const err = e;
@@ -239,19 +252,30 @@ export class RequestList {
      * @inheritDoc
      */
     async persistState() {
-        if (!this.persistStateKey) {
+        if (!this.#persistStateKey) {
             throw new Error('Cannot persist state. options.persistStateKey is not set.');
         }
         if (this.isStatePersisted)
             return;
         try {
-            this.store ??= await KeyValueStore.open();
-            await this.store.setValue(this.persistStateKey, this.getState());
+            this.#store ??= await KeyValueStore.open();
+            await this.#store.setValue(this.#persistStateKey, this.getState());
             this.isStatePersisted = true;
         }
         catch (e) {
             const err = e;
-            this.log.exception(err, 'Attempted to persist state, but failed.');
+            this.#log.exception(err, 'Attempted to persist state, but failed.');
+        }
+    }
+    /**
+     * Removes the `PERSIST_STATE` event listener registered during initialization and persists
+     * the current state one last time. Call this when you are done with the `RequestList` to avoid
+     * leaking the listener (and the requests it retains) on the shared event manager.
+     */
+    async teardown() {
+        serviceLocator.getEventManager().off(EventType.PERSIST_STATE, this.persistState);
+        if (this.#persistStateKey) {
+            await this.persistState();
         }
     }
     /**
@@ -259,16 +283,16 @@ export class RequestList {
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
      * but there's no reason to persist it again afterwards, because RequestList is immutable.
      */
-    async _persistRequests() {
+    async persistRequests() {
         const serializedRequests = await serializeArray(this.requests);
-        this.store ??= await KeyValueStore.open();
-        await this.store.setValue(this.persistRequestsKey, serializedRequests, { contentType: CONTENT_TYPE_BINARY });
+        this.#store ??= await KeyValueStore.open();
+        await this.#store.setValue(this.#persistRequestsKey, serializedRequests, { contentType: CONTENT_TYPE_BINARY });
         this.areRequestsPersisted = true;
     }
     /**
      * Restores RequestList state from a state object.
      */
-    _restoreState(state) {
+    restoreState(state) {
         // If there's no state it means we've not persisted any (yet).
         if (!state)
             return;
@@ -285,7 +309,7 @@ export class RequestList {
         }
         const deleteFromInProgress = [];
         state.inProgress.forEach((uniqueKey) => {
-            const index = this.uniqueKeyToIndex[uniqueKey];
+            const index = this.#uniqueKeyToIndex[uniqueKey];
             if (typeof index !== 'number') {
                 throw new Error('The state object is not consistent with RequestList. Unknown uniqueKey is present in the state.');
             }
@@ -293,7 +317,7 @@ export class RequestList {
                 deleteFromInProgress.push(uniqueKey);
             }
         });
-        this.nextIndex = state.nextIndex;
+        this.#nextIndex = state.nextIndex;
         this.inProgress = new Set(state.inProgress);
         // WORKAROUND:
         // It happened to some users that state object contained something like:
@@ -309,36 +333,36 @@ export class RequestList {
         // As a workaround, we just remove all inProgress requests whose index >= nextIndex,
         // since they will be crawled again.
         if (deleteFromInProgress.length) {
-            this.log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries", {
+            this.#log.warning("RequestList's in-progress field is not consistent, skipping invalid in-progress entries", {
                 deleteFromInProgress,
             });
             for (const uniqueKey of deleteFromInProgress) {
                 this.inProgress.delete(uniqueKey);
             }
         }
-        // All in-progress requests need to be re-crawled
-        this.reclaimed = new Set(this.inProgress);
+        // All in-progress requests were interrupted and need to be re-crawled.
+        this.#requestsToRetry = [...this.inProgress];
     }
     /**
      * Attempts to load state and requests using the `RequestList` configuration
      * and returns a tuple of [state, requests] where each may be null if not loaded.
      */
-    async _loadStateAndPersistedRequests() {
+    async loadStateAndPersistedRequests() {
         let state;
         let persistedRequests;
-        if (this.initialState) {
-            state = this.initialState;
-            this.log.debug('Loaded state from options.state argument.');
+        if (this.#initialState) {
+            state = this.#initialState;
+            this.#log.debug('Loaded state from options.state argument.');
         }
-        else if (this.persistStateKey) {
-            state = await this._getPersistedState(this.persistStateKey);
+        else if (this.#persistStateKey) {
+            state = await this.getPersistedState(this.#persistStateKey);
             if (state)
-                this.log.debug('Loaded state from key value store using the persistStateKey.');
+                this.#log.debug('Loaded state from key value store using the persistStateKey.');
         }
-        if (this.persistRequestsKey) {
-            persistedRequests = await this._getPersistedState(this.persistRequestsKey);
+        if (this.#persistRequestsKey) {
+            persistedRequests = await this.getPersistedState(this.#persistRequestsKey);
             if (persistedRequests)
-                this.log.debug('Loaded requests from key value store using the persistRequestsKey.');
+                this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');
         }
         return [state, persistedRequests];
     }
@@ -347,45 +371,41 @@ export class RequestList {
      * Note that the object's fields can change in future releases.
      */
     getState() {
-        this._ensureIsInitialized();
+        this.ensureIsInitialized();
         return {
-            nextIndex: this.nextIndex,
-            nextUniqueKey: this.nextIndex < this.requests.length ? this.requests[this.nextIndex].uniqueKey : null,
+            nextIndex: this.#nextIndex,
+            nextUniqueKey: this.#nextIndex < this.requests.length ? this.requests[this.#nextIndex].uniqueKey : null,
             inProgress: [...this.inProgress],
         };
     }
     /**
      * @inheritDoc
      */
-    async isEmpty() {
-        this._ensureIsInitialized();
-        return this.reclaimed.size === 0 && this.nextIndex >= this.requests.length;
-    }
-    /**
-     * @inheritDoc
-     */
-    async isFinished() {
-        this._ensureIsInitialized();
-        return this.inProgress.size === 0 && this.nextIndex >= this.requests.length;
+    async checkReadiness() {
+        this.ensureIsInitialized();
+        if (this.#requestsToRetry.length > 0 || this.#nextIndex < this.requests.length) {
+            return { status: 'ready' };
+        }
+        // `#requestsToRetry` is a subset of `inProgress`, so nothing in progress means nothing left to re-serve.
+        return this.inProgress.size === 0 ? { status: 'finished' } : { status: 'waiting' };
     }
     /**
      * @inheritDoc
      */
     async fetchNextRequest() {
-        this._ensureIsInitialized();
-        // First return reclaimed requests if any.
-        const uniqueKey = this.reclaimed.values().next().value;
+        this.ensureIsInitialized();
+        // First re-serve any requests that were interrupted before the last state persist.
+        const uniqueKey = this.#requestsToRetry.shift();
         if (uniqueKey) {
-            this.reclaimed.delete(uniqueKey);
-            const index = this.uniqueKeyToIndex[uniqueKey];
+            const index = this.#uniqueKeyToIndex[uniqueKey];
             return this.ensureRequest(this.requests[index], index);
         }
         // Otherwise return next request.
-        if (this.nextIndex < this.requests.length) {
-            const index = this.nextIndex;
+        if (this.#nextIndex < this.requests.length) {
+            const index = this.#nextIndex;
             const request = this.requests[index];
             this.inProgress.add(request.uniqueKey);
-            this.nextIndex++;
+            this.#nextIndex++;
             this.isStatePersisted = false;
             return this.ensureRequest(request, index);
         }
@@ -412,34 +432,24 @@ export class RequestList {
     /**
      * @inheritDoc
      */
-    async markRequestHandled(request) {
+    async markRequestAsHandled(request) {
         const { uniqueKey } = request;
-        this._ensureUniqueKeyValid(uniqueKey);
-        this._ensureInProgressAndNotReclaimed(uniqueKey);
-        this._ensureIsInitialized();
+        this.ensureUniqueKeyValid(uniqueKey);
+        this.ensureInProgress(uniqueKey);
+        this.ensureIsInitialized();
         this.inProgress.delete(uniqueKey);
         this.isStatePersisted = false;
     }
-    /**
-     * @inheritDoc
-     */
-    async reclaimRequest(request) {
-        const { uniqueKey } = request;
-        this._ensureUniqueKeyValid(uniqueKey);
-        this._ensureInProgressAndNotReclaimed(uniqueKey);
-        this._ensureIsInitialized();
-        this.reclaimed.add(uniqueKey);
-    }
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    async _addFetchedRequests(source, fetchedRequests) {
+    async addFetchedRequests(source, fetchedRequests) {
         const { requestsFromUrl, regex } = source;
         const originalLength = this.requests.length;
-        fetchedRequests.forEach((request) => this._addRequest(request));
+        fetchedRequests.forEach((request) => this.addRequest(request));
         const fetchedCount = fetchedRequests.length;
         const importedCount = this.requests.length - originalLength;
-        this.log.info('Fetched and loaded Requests from a remote resource.', {
+        this.#log.info('Fetched and loaded Requests from a remote resource.', {
             requestsFromUrl,
             regex,
             fetchedCount,
@@ -448,23 +458,23 @@ export class RequestList {
             sample: JSON.stringify(fetchedRequests.slice(0, 5)),
         });
     }
-    async _getPersistedState(key) {
-        this.store ??= await KeyValueStore.open();
-        const state = await this.store.getValue(key);
+    async getPersistedState(key) {
+        this.#store ??= await KeyValueStore.open();
+        const state = await this.#store.getValue(key);
         return state;
     }
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    async _fetchRequestsFromUrl(source) {
+    async fetchRequestsFromUrl(source) {
         const { requestsFromUrl, regex, ...sharedOpts } = source;
         // Download remote resource and parse URLs.
         let urlsArr;
         try {
-            urlsArr = await this._downloadListOfUrls({
+            urlsArr = await this.downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
-                proxyUrl: await this.proxyConfiguration?.newUrl(),
+                proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,
             });
         }
         catch (err) {
@@ -472,7 +482,7 @@ export class RequestList {
         }
         // Skip if resource contained no URLs.
         if (!urlsArr.length) {
-            this.log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex });
+            this.#log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex });
             return [];
         }
         return urlsArr.map((url) => ({ url, ...sharedOpts }));
@@ -482,7 +492,7 @@ export class RequestList {
      * If the `source` parameter is a string or plain object and not an instance
      * of a `Request`, then the function creates a `Request` instance.
      */
-    _addRequest(source) {
+    addRequest(source) {
         let request;
         const type = typeof source;
         if (type === 'string') {
@@ -500,61 +510,76 @@ export class RequestList {
         const hasUniqueKey = Reflect.has(Object(source), 'uniqueKey');
         request.uniqueKey ??= Request.computeUniqueKey(request);
         // Add index to uniqueKey if duplicates are to be kept
-        if (this.keepDuplicateUrls && !hasUniqueKey) {
+        if (this.#keepDuplicateUrls && !hasUniqueKey) {
             request.uniqueKey += `-${this.requests.length}`;
         }
         const { uniqueKey } = request;
-        this._ensureUniqueKeyValid(uniqueKey);
+        this.ensureUniqueKeyValid(uniqueKey);
         // Skip requests with duplicate uniqueKey
-        if (!Object.hasOwn(this.uniqueKeyToIndex, uniqueKey)) {
-            this.uniqueKeyToIndex[uniqueKey] = this.requests.length;
+        if (!Object.hasOwn(this.#uniqueKeyToIndex, uniqueKey)) {
+            this.#uniqueKeyToIndex[uniqueKey] = this.requests.length;
             this.requests.push(request);
         }
-        else if (this.keepDuplicateUrls) {
-            this.log.warning(`Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`);
+        else if (this.#keepDuplicateUrls) {
+            this.#log.warning(`Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`);
         }
     }
     /**
      * Helper function that validates unique key.
      * Throws an error if uniqueKey is not a non-empty string.
      */
-    _ensureUniqueKeyValid(uniqueKey) {
+    ensureUniqueKeyValid(uniqueKey) {
         if (typeof uniqueKey !== 'string' || !uniqueKey) {
             throw new Error("Request object's uniqueKey must be a non-empty string");
         }
     }
     /**
-     * Checks that request is not reclaimed and throws an error if so.
+     * Checks that a request is currently being processed and throws an error if not.
      */
-    _ensureInProgressAndNotReclaimed(uniqueKey) {
+    ensureInProgress(uniqueKey) {
         if (!this.inProgress.has(uniqueKey)) {
             throw new Error(`The request is not being processed (uniqueKey: ${uniqueKey})`);
         }
-        if (this.reclaimed.has(uniqueKey)) {
-            throw new Error(`The request was already reclaimed (uniqueKey: ${uniqueKey})`);
-        }
     }
     /**
      * Throws an error if request list wasn't initialized.
      */
-    _ensureIsInitialized() {
-        if (!this.isInitialized) {
+    ensureIsInitialized() {
+        if (!this.#isInitialized) {
             throw new Error('RequestList is not initialized; you must call "await requestList.initialize()" before using it!');
         }
     }
     /**
      * Returns the total number of unique requests present in the `RequestList`.
      */
-    length() {
-        this._ensureIsInitialized();
+    async getTotalCount() {
+        this.ensureIsInitialized();
         return this.requests.length;
     }
+    /**
+     * Returns the number of pending requests in the `RequestList`.
+     */
+    async getPendingCount() {
+        this.ensureIsInitialized();
+        return this.requests.length - (this.#nextIndex - this.inProgress.size);
+    }
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    async toTandem(requestManager) {
+        // Import here to avoid circular imports.
+        const { RequestManagerTandem } = await import('./request_manager_tandem.js');
+        const { RequestQueue } = await import('./request_queue.js');
+        return new RequestManagerTandem(this, requestManager ?? (await RequestQueue.open()));
+    }
     /**
      * @inheritDoc
      */
-    handledCount() {
-        this._ensureIsInitialized();
-        return this.nextIndex - this.inProgress.size;
+    async getHandledCount() {
+        this.ensureIsInitialized();
+        return this.#nextIndex - this.inProgress.size;
     }
     /**
      * Opens a request list and returns a promise resolving to an instance
@@ -617,9 +642,9 @@ export class RequestList {
             return rl;
         }
         const listName = listNameOrOptions;
-        ow(listName, ow.optional.any(ow.string, ow.null));
-        ow(sources, ow.array);
-        ow(options, ow.object.is((v) => !Array.isArray(v)));
+        parseArgument(listName, listNameSchema);
+        parseArgument(sources, schemas.anyArray);
+        parseArgument(options, openOptionsSchema);
         const rl = new RequestList({
             ...options,
             persistStateKey: listName ? `${listName}-${STATE_PERSISTENCE_KEY}` : options.persistStateKey,
@@ -632,8 +657,10 @@ export class RequestList {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    async _downloadListOfUrls(options) {
-        return downloadListOfUrls(options);
+    async downloadListOfUrls(options) {
+        return downloadListOfUrls({
+            ...options,
+            httpClient: this.#httpClient,
+        });
     }
 }
-//# sourceMappingURL=request_list.js.map