@crawlee/core 4.0.0-beta.8 → 4.0.0-beta.80

package/storages/request_dedup_cache.d.ts

@@ -0,0 +1,23 @@
+/**
+ * A fixed-size, direct-mapped cache for `uniqueKey`-based request deduplication.
+ *
+ * `RequestProvider.requestCache` only remembers the first batch of requests, so repeated
+ * `addRequestsBatched()` calls with overlapping URLs re-submit already-enqueued requests
+ * (https://github.com/apify/crawlee/issues/3120). This is a separate, cheaper cache we can populate on
+ * every batch: a fixed number of slots indexed by a hash of the request's cache key, storing the
+ * server-assigned `requestId`. Memory is capped by the slot count regardless of the working set size;
+ * a hash collision just overwrites a slot, causing an occasional cache miss (a harmless re-submission)
+ * but never a false hit — so a genuinely new request is never dropped.
+ *
+ * @internal
+ */
+export declare class RequestDeduplicationCache {
+    private readonly size;
+    private keys;
+    private ids;
+    constructor(size?: number);
+    get(cacheKey: string): string | null;
+    add(cacheKey: string, requestId: string): void;
+    clear(): void;
+    private indexOf;
+}

package/storages/request_dedup_cache.js

@@ -0,0 +1,48 @@
+/**
+ * A fixed-size, direct-mapped cache for `uniqueKey`-based request deduplication.
+ *
+ * `RequestProvider.requestCache` only remembers the first batch of requests, so repeated
+ * `addRequestsBatched()` calls with overlapping URLs re-submit already-enqueued requests
+ * (https://github.com/apify/crawlee/issues/3120). This is a separate, cheaper cache we can populate on
+ * every batch: a fixed number of slots indexed by a hash of the request's cache key, storing the
+ * server-assigned `requestId`. Memory is capped by the slot count regardless of the working set size;
+ * a hash collision just overwrites a slot, causing an occasional cache miss (a harmless re-submission)
+ * but never a false hit — so a genuinely new request is never dropped.
+ *
+ * @internal
+ */
+export class RequestDeduplicationCache {
+    size;
+    keys;
+    ids;
+    // The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.
+    constructor(size = 1_000_000) {
+        this.size = size;
+        this.keys = new Array(size);
+        this.ids = new Array(size);
+    }
+    get(cacheKey) {
+        const index = this.indexOf(cacheKey);
+        return this.keys[index] === cacheKey ? this.ids[index] : null;
+    }
+    add(cacheKey, requestId) {
+        const index = this.indexOf(cacheKey);
+        this.keys[index] = cacheKey;
+        this.ids[index] = requestId;
+    }
+    clear() {
+        this.keys = new Array(this.size);
+        this.ids = new Array(this.size);
+    }
+    // A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.
+    indexOf(cacheKey) {
+        /* eslint-disable no-bitwise */
+        let hash = 0x811c9dc5;
+        for (let i = 0; i < cacheKey.length; i++) {
+            hash ^= cacheKey.charCodeAt(i);
+            hash = Math.imul(hash, 0x01000193);
+        }
+        return (hash >>> 0) % this.size;
+        /* eslint-enable no-bitwise */
+    }
+}

package/storages/request_list.d.ts

@@ -1,75 +1,13 @@
-import type { Dictionary } from '@crawlee/types';
-import { Configuration } from '../configuration.js';
+import type { BaseHttpClient, Dictionary } from '@crawlee/types';
+import type { Configuration } from '../configuration.js';
 import type { ProxyConfiguration } from '../proxy_configuration.js';
 import { type InternalSource, Request, type RequestOptions, type Source } from '../request.js';
+import type { IRequestLoader } from './request_loader.js';
+import type { IRequestManager } from './request_manager.js';
 /** @internal */
 export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
 /** @internal */
 export declare const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
-/**
- * Represents a static list of URLs to crawl.
- */
-export interface IRequestList {
-    /**
-     * Returns the total number of unique requests present in the list.
-     */
-    length(): number;
-    /**
-     * Returns `true` if all requests were already handled and there are no more left.
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * Resolves to `true` if the next call to {@link IRequestList.fetchNextRequest} function
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the list is empty, there might be some pending requests currently being processed.
-     */
-    isEmpty(): Promise<boolean>;
-    /**
-     * Returns number of handled requests.
-     */
-    handledCount(): number;
-    /**
-     * Persists the current state of the `IRequestList` into the default {@link KeyValueStore}.
-     * The state is persisted automatically in regular intervals, but calling this method manually
-     * is useful in cases where you want to have the most current state available after you pause
-     * or stop fetching its requests. For example after you pause or abort a crawl. Or just before
-     * a server migration.
-     */
-    persistState(): Promise<void>;
-    /**
-     * Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
-     * using the {@link RequestList.reclaimRequest} function, if there is any.
-     * Otherwise it gets the next request from sources.
-     *
-     * The function's `Promise` resolves to `null` if there are no more
-     * requests to process.
-     */
-    fetchNextRequest(): Promise<Request | null>;
-    /**
-     * Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
-     * using the {@link RequestList.reclaimRequest} function, if there is any.
-     * Otherwise it gets the next request from sources.
-     *
-     * The function resolves to `null` if there are no more requests to process.
-     *
-     * Can be used to iterate over the `RequestList` instance in a `for await .. of` loop.
-     * Provides an alternative for the repeated use of `fetchNextRequest`.
-     */
-    [Symbol.asyncIterator](): AsyncGenerator<Request>;
-    /**
-     * Reclaims request to the list if its processing failed.
-     * The request will become available in the next `this.fetchNextRequest()`.
-     */
-    reclaimRequest(request: Request): Promise<void>;
-    /**
-     * Marks request as handled after successful processing.
-     */
-    markRequestHandled(request: Request): Promise<void>;
-    /**
-     * @internal
-     */
-    inProgress: Set<string>;
-}
 export interface RequestListOptions {
     /**
      * An array of sources of URLs for the {@link RequestList}. It can be either an array of strings,
@@ -209,6 +147,12 @@ export interface RequestListOptions {
     keepDuplicateUrls?: boolean;
     /** @internal */
     config?: Configuration;
+    /**
+     * The HTTP client to be used to download `requestsFromUrl` URLs.
+     *
+     * If not specified the `RequestList` will use the default HTTP client.
+     */
+    httpClient?: BaseHttpClient;
 }
 /**
  * Represents a static list of URLs to crawl.
@@ -232,8 +176,8 @@ export interface RequestListOptions {
  * > In practical terms, such a combination can be useful when there is a large number of initial URLs,
  * > but more URLs would be added dynamically by the crawler.
  *
- * `RequestList` has an internal state where it stores information about which requests were already handled,
- * which are in progress and which were reclaimed. The state may be automatically persisted to the default
+ * `RequestList` has an internal state where it stores information about which requests were already handled
+ * and which are in progress. The state may be automatically persisted to the default
  * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
  * event that is periodically emitted by {@link EventManager}.
@@ -271,7 +215,7 @@ export interface RequestListOptions {
  * ```
  * @category Sources
  */
-export declare class RequestList implements IRequestList {
+export declare class RequestList implements IRequestLoader {
     private log;
     /**
      * Array of all requests from all sources, in the order as they appeared in sources.
@@ -289,10 +233,11 @@ export declare class RequestList implements IRequestList {
      */
     inProgress: Set<string>;
     /**
-     * Set of `uniqueKey`s of requests for which reclaimRequest() was called.
+     * `uniqueKey`s of requests that were in progress when the state was last persisted and thus need to be
+     * re-crawled after a restart. They are served before advancing through the rest of the sources.
      * @internal
      */
-    reclaimed: Set<string>;
+    private requestsToRetry;
     /**
      * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
      * @internal
@@ -313,7 +258,7 @@ export declare class RequestList implements IRequestList {
     private sources;
     private sourcesFunction?;
     private proxyConfiguration?;
-    private events;
+    private httpClient?;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
@@ -342,6 +287,12 @@ export declare class RequestList implements IRequestList {
      * @inheritDoc
      */
     persistState(): Promise<void>;
+    /**
+     * Removes the `PERSIST_STATE` event listener registered during initialization and persists
+     * the current state one last time. Call this when you are done with the `RequestList` to avoid
+     * leaking the listener (and the requests it retains) on the shared event manager.
+     */
+    teardown(): Promise<void>;
     /**
      * Unlike persistState(), this is used only internally, since the sources
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
@@ -382,11 +333,7 @@ export declare class RequestList implements IRequestList {
     /**
      * @inheritDoc
      */
-    markRequestHandled(request: Request): Promise<void>;
-    /**
-     * @inheritDoc
-     */
-    reclaimRequest(request: Request): Promise<void>;
+    markRequestAsHandled(request: Request): Promise<void>;
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
@@ -408,9 +355,9 @@ export declare class RequestList implements IRequestList {
      */
     protected _ensureUniqueKeyValid(uniqueKey: string): void;
     /**
-     * Checks that request is not reclaimed and throws an error if so.
+     * Checks that a request is currently being processed and throws an error if not.
      */
-    protected _ensureInProgressAndNotReclaimed(uniqueKey: string): void;
+    protected _ensureInProgress(uniqueKey: string): void;
     /**
      * Throws an error if request list wasn't initialized.
      */
@@ -418,11 +365,21 @@ export declare class RequestList implements IRequestList {
     /**
      * Returns the total number of unique requests present in the `RequestList`.
      */
-    length(): number;
+    getTotalCount(): Promise<number>;
+    /**
+     * Returns the number of pending requests in the `RequestList`.
+     */
+    getPendingCount(): Promise<number>;
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
     /**
      * @inheritDoc
      */
-    handledCount(): number;
+    getHandledCount(): Promise<number>;
     /**
      * Opens a request list and returns a promise resolving to an instance
      * of the {@link RequestList} class that is already initialized.
@@ -509,4 +466,3 @@ export interface RequestListState {
 type RequestListSource = string | Source;
 export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;
 export {};
-//# sourceMappingURL=request_list.d.ts.map

package/storages/request_list.js

@@ -1,9 +1,8 @@
 import { downloadListOfUrls } from '@crawlee/utils';
 import ow, { ArgumentError } from 'ow';
-import { Configuration } from '../configuration.js';
-import { log } from '../log.js';
 import { Request } from '../request.js';
 import { createDeserialize, serializeArray } from '../serialization.js';
+import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from './key_value_store.js';
 import { purgeDefaultStorages } from './utils.js';
 /** @internal */
@@ -33,8 +32,8 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
  * > In practical terms, such a combination can be useful when there is a large number of initial URLs,
  * > but more URLs would be added dynamically by the crawler.
  *
- * `RequestList` has an internal state where it stores information about which requests were already handled,
- * which are in progress and which were reclaimed. The state may be automatically persisted to the default
+ * `RequestList` has an internal state where it stores information about which requests were already handled
+ * and which are in progress. The state may be automatically persisted to the default
  * {@link KeyValueStore} by setting the `persistStateKey` option so that if the Node.js process is restarted,
  * the crawling can continue where it left off. The automated persisting is launched upon receiving the `persistState`
  * event that is periodically emitted by {@link EventManager}.
@@ -73,7 +72,7 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
  * @category Sources
  */
 export class RequestList {
-    log = log.child({ prefix: 'RequestList' });
+    log = serviceLocator.getLogger().child({ prefix: 'RequestList' });
     /**
      * Array of all requests from all sources, in the order as they appeared in sources.
      * All requests in the array have distinct uniqueKey!
@@ -90,10 +89,11 @@ export class RequestList {
      */
     inProgress = new Set();
     /**
-     * Set of `uniqueKey`s of requests for which reclaimRequest() was called.
+     * `uniqueKey`s of requests that were in progress when the state was last persisted and thus need to be
+     * re-crawled after a restart. They are served before advancing through the rest of the sources.
      * @internal
      */
-    reclaimed = new Set();
+    requestsToRetry = [];
     /**
      * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
      * @internal
@@ -114,14 +114,14 @@ export class RequestList {
     sources;
     sourcesFunction;
     proxyConfiguration;
-    events;
+    httpClient;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
      * @internal
      */
     constructor(options = {}) {
-        const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, proxyConfiguration, keepDuplicateUrls = false, config = Configuration.getGlobalConfig(), } = options;
+        const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, proxyConfiguration, keepDuplicateUrls = false, httpClient, } = options;
         if (!(sources || sourcesFunction)) {
             throw new ArgumentError('At least one of "sources" or "sourcesFunction" must be provided.', this.constructor);
         }
@@ -137,11 +137,12 @@ export class RequestList {
             }),
             keepDuplicateUrls: ow.optional.boolean,
             proxyConfiguration: ow.optional.object,
+            httpClient: ow.optional.object,
         }));
-        this.persistStateKey = persistStateKey ? `SDK_${persistStateKey}` : persistStateKey;
-        this.persistRequestsKey = persistRequestsKey ? `SDK_${persistRequestsKey}` : persistRequestsKey;
+        this.persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey;
+        this.persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey;
         this.initialState = state;
-        this.events = config.getEventManager();
+        this.httpClient = httpClient;
         // If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.
         this.keepDuplicateUrls = keepDuplicateUrls;
         // Will be empty after initialization to save memory.
@@ -149,6 +150,7 @@ export class RequestList {
         this.sourcesFunction = sourcesFunction;
         // The proxy configuration used for `requestsFromUrl` requests.
         this.proxyConfiguration = proxyConfiguration;
+        this.persistState = this.persistState.bind(this);
     }
     /**
      * Loads all remote sources of URLs and potentially starts periodic state persistence.
@@ -174,7 +176,7 @@ export class RequestList {
         if (this.persistRequestsKey && !this.areRequestsPersisted)
             await this._persistRequests();
         if (this.persistStateKey) {
-            this.events.on("persistState" /* EventType.PERSIST_STATE */, this.persistState.bind(this));
+            serviceLocator.getEventManager().on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
         }
         return this;
     }
@@ -187,6 +189,7 @@ export class RequestList {
         // We don't need the sources so we purge them to
         // prevent them from hanging in memory.
         for (let i = 0; i < this.sources.length; i++) {
+            // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
             delete this.sources[i];
         }
         this.sources = [];
@@ -209,6 +212,7 @@ export class RequestList {
             const source = this.sources[i];
             // Using delete here to drop the original object ASAP to free memory
             // .pop would reverse the array and .shift is SLOW.
+            // oxlint-disable-next-line typescript/no-array-delete
             delete this.sources[i];
             if (typeof source === 'object' && source.requestsFromUrl) {
                 const fetchedRequests = await this._fetchRequestsFromUrl(source);
@@ -225,9 +229,12 @@ export class RequestList {
                 const sourcesFromFunction = await this.sourcesFunction();
                 const sourcesFromFunctionCount = sourcesFromFunction.length;
                 for (let i = 0; i < sourcesFromFunctionCount; i++) {
-                    const source = sourcesFromFunction.shift();
+                    const source = sourcesFromFunction[i];
+                    // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
+                    delete sourcesFromFunction[i];
                     this._addRequest(source);
                 }
+                sourcesFromFunction.length = 0;
             }
             catch (e) {
                 const err = e;
@@ -254,6 +261,17 @@ export class RequestList {
             this.log.exception(err, 'Attempted to persist state, but failed.');
         }
     }
+    /**
+     * Removes the `PERSIST_STATE` event listener registered during initialization and persists
+     * the current state one last time. Call this when you are done with the `RequestList` to avoid
+     * leaking the listener (and the requests it retains) on the shared event manager.
+     */
+    async teardown() {
+        serviceLocator.getEventManager().off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        if (this.persistStateKey) {
+            await this.persistState();
+        }
+    }
     /**
      * Unlike persistState(), this is used only internally, since the sources
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
@@ -316,8 +334,8 @@ export class RequestList {
                 this.inProgress.delete(uniqueKey);
             }
         }
-        // All in-progress requests need to be re-crawled
-        this.reclaimed = new Set(this.inProgress);
+        // All in-progress requests were interrupted and need to be re-crawled.
+        this.requestsToRetry = [...this.inProgress];
     }
     /**
      * Attempts to load state and requests using the `RequestList` configuration
@@ -359,7 +377,7 @@ export class RequestList {
      */
     async isEmpty() {
         this._ensureIsInitialized();
-        return this.reclaimed.size === 0 && this.nextIndex >= this.requests.length;
+        return this.requestsToRetry.length === 0 && this.nextIndex >= this.requests.length;
     }
     /**
      * @inheritDoc
@@ -373,10 +391,9 @@ export class RequestList {
      */
     async fetchNextRequest() {
         this._ensureIsInitialized();
-        // First return reclaimed requests if any.
-        const uniqueKey = this.reclaimed.values().next().value;
+        // First re-serve any requests that were interrupted before the last state persist.
+        const uniqueKey = this.requestsToRetry.shift();
         if (uniqueKey) {
-            this.reclaimed.delete(uniqueKey);
             const index = this.uniqueKeyToIndex[uniqueKey];
             return this.ensureRequest(this.requests[index], index);
         }
@@ -412,24 +429,14 @@ export class RequestList {
     /**
      * @inheritDoc
      */
-    async markRequestHandled(request) {
+    async markRequestAsHandled(request) {
         const { uniqueKey } = request;
         this._ensureUniqueKeyValid(uniqueKey);
-        this._ensureInProgressAndNotReclaimed(uniqueKey);
+        this._ensureInProgress(uniqueKey);
         this._ensureIsInitialized();
         this.inProgress.delete(uniqueKey);
         this.isStatePersisted = false;
     }
-    /**
-     * @inheritDoc
-     */
-    async reclaimRequest(request) {
-        const { uniqueKey } = request;
-        this._ensureUniqueKeyValid(uniqueKey);
-        this._ensureInProgressAndNotReclaimed(uniqueKey);
-        this._ensureIsInitialized();
-        this.reclaimed.add(uniqueKey);
-    }
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
@@ -524,15 +531,12 @@ export class RequestList {
         }
     }
     /**
-     * Checks that request is not reclaimed and throws an error if so.
+     * Checks that a request is currently being processed and throws an error if not.
      */
-    _ensureInProgressAndNotReclaimed(uniqueKey) {
+    _ensureInProgress(uniqueKey) {
         if (!this.inProgress.has(uniqueKey)) {
             throw new Error(`The request is not being processed (uniqueKey: ${uniqueKey})`);
         }
-        if (this.reclaimed.has(uniqueKey)) {
-            throw new Error(`The request was already reclaimed (uniqueKey: ${uniqueKey})`);
-        }
     }
     /**
      * Throws an error if request list wasn't initialized.
@@ -545,14 +549,32 @@ export class RequestList {
     /**
      * Returns the total number of unique requests present in the `RequestList`.
      */
-    length() {
+    async getTotalCount() {
         this._ensureIsInitialized();
         return this.requests.length;
     }
+    /**
+     * Returns the number of pending requests in the `RequestList`.
+     */
+    async getPendingCount() {
+        this._ensureIsInitialized();
+        return this.requests.length - (this.nextIndex - this.inProgress.size);
+    }
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    async toTandem(requestManager) {
+        // Import here to avoid circular imports.
+        const { RequestManagerTandem } = await import('./request_manager_tandem.js');
+        const { RequestQueue } = await import('./request_queue.js');
+        return new RequestManagerTandem(this, requestManager ?? (await RequestQueue.open()));
+    }
     /**
      * @inheritDoc
      */
-    handledCount() {
+    async getHandledCount() {
         this._ensureIsInitialized();
         return this.nextIndex - this.inProgress.size;
     }
@@ -633,7 +655,9 @@ export class RequestList {
      * @internal wraps public utility for mocking purposes
      */
     async _downloadListOfUrls(options) {
-        return downloadListOfUrls(options);
+        return downloadListOfUrls({
+            ...options,
+            httpClient: this.httpClient,
+        });
     }
 }
-//# sourceMappingURL=request_list.js.map

package/storages/request_loader.d.ts

@@ -0,0 +1,96 @@
+import type { Dictionary } from '@crawlee/types';
+import type { Request } from '../request.js';
+import type { IRequestManager } from './request_manager.js';
+import type { RequestQueueOperationInfo } from './request_queue.js';
+/**
+ * An abstract interface defining a read-only stream of requests to crawl.
+ *
+ * Request loaders are used to manage and provide access to a storage of crawling requests.
+ *
+ * Key responsibilities:
+ * - Fetching the next request to be processed.
+ * - Marking requests as handled once they are no longer in progress.
+ * - Managing state information such as the total and handled request counts.
+ *
+ * ## Request lifecycle contract
+ *
+ * Every request returned by {@link IRequestLoader.fetchNextRequest} is considered **in progress**
+ * until it is passed to {@link IRequestLoader.markRequestAsHandled}. Once you fetch a request, you are
+ * obligated to eventually mark it as handled — there is no way to hand a request back to a loader
+ * (only an {@link IRequestManager} can reclaim requests for a retry). "Handled" therefore means
+ * "finished with this request", whether processing succeeded or was abandoned after exhausting retries.
+ *
+ * Honoring this contract matters for three reasons:
+ * - **Restarts and migrations:** loaders that persist their state (see {@link IRequestLoader.persistState})
+ *   treat in-progress requests as interrupted and re-serve them after a restart. A request that is fetched
+ *   but never marked handled will be crawled again.
+ * - **Termination detection:** {@link IRequestLoader.isFinished} only resolves to `true` once nothing is
+ *   in progress. Leaving a request unmarked keeps the crawler running indefinitely.
+ * - **Bookkeeping:** the handled and pending counts are derived from the set of in-progress requests, so
+ *   skipping {@link IRequestLoader.markRequestAsHandled} corrupts {@link IRequestLoader.getHandledCount}
+ *   and {@link IRequestLoader.getPendingCount}.
+ *
+ * Concrete implementations such as {@link RequestList} or {@link SitemapRequestLoader} build on this interface.
+ * The {@link IRequestManager} interface extends it with the capability to enqueue and reclaim requests.
+ */
+export interface IRequestLoader {
+    /**
+     * Returns an approximation of the total number of requests in the loader (i.e. pending + handled).
+     */
+    getTotalCount(): Promise<number>;
+    /**
+     * Returns an approximation of the number of pending requests in the loader.
+     */
+    getPendingCount(): Promise<number>;
+    /**
+     * Returns the number of requests in the loader that have been handled.
+     */
+    getHandledCount(): Promise<number>;
+    /**
+     * Returns `true` if all requests were already handled and there are no more left.
+     */
+    isFinished(): Promise<boolean>;
+    /**
+     * Resolves to `true` if the next call to {@link IRequestLoader.fetchNextRequest} function
+     * would return `null`, otherwise it resolves to `false`.
+     * Note that even if the loader is empty, there might be some pending requests currently being processed.
+     */
+    isEmpty(): Promise<boolean>;
+    /**
+     * Gets the next {@link Request} to process, or `null` if there are no more pending requests.
+     *
+     * The returned request is marked as **in progress** and remains so until it is passed to
+     * {@link IRequestLoader.markRequestAsHandled}. The caller is responsible for eventually marking
+     * every fetched request as handled; otherwise the loader never considers itself finished and the
+     * request may be re-served after a restart. See the request lifecycle contract on {@link IRequestLoader}.
+     */
+    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+    /**
+     * Can be used to iterate over the loader instance in a `for await .. of` loop.
+     * Provides an alternative for the repeated use of `fetchNextRequest`.
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Request>;
+    /**
+     * Marks a request previously returned by {@link IRequestLoader.fetchNextRequest} as handled,
+     * removing it from the set of in-progress requests.
+     *
+     * Call this once you are done with the request — whether processing succeeded or was abandoned after
+     * exhausting retries. Because a loader cannot take a request back, marking it handled is the only way to
+     * signal completion; failing to do so prevents {@link IRequestLoader.isFinished} from ever resolving to
+     * `true` and skews the handled and pending counts. See the request lifecycle contract on {@link IRequestLoader}.
+     */
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    /**
+     * Persists the current state of the loader into the default {@link KeyValueStore}.
+     *
+     * Not all loaders support persistence; implementations that do not should leave this `undefined`.
+     */
+    persistState?(): Promise<void>;
+    /**
+     * Combines the loader with a request manager to support adding and reclaiming requests.
+     *
+     * @param requestManager Request manager to combine the loader with. If not provided, the default
+     *  {@link RequestQueue} is used.
+     */
+    toTandem?(requestManager?: IRequestManager): Promise<IRequestManager>;
+}

package/storages/request_loader.js

@@ -0,0 +1 @@
+export {};

package/storages/request_manager.d.ts

@@ -0,0 +1,33 @@
+import type { Request, Source } from '../request.js';
+import type { IRequestLoader } from './request_loader.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
+export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
+/**
+ * Extends the read-only {@link IRequestLoader} interface with the capability to enqueue new requests
+ * and reclaim failed ones.
+ */
+export interface IRequestManager extends IRequestLoader {
+    /**
+     * Reclaims request to the provider if its processing failed.
+     * The request will be returned by some subsequent `fetchNextRequest()` call.
+     */
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Remove all requests from the queue but keep the queue itself, resetting it
+     * so it can be reused (e.g. across multiple `crawler.run()` calls).
+     *
+     * Implementations that do not support purging may leave this `undefined`.
+     */
+    purge?(): Promise<void>;
+    /**
+     * Tells the manager how long a consumer expects to hold a request fetched via `fetchNextRequest()`
+     * before marking it handled or reclaiming it (typically the request-handler timeout plus padding).
+     *
+     * Managers backed by a storage backend that reserves requests via locking use this to avoid handing
+     * the same request out again while it is still being processed. Implementations that do not need
+     * this hint may leave it `undefined`.
+     */
+    setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
+}

package/storages/request_manager.js

@@ -0,0 +1 @@
+export {};