@crawlee/core 4.0.0-beta.20 → 4.0.0-beta.200

package/storages/request_loader.d.ts

@@ -0,0 +1,127 @@
+import type { Dictionary } from '@crawlee/types';
+import type { Request } from '../request.js';
+import type { IRequestManager } from './request_manager.js';
+import type { RequestQueueOperationInfo } from './request_queue.js';
+/**
+ * A request source's own availability, in a single answer.
+ *
+ * - `ready` — the next {@link IRequestLoader.fetchNextRequest} is expected to hand something over.
+ * - `waiting` — nothing to fetch right now, but the source is not done: requests are in progress, are being
+ *   added in the background, or are held back until `readyAt`.
+ * - `stalled` — the source holds requests it cannot make progress on. Only a manager that paces its own
+ *   dispatch can reach this; see {@link ThrottlingRequestManager}.
+ * - `finished` — everything has been handled.
+ */
+export type RequestSourceStatus = {
+    status: 'ready';
+} | {
+    status: 'waiting';
+    /**
+     * A `Date.now()` timestamp at which the source expects to become `ready`. Absent when the wait has
+     * no clock (an in-progress request, a background add), leaving a consumer to poll.
+     */
+    readyAt?: number;
+} | {
+    status: 'stalled';
+    reason: string;
+} | {
+    status: 'finished';
+};
+/** Loaders never stall — only a manager that paces its own dispatch can. */
+export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
+    status: 'stalled';
+}>;
+/**
+ * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
+ *
+ * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
+ * nothing.
+ *
+ * @internal
+ */
+export declare function joinRequestSourceStatuses(a: RequestSourceStatus, b: RequestSourceStatus): RequestSourceStatus;
+/**
+ * An abstract interface defining a read-only stream of requests to crawl.
+ *
+ * Request loaders are used to manage and provide access to a storage of crawling requests.
+ *
+ * Key responsibilities:
+ * - Fetching the next request to be processed.
+ * - Marking requests as handled once they are no longer in progress.
+ * - Managing state information such as the total and handled request counts.
+ *
+ * ## Request lifecycle contract
+ *
+ * Every request returned by {@link IRequestLoader.fetchNextRequest} is considered **in progress**
+ * until it is passed to {@link IRequestLoader.markRequestAsHandled}. Once you fetch a request, you are
+ * obligated to eventually mark it as handled — there is no way to hand a request back to a loader
+ * (only an {@link IRequestManager} can reclaim requests for a retry). "Handled" therefore means
+ * "finished with this request", whether processing succeeded or was abandoned after exhausting retries.
+ *
+ * Honoring this contract matters for three reasons:
+ * - **Restarts and migrations:** loaders that persist their state (such as {@link RequestList})
+ *   treat in-progress requests as interrupted and re-serve them after a restart. A request that is fetched
+ *   but never marked handled will be crawled again.
+ * - **Termination detection:** {@link IRequestLoader.checkReadiness} only reports `finished` once nothing is
+ *   in progress. Leaving a request unmarked keeps the crawler running indefinitely.
+ * - **Bookkeeping:** the handled and pending counts are derived from the set of in-progress requests, so
+ *   skipping {@link IRequestLoader.markRequestAsHandled} corrupts {@link IRequestLoader.getHandledCount}
+ *   and {@link IRequestLoader.getPendingCount}.
+ *
+ * Concrete implementations such as {@link RequestList} or {@link SitemapRequestLoader} build on this interface.
+ * The {@link IRequestManager} interface extends it with the capability to enqueue and reclaim requests.
+ */
+export interface IRequestLoader {
+    /**
+     * Returns an approximation of the total number of requests in the loader (i.e. pending + handled).
+     */
+    getTotalCount(): Promise<number>;
+    /**
+     * Returns an approximation of the number of pending requests in the loader.
+     */
+    getPendingCount(): Promise<number>;
+    /**
+     * Returns the number of requests in the loader that have been handled.
+     */
+    getHandledCount(): Promise<number>;
+    /**
+     * Reports whether the loader has a request to hand over, is waiting on one, or is done — see
+     * {@link RequestSourceStatus}.
+     *
+     * A consumer's task loop is gated on this, so implementations MUST answer `ready` before evaluating
+     * anything else. `finished` may arrive late behind distributed storage, but it is never wrong.
+     */
+    checkReadiness(): Promise<RequestSourceStatus>;
+    /**
+     * Gets the next {@link Request} to process, or `null` if there are no more pending requests.
+     *
+     * The returned request is marked as **in progress** and remains so until it is passed to
+     * {@link IRequestLoader.markRequestAsHandled}. The caller is responsible for eventually marking
+     * every fetched request as handled; otherwise the loader never considers itself finished and the
+     * request may be re-served after a restart. See the request lifecycle contract on {@link IRequestLoader}.
+     */
+    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+    /**
+     * Can be used to iterate over the loader instance in a `for await .. of` loop.
+     * Provides an alternative for the repeated use of `fetchNextRequest`.
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Request>;
+    /**
+     * Marks a request previously returned by {@link IRequestLoader.fetchNextRequest} as handled,
+     * removing it from the set of in-progress requests.
+     *
+     * Call this once you are done with the request — whether processing succeeded or was abandoned after
+     * exhausting retries. Because a loader cannot take a request back, marking it handled is the only way to
+     * signal completion; failing to do so prevents {@link IRequestLoader.checkReadiness} from ever reporting
+     * `finished` and skews the handled and pending counts. See the request lifecycle contract on
+     * {@link IRequestLoader}.
+     */
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    /**
+     * Combines the loader with a request manager to support adding and reclaiming requests.
+     *
+     * @param requestManager Request manager to combine the loader with. If not provided, the default
+     *  {@link RequestQueue} is used.
+     */
+    toTandem?(requestManager?: IRequestManager): Promise<IRequestManager>;
+}

package/storages/request_loader.js

@@ -0,0 +1,36 @@
+/**
+ * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
+ *
+ * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
+ * nothing.
+ *
+ * @internal
+ */
+export function joinRequestSourceStatuses(a, b) {
+    if (a.status === 'ready') {
+        return a;
+    }
+    if (b.status === 'ready') {
+        return b;
+    }
+    // `ready` outranking `stalled` masks a stalled source while the other still has work. That is parity with
+    // the crawler before this was a single answer: the stall check was only reached from `isFinishedFunction`,
+    // which the task loop calls only when nothing is in flight and nothing is ready. "Fixing" the masking
+    // turns a crawl that is progressing elsewhere into a `PersistentRateLimitError`. `stalled` outranking
+    // `waiting` is the same parity - that call site fired regardless of other domains' clocks.
+    if (a.status === 'stalled') {
+        return a;
+    }
+    if (b.status === 'stalled') {
+        return b;
+    }
+    if (a.status === 'waiting') {
+        // The earlier of the two known wake-up times - unknown only if neither source announced one.
+        if (b.status !== 'waiting' || b.readyAt === undefined) {
+            return a;
+        }
+        return a.readyAt !== undefined && a.readyAt <= b.readyAt ? a : b;
+    }
+    // `a` is finished, so `b` decides.
+    return b;
+}

package/storages/request_manager.d.ts

@@ -0,0 +1,119 @@
+import type { LiteralUnion } from 'type-fest';
+import type { Request, Source } from '../request.js';
+import type { IRequestLoader } from './request_loader.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
+export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
+/**
+ * Extends the read-only {@link IRequestLoader} interface with the capability to enqueue new requests
+ * and reclaim failed ones.
+ */
+export interface IRequestManager extends IRequestLoader {
+    /**
+     * Reclaims request to the provider if its processing failed.
+     * The request will be returned by some subsequent `fetchNextRequest()` call.
+     */
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Remove all requests from the queue but keep the queue itself, resetting it
+     * so it can be reused (e.g. across multiple `crawler.run()` calls).
+     *
+     * Implementations that do not support purging may leave this `undefined`.
+     */
+    purge?(): Promise<void>;
+    /**
+     * Tells the manager how long a consumer expects to hold a request fetched via `fetchNextRequest()`
+     * before marking it handled or reclaiming it (typically the request-handler timeout plus padding).
+     *
+     * Managers backed by a storage backend that reserves requests via locking use this to avoid handing
+     * the same request out again while it is still being processed. Implementations that do not need
+     * this hint may leave it `undefined`.
+     */
+    setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
+    /**
+     * Records something said about the pace requests should go out at, so that a manager which paces its own
+     * dispatch can hold requests back.
+     *
+     * Required rather than optional, so that a wrapping manager always forwards it and a pacer nested in a
+     * composition still receives it; a manager that does not pace returns `false`.
+     *
+     * @returns `true` if anything in the composition took responsibility for the signal.
+     */
+    recordPacingSignal(signal: PacingSignal): boolean;
+    /**
+     * Extends the lock on a request previously handed out by `fetchNextRequest()` and still being
+     * processed, on storage backends that reserve requests via locking (e.g. via
+     * {@link CrawlingContext.extendTimeout|`context.extendTimeout`}).
+     *
+     * @returns `true` when the lock was prolonged, `false` when this manager does not lock
+     * requests or no longer holds this one; non-locking implementations may leave it `undefined`,
+     * which callers treat as `false`.
+     */
+    extendRequestProcessingTimeSecs?(request: Request, secs: number): Promise<boolean>;
+}
+/**
+ * How much of the URL space a {@link PacingSignal} covers.
+ *
+ * Open on purpose: `'hostname'` and `'registrableDomain'` are what Crawlee's own reporters send and what
+ * {@link ThrottlingRequestManager} understands, but any string is accepted, so a pacer keyed on something
+ * else can be reported to in its own vocabulary.
+ */
+export type PacingScope = LiteralUnion<'hostname' | 'registrableDomain', string>;
+/**
+ * Something said about the pace requests should go out at, reported to a request manager through
+ * {@link IRequestManager.recordPacingSignal}.
+ *
+ * One shape rather than a method per channel: a pacing manager switches on `reason`, a wrapping one forwards the
+ * value without knowing what is in it, and a new kind of signal costs the interface nothing. The `url` travels
+ * inside the value because the crawl-wide variant has none. Nothing here names the mechanism a signal came
+ * from - status codes, headers and robots.txt are the crawler's business - and every delay is in milliseconds.
+ *
+ * ## Scope
+ *
+ * A manager may apply a signal to a **wider** scope than it was given - a floor that holds for one host still
+ * holds when a whole site is paced by it - but never to a narrower one, which would leave some of the URLs the
+ * signal covers running unpaced. A manager that can only do the latter, or that does not recognise the scope at
+ * all, MUST throw rather than quietly under-apply it.
+ */
+export type PacingSignal = {
+    /**
+     * The source turned a request away because we were going too fast — an HTTP 429 or 503, an exhausted
+     * quota. Reactive and transient: a pacer typically backs off while refusals continue, and lets that
+     * decay once they stop.
+     */
+    reason: 'rateLimited';
+    /** The URL that was turned away. */
+    url: string;
+    /** How long the source asked us to wait before trying again, if it said. */
+    waitMs?: number;
+    /**
+     * How far this refusal reaches, if the reporter can tell — a 429 rarely says. Left out, it asks the
+     * manager to apply the signal however it happens to group requests.
+     */
+    scope?: PacingScope;
+} | {
+    /**
+     * The source declared a standing floor on how often it may be requested — a robots.txt `Crawl-delay`,
+     * a documented quota. A property of the source rather than of the run, so a pacer keeps it for the
+     * whole crawl.
+     */
+    reason: 'minInterval';
+    /** A URL of the source that declared the interval. */
+    url: string;
+    /** The declared minimum interval between two requests to the source. */
+    intervalMs: number;
+    /** Required, since whoever declares an interval knows what it applies to. */
+    scope: PacingScope;
+} | {
+    /**
+     * A standing floor under the pace of **every** domain the manager dispatches to, declared by whoever
+     * owns the crawl rather than by a source — a crawler's `sameDomainDelaySecs`. A manager that paces only
+     * some of its domains MUST throw rather than under-apply it.
+     */
+    reason: 'minIntervalEverywhere';
+    /** The declared minimum interval between two requests to any one source. */
+    intervalMs: number;
+    /** At what granularity the floor applies. */
+    scope: PacingScope;
+};

package/storages/request_manager.js

@@ -0,0 +1 @@
+export {};

package/storages/request_manager_tandem.d.ts

@@ -1,49 +1,56 @@
 import type { Dictionary } from '@crawlee/types';
 import type { Request, Source } from '../request.js';
-import type { IRequestList } from './request_list.js';
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, IRequestManager, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike } from './request_provider.js';
+import type { IRequestLoader, RequestSourceStatus } from './request_loader.js';
+import type { IRequestManager, PacingSignal, RequestsLike } from './request_manager.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
 /**
- * A request manager that combines a RequestList and a RequestQueue.
- * It first reads requests from the RequestList and then, when needed,
- * transfers them in batches to the RequestQueue.
+ * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
+ * {@link IRequestManager} (such as a `RequestQueue`).
+ * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
  */
 export declare class RequestManagerTandem implements IRequestManager {
-    private log;
-    private requestList;
-    private requestQueue;
-    constructor(requestList: IRequestList, requestQueue: IRequestManager);
+    #private;
     /**
-     * Transfers a batch of requests from the RequestList to the RequestQueue.
-     * Handles both successful transfers and failures appropriately.
-     * @private
+     * @param requestLoader The read-only loader to read requests from first.
+     * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
+     *  factory function so that the tandem can be constructed synchronously and the manager opened lazily on first use
+     *  (e.g. a lazily-opened default {@link RequestQueue}).
      */
-    private transferNextBatchToQueue;
+    constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
     /**
-     * Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
-     * is not finished, it will transfer a batch of requests from the list to the queue first.
-     * @inheritdoc
+     * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
      */
-    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+    private getRequestManager;
+    /**
+     * Transfers a single request from the read-only loader to the writable manager.
+     * If the transfer fails, the request is dropped (and logged) rather than reclaimed.
+     *
+     * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
+     *  transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
+     */
+    private transferNextRequestToQueue;
     /**
+     * Fetches the next request, transferring one from the loader first if the loader still has work.
      * @inheritdoc
      */
-    isFinished(): Promise<boolean>;
+    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
     /**
+     * The loader and the manager read as one source.
      * @inheritdoc
      */
-    isEmpty(): Promise<boolean>;
+    checkReadiness(): Promise<RequestSourceStatus>;
     /**
      * @inheritdoc
      */
-    handledCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
     /**
      * @inheritdoc
      */
-    getTotalCount(): number;
+    getTotalCount(): Promise<number>;
     /**
      * @inheritdoc
      */
-    getPendingCount(): number;
+    getPendingCount(): Promise<number>;
     /**
      * @inheritdoc
      */
@@ -51,7 +58,7 @@ export declare class RequestManagerTandem implements IRequestManager {
     /**
      * @inheritdoc
      */
-    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
     /**
      * @inheritdoc
      */
@@ -64,5 +71,24 @@ export declare class RequestManagerTandem implements IRequestManager {
      * @inheritdoc
      */
     addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
+     * The read-only loader is immutable and cannot be purged, so only the manager side is reset.
+     * @inheritdoc
+     */
+    purge(): Promise<void>;
+    /**
+     * Forwards the hint to the writable request manager — that is where requests are fetched from and
+     * reserved. The manager is opened lazily, so the value is remembered and applied once it resolves.
+     * @inheritdoc
+     */
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
+     * its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
+     * pacing.
+     * @inheritdoc
+     */
+    recordPacingSignal(signal: PacingSignal): boolean;
+    extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
 }
-//# sourceMappingURL=request_manager_tandem.d.ts.map

package/storages/request_manager_tandem.js

@@ -1,89 +1,137 @@
-import { log } from '../log.js';
+import { serviceLocator } from '../service_locator.js';
+import { joinRequestSourceStatuses } from './request_loader.js';
 /**
- * A request manager that combines a RequestList and a RequestQueue.
- * It first reads requests from the RequestList and then, when needed,
- * transfers them in batches to the RequestQueue.
+ * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
+ * {@link IRequestManager} (such as a `RequestQueue`).
+ * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
  */
 export class RequestManagerTandem {
-    log;
-    requestList;
-    requestQueue;
-    constructor(requestList, requestQueue) {
-        this.log = log.child({ prefix: 'RequestManagerTandem' });
-        this.requestList = requestList;
-        this.requestQueue = requestQueue;
+    #log;
+    #requestLoader;
+    #requestManagerPromise;
+    #resolvedRequestManager;
+    #requestManagerFactory;
+    /**
+     * The latest expected request-processing time hinted via {@link setExpectedRequestProcessingTimeSecs}.
+     * Remembered so it can be applied to the writable manager once it is lazily resolved.
+     */
+    #expectedRequestProcessingSecs;
+    /**
+     * @param requestLoader The read-only loader to read requests from first.
+     * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
+     *  factory function so that the tandem can be constructed synchronously and the manager opened lazily on first use
+     *  (e.g. a lazily-opened default {@link RequestQueue}).
+     */
+    constructor(requestLoader, requestManager) {
+        this.#log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
+        this.#requestLoader = requestLoader;
+        if (typeof requestManager === 'function') {
+            this.#requestManagerFactory = requestManager;
+        }
+        else {
+            // Nothing to open, so mark it resolved up front - synchronous pacing signals can then reach it.
+            this.#resolvedRequestManager = requestManager;
+            this.#requestManagerFactory = () => requestManager;
+        }
+    }
+    /**
+     * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
+     */
+    async getRequestManager() {
+        if (this.#resolvedRequestManager === undefined) {
+            this.#requestManagerPromise ??= Promise.resolve(this.#requestManagerFactory());
+            this.#resolvedRequestManager = await this.#requestManagerPromise;
+            // Apply any hint received before the manager was resolved.
+            if (this.#expectedRequestProcessingSecs !== undefined) {
+                await this.#resolvedRequestManager.setExpectedRequestProcessingTimeSecs?.(this.#expectedRequestProcessingSecs);
+            }
+        }
+        return this.#resolvedRequestManager;
     }
     /**
-     * Transfers a batch of requests from the RequestList to the RequestQueue.
-     * Handles both successful transfers and failures appropriately.
-     * @private
+     * Transfers a single request from the read-only loader to the writable manager.
+     * If the transfer fails, the request is dropped (and logged) rather than reclaimed.
+     *
+     * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
+     *  transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
      */
-    async transferNextBatchToQueue() {
-        const request = await this.requestList.fetchNextRequest();
+    async transferNextRequestToQueue() {
+        const request = await this.#requestLoader.fetchNextRequest();
         if (request === null) {
-            return;
+            return true;
         }
+        const requestManager = await this.getRequestManager();
         try {
-            await this.requestQueue.addRequest(request, { forefront: true });
+            await requestManager.addRequest(request, { forefront: true });
+            return true;
         }
         catch (error) {
-            // If requestQueue.addRequest() fails here then we must reclaim it back to
-            // the RequestList because probably it's not yet in the queue!
-            this.log.error('Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.', { request });
-            await this.requestList.reclaimRequest(request);
-            return;
+            this.#log.exception(error, 'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.', { url: request.url, uniqueKey: request.uniqueKey });
+            return false;
+        }
+        finally {
+            // Mark it as handled so that the request doesn't get stuck in the `inProgress` state in the loader.
+            await this.#requestLoader.markRequestAsHandled(request);
         }
-        await this.requestList.markRequestHandled(request);
     }
     /**
-     * Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
-     * is not finished, it will transfer a batch of requests from the list to the queue first.
+     * Fetches the next request, transferring one from the loader first if the loader still has work.
      * @inheritdoc
      */
     async fetchNextRequest() {
-        // First, try to transfer a request from the requestList
-        const [listEmpty, listFinished] = await Promise.all([
-            this.requestList.isEmpty(),
-            this.requestList.isFinished(),
-        ]);
-        if (!listEmpty && !listFinished) {
-            await this.transferNextBatchToQueue();
+        // Only the loader's own state decides this: a manager waiting out a backoff must not freeze the
+        // loader's unrelated requests for the length of it.
+        if ((await this.#requestLoader.checkReadiness()).status === 'ready') {
+            // If the transfer failed, the request was dropped; don't fetch from the manager this round (matching
+            // crawlee-python behaviour). The next `fetchNextRequest()` call will pick up where we left off.
+            if (!(await this.transferNextRequestToQueue())) {
+                return null;
+            }
         }
-        // Try to fetch from queue after potential transfer
-        return this.requestQueue.fetchNextRequest();
-    }
-    /**
-     * @inheritdoc
-     */
-    async isFinished() {
-        const storagesFinished = await Promise.all([this.requestList.isFinished(), this.requestQueue.isFinished()]);
-        return storagesFinished.every(Boolean);
+        // Try to fetch from manager after the transfer
+        return (await this.getRequestManager()).fetchNextRequest();
     }
     /**
+     * The loader and the manager read as one source.
      * @inheritdoc
      */
-    async isEmpty() {
-        const storagesEmpty = await Promise.all([this.requestList.isEmpty(), this.requestQueue.isEmpty()]);
-        return storagesEmpty.every(Boolean);
+    async checkReadiness() {
+        const requestManager = await this.getRequestManager();
+        const [loaderStatus, managerStatus] = await Promise.all([
+            this.#requestLoader.checkReadiness(),
+            requestManager.checkReadiness(),
+        ]);
+        return joinRequestSourceStatuses(loaderStatus, managerStatus);
     }
     /**
      * @inheritdoc
      */
-    async handledCount() {
-        // Since one of the stores needs to have priority when both are present, we query the request queue - the request list will first be dumped into the queue and then left empty.
-        return await this.requestQueue.handledCount();
+    async getHandledCount() {
+        // Since one of the stores needs to have priority when both are present, we query the request manager - the request loader will first be dumped into the manager and then left empty.
+        return (await this.getRequestManager()).getHandledCount();
     }
     /**
      * @inheritdoc
      */
-    getTotalCount() {
-        return this.requestQueue.getTotalCount();
+    async getTotalCount() {
+        const requestManager = await this.getRequestManager();
+        const [managerTotal, loaderTotal] = await Promise.all([
+            requestManager.getTotalCount(),
+            // count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager
+            this.#requestLoader.getPendingCount(),
+        ]);
+        return managerTotal + loaderTotal;
     }
     /**
      * @inheritdoc
      */
-    getPendingCount() {
-        return this.requestQueue.getPendingCount() + this.requestList.length() - this.requestList.handledCount();
+    async getPendingCount() {
+        const requestManager = await this.getRequestManager();
+        const [managerPending, loaderPending] = await Promise.all([
+            requestManager.getPendingCount(),
+            this.#requestLoader.getPendingCount(),
+        ]);
+        return managerPending + loaderPending;
     }
     /**
      * @inheritdoc
@@ -99,26 +147,54 @@ export class RequestManagerTandem {
     /**
      * @inheritdoc
      */
-    async markRequestHandled(request) {
-        return this.requestQueue.markRequestHandled(request);
+    async markRequestAsHandled(request) {
+        return (await this.getRequestManager()).markRequestAsHandled(request);
     }
     /**
      * @inheritdoc
      */
     async reclaimRequest(request, options) {
-        return await this.requestQueue.reclaimRequest(request, options);
+        return (await this.getRequestManager()).reclaimRequest(request, options);
     }
     /**
      * @inheritdoc
      */
     async addRequest(requestLike, options) {
-        return await this.requestQueue.addRequest(requestLike, options);
+        return (await this.getRequestManager()).addRequest(requestLike, options);
     }
     /**
      * @inheritdoc
      */
     async addRequestsBatched(requests, options) {
-        return await this.requestQueue.addRequestsBatched(requests, options);
+        return (await this.getRequestManager()).addRequestsBatched(requests, options);
+    }
+    /**
+     * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
+     * The read-only loader is immutable and cannot be purged, so only the manager side is reset.
+     * @inheritdoc
+     */
+    async purge() {
+        await (await this.getRequestManager()).purge?.();
+    }
+    /**
+     * Forwards the hint to the writable request manager — that is where requests are fetched from and
+     * reserved. The manager is opened lazily, so the value is remembered and applied once it resolves.
+     * @inheritdoc
+     */
+    async setExpectedRequestProcessingTimeSecs(secs) {
+        this.#expectedRequestProcessingSecs = secs;
+        await this.#resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
+    }
+    /**
+     * Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
+     * its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
+     * pacing.
+     * @inheritdoc
+     */
+    recordPacingSignal(signal) {
+        return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;
+    }
+    async extendRequestProcessingTimeSecs(request, secs) {
+        return (await this.getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;
     }
 }
-//# sourceMappingURL=request_manager_tandem.js.map