@crawlee/core 4.0.0-beta.145 → 4.0.0-beta.147

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.145",
+    "version": "4.0.0-beta.147",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.145",
-        "@crawlee/http-client": "4.0.0-beta.145",
-        "@crawlee/types": "4.0.0-beta.145",
-        "@crawlee/utils": "4.0.0-beta.145",
+        "@crawlee/fs-storage": "4.0.0-beta.147",
+        "@crawlee/http-client": "4.0.0-beta.147",
+        "@crawlee/types": "4.0.0-beta.147",
+        "@crawlee/utils": "4.0.0-beta.147",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "e8b4d50265b9e433ae61c69d16cc1fee00a4d7e9"
+    "gitHead": "bb04462b453da49b08ea0af0038cb734eabe3d01"
 }

package/storages/request_list.d.ts

@@ -3,7 +3,7 @@ import type { Dictionary } from '@crawlee/types';
 import type { Configuration } from '../configuration.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 import { Request, type RequestOptions, type Source } from '../request.js';
-import type { IRequestLoader } from './request_loader.js';
+import type { IRequestLoader, RequestLoaderStatus } from './request_loader.js';
 import type { IRequestManager } from './request_manager.js';
 /** @internal */
 export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
@@ -296,11 +296,7 @@ export declare class RequestList implements IRequestLoader {
     /**
      * @inheritDoc
      */
-    isEmpty(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    isFinished(): Promise<boolean>;
+    checkReadiness(): Promise<RequestLoaderStatus>;
     /**
      * @inheritDoc
      */

package/storages/request_list.js

@@ -381,16 +381,13 @@ export class RequestList {
     /**
      * @inheritDoc
      */
-    async isEmpty() {
+    async checkReadiness() {
         this.ensureIsInitialized();
-        return this.#requestsToRetry.length === 0 && this.#nextIndex >= this.requests.length;
-    }
-    /**
-     * @inheritDoc
-     */
-    async isFinished() {
-        this.ensureIsInitialized();
-        return this.inProgress.size === 0 && this.#nextIndex >= this.requests.length;
+        if (this.#requestsToRetry.length > 0 || this.#nextIndex < this.requests.length) {
+            return { status: 'ready' };
+        }
+        // `#requestsToRetry` is a subset of `inProgress`, so nothing in progress means nothing left to re-serve.
+        return this.inProgress.size === 0 ? { status: 'finished' } : { status: 'waiting' };
     }
     /**
      * @inheritDoc

package/storages/request_loader.d.ts

@@ -2,6 +2,44 @@ import type { Dictionary } from '@crawlee/types';
 import type { Request } from '../request.js';
 import type { IRequestManager } from './request_manager.js';
 import type { RequestQueueOperationInfo } from './request_queue.js';
+/**
+ * A request source's own availability, in a single answer.
+ *
+ * - `ready` — the next {@link IRequestLoader.fetchNextRequest} is expected to hand something over.
+ * - `waiting` — nothing to fetch right now, but the source is not done: requests are in progress, are being
+ *   added in the background, or are held back until `readyAt`.
+ * - `stalled` — the source holds requests it cannot make progress on. Only a manager that paces its own
+ *   dispatch can reach this; see {@link ThrottlingRequestManager}.
+ * - `finished` — everything has been handled.
+ */
+export type RequestSourceStatus = {
+    status: 'ready';
+} | {
+    status: 'waiting';
+    /**
+     * A `Date.now()` timestamp at which the source expects to become `ready`. Absent when the wait has
+     * no clock (an in-progress request, a background add), leaving a consumer to poll.
+     */
+    readyAt?: number;
+} | {
+    status: 'stalled';
+    reason: string;
+} | {
+    status: 'finished';
+};
+/** Loaders never stall — only a manager that paces its own dispatch can. */
+export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
+    status: 'stalled';
+}>;
+/**
+ * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
+ *
+ * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
+ * nothing.
+ *
+ * @internal
+ */
+export declare function joinRequestSourceStatuses(a: RequestSourceStatus, b: RequestSourceStatus): RequestSourceStatus;
 /**
  * An abstract interface defining a read-only stream of requests to crawl.
  *
@@ -24,7 +62,7 @@ import type { RequestQueueOperationInfo } from './request_queue.js';
  * - **Restarts and migrations:** loaders that persist their state (see {@link IRequestLoader.persistState})
  *   treat in-progress requests as interrupted and re-serve them after a restart. A request that is fetched
  *   but never marked handled will be crawled again.
- * - **Termination detection:** {@link IRequestLoader.isFinished} only resolves to `true` once nothing is
+ * - **Termination detection:** {@link IRequestLoader.checkReadiness} only reports `finished` once nothing is
  *   in progress. Leaving a request unmarked keeps the crawler running indefinitely.
  * - **Bookkeeping:** the handled and pending counts are derived from the set of in-progress requests, so
  *   skipping {@link IRequestLoader.markRequestAsHandled} corrupts {@link IRequestLoader.getHandledCount}
@@ -47,20 +85,13 @@ export interface IRequestLoader {
      */
     getHandledCount(): Promise<number>;
     /**
-     * Returns `true` if all requests were already handled and there are no more left.
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * Resolves to `true` if the next call to {@link IRequestLoader.fetchNextRequest} function
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the loader is empty, there might be some pending requests currently being processed.
+     * Reports whether the loader has a request to hand over, is waiting on one, or is done — see
+     * {@link RequestSourceStatus}.
      *
-     * This is a statement about what the *next fetch* would return, not about how much work is left, so it
-     * may report `true` while {@link IRequestLoader.getPendingCount} is non-zero - a loader that withholds
-     * requests for a while (as {@link ThrottlingRequestManager} does for a rate-limited domain) is empty
-     * for as long as it will not hand anything over. Use `isFinished()` to ask whether the work is done.
+     * A consumer's task loop is gated on this, so implementations MUST answer `ready` before evaluating
+     * anything else. `finished` may arrive late behind distributed storage, but it is never wrong.
      */
-    isEmpty(): Promise<boolean>;
+    checkReadiness(): Promise<RequestSourceStatus>;
     /**
      * Gets the next {@link Request} to process, or `null` if there are no more pending requests.
      *
@@ -81,8 +112,9 @@ export interface IRequestLoader {
      *
      * Call this once you are done with the request — whether processing succeeded or was abandoned after
      * exhausting retries. Because a loader cannot take a request back, marking it handled is the only way to
-     * signal completion; failing to do so prevents {@link IRequestLoader.isFinished} from ever resolving to
-     * `true` and skews the handled and pending counts. See the request lifecycle contract on {@link IRequestLoader}.
+     * signal completion; failing to do so prevents {@link IRequestLoader.checkReadiness} from ever reporting
+     * `finished` and skews the handled and pending counts. See the request lifecycle contract on
+     * {@link IRequestLoader}.
      */
     markRequestAsHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
     /**

package/storages/request_loader.js

@@ -1 +1,36 @@
-export {};
+/**
+ * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
+ *
+ * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
+ * nothing.
+ *
+ * @internal
+ */
+export function joinRequestSourceStatuses(a, b) {
+    if (a.status === 'ready') {
+        return a;
+    }
+    if (b.status === 'ready') {
+        return b;
+    }
+    // `ready` outranking `stalled` masks a stalled source while the other still has work. That is parity with
+    // the crawler before this was a single answer: the stall check was only reached from `isFinishedFunction`,
+    // which the task loop calls only when nothing is in flight and nothing is ready. "Fixing" the masking
+    // turns a crawl that is progressing elsewhere into a `PersistentRateLimitError`. `stalled` outranking
+    // `waiting` is the same parity - that call site fired regardless of other domains' clocks.
+    if (a.status === 'stalled') {
+        return a;
+    }
+    if (b.status === 'stalled') {
+        return b;
+    }
+    if (a.status === 'waiting') {
+        // The earlier of the two known wake-up times - unknown only if neither source announced one.
+        if (b.status !== 'waiting' || b.readyAt === undefined) {
+            return a;
+        }
+        return a.readyAt !== undefined && a.readyAt <= b.readyAt ? a : b;
+    }
+    // `a` is finished, so `b` decides.
+    return b;
+}

package/storages/request_manager.d.ts

@@ -1,3 +1,4 @@
+import type { LiteralUnion } from 'type-fest';
 import type { Request, Source } from '../request.js';
 import type { IRequestLoader } from './request_loader.js';
 import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
@@ -30,4 +31,79 @@ export interface IRequestManager extends IRequestLoader {
      * this hint may leave it `undefined`.
      */
     setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
+    /**
+     * Records something said about the pace requests should go out at, so that a manager which paces its own
+     * dispatch can hold requests back.
+     *
+     * Required rather than optional, so that a wrapping manager always forwards it and a pacer nested in a
+     * composition still receives it; a manager that does not pace returns `false`.
+     *
+     * @returns `true` if anything in the composition took responsibility for the signal.
+     */
+    recordPacingSignal(signal: PacingSignal): boolean;
 }
+/**
+ * How much of the URL space a {@link PacingSignal} covers.
+ *
+ * Open on purpose: `'hostname'` and `'registrableDomain'` are what Crawlee's own reporters send and what
+ * {@link ThrottlingRequestManager} understands, but any string is accepted, so a pacer keyed on something
+ * else can be reported to in its own vocabulary.
+ */
+export type PacingScope = LiteralUnion<'hostname' | 'registrableDomain', string>;
+/**
+ * Something said about the pace requests should go out at, reported to a request manager through
+ * {@link IRequestManager.recordPacingSignal}.
+ *
+ * One shape rather than a method per channel: a pacing manager switches on `reason`, a wrapping one forwards the
+ * value without knowing what is in it, and a new kind of signal costs the interface nothing. The `url` travels
+ * inside the value because the crawl-wide variant has none. Nothing here names the mechanism a signal came
+ * from - status codes, headers and robots.txt are the crawler's business - and every delay is in milliseconds.
+ *
+ * ## Scope
+ *
+ * A manager may apply a signal to a **wider** scope than it was given - a floor that holds for one host still
+ * holds when a whole site is paced by it - but never to a narrower one, which would leave some of the URLs the
+ * signal covers running unpaced. A manager that can only do the latter, or that does not recognise the scope at
+ * all, MUST throw rather than quietly under-apply it.
+ */
+export type PacingSignal = {
+    /**
+     * The source turned a request away because we were going too fast — an HTTP 429 or 503, an exhausted
+     * quota. Reactive and transient: a pacer typically backs off while refusals continue, and lets that
+     * decay once they stop.
+     */
+    reason: 'rateLimited';
+    /** The URL that was turned away. */
+    url: string;
+    /** How long the source asked us to wait before trying again, if it said. */
+    waitMs?: number;
+    /**
+     * How far this refusal reaches, if the reporter can tell — a 429 rarely says. Left out, it asks the
+     * manager to apply the signal however it happens to group requests.
+     */
+    scope?: PacingScope;
+} | {
+    /**
+     * The source declared a standing floor on how often it may be requested — a robots.txt `Crawl-delay`,
+     * a documented quota. A property of the source rather than of the run, so a pacer keeps it for the
+     * whole crawl.
+     */
+    reason: 'minInterval';
+    /** A URL of the source that declared the interval. */
+    url: string;
+    /** The declared minimum interval between two requests to the source. */
+    intervalMs: number;
+    /** Required, since whoever declares an interval knows what it applies to. */
+    scope: PacingScope;
+} | {
+    /**
+     * A standing floor under the pace of **every** domain the manager dispatches to, declared by whoever
+     * owns the crawl rather than by a source — a crawler's `sameDomainDelaySecs`. A manager that paces only
+     * some of its domains MUST throw rather than under-apply it.
+     */
+    reason: 'minIntervalEverywhere';
+    /** The declared minimum interval between two requests to any one source. */
+    intervalMs: number;
+    /** At what granularity the floor applies. */
+    scope: PacingScope;
+};

package/storages/request_manager_tandem.d.ts

@@ -1,7 +1,7 @@
 import type { Dictionary } from '@crawlee/types';
 import type { Request, Source } from '../request.js';
-import type { IRequestLoader } from './request_loader.js';
-import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { IRequestLoader, RequestSourceStatus } from './request_loader.js';
+import type { IRequestManager, PacingSignal, RequestsLike } from './request_manager.js';
 import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_queue.js';
 /**
  * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
@@ -32,19 +32,15 @@ export declare class RequestManagerTandem implements IRequestManager {
      */
     private transferNextRequestToQueue;
     /**
-     * Fetches the next request from the request manager. If the manager is empty and the loader
-     * is not finished, it will transfer a request from the loader to the manager first.
+     * Fetches the next request, transferring one from the loader first if the loader still has work.
      * @inheritdoc
      */
     fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
     /**
+     * The loader and the manager read as one source.
      * @inheritdoc
      */
-    isFinished(): Promise<boolean>;
-    /**
-     * @inheritdoc
-     */
-    isEmpty(): Promise<boolean>;
+    checkReadiness(): Promise<RequestSourceStatus>;
     /**
      * @inheritdoc
      */
@@ -94,4 +90,11 @@ export declare class RequestManagerTandem implements IRequestManager {
      * @inheritdoc
      */
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
+     * its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
+     * pacing.
+     * @inheritdoc
+     */
+    recordPacingSignal(signal: PacingSignal): boolean;
 }

package/storages/request_manager_tandem.js

@@ -1,4 +1,5 @@
 import { serviceLocator } from '../service_locator.js';
+import { joinRequestSourceStatuses } from './request_loader.js';
 /**
  * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
  * {@link IRequestManager} (such as a `RequestQueue`).
@@ -24,7 +25,14 @@ export class RequestManagerTandem {
     constructor(requestLoader, requestManager) {
         this.#log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
         this.#requestLoader = requestLoader;
-        this.#requestManagerFactory = typeof requestManager === 'function' ? requestManager : () => requestManager;
+        if (typeof requestManager === 'function') {
+            this.#requestManagerFactory = requestManager;
+        }
+        else {
+            // Nothing to open, so mark it resolved up front - synchronous pacing signals can then reach it.
+            this.#resolvedRequestManager = requestManager;
+            this.#requestManagerFactory = () => requestManager;
+        }
     }
     /**
      * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
@@ -69,17 +77,13 @@ export class RequestManagerTandem {
         }
     }
     /**
-     * Fetches the next request from the request manager. If the manager is empty and the loader
-     * is not finished, it will transfer a request from the loader to the manager first.
+     * Fetches the next request, transferring one from the loader first if the loader still has work.
      * @inheritdoc
      */
     async fetchNextRequest() {
-        // First, try to transfer a request from the requestList
-        const [listEmpty, listFinished] = await Promise.all([
-            this.#requestLoader.isEmpty(),
-            this.#requestLoader.isFinished(),
-        ]);
-        if (!listEmpty && !listFinished) {
+        // Only the loader's own state decides this: a manager waiting out a backoff must not freeze the
+        // loader's unrelated requests for the length of it.
+        if ((await this.#requestLoader.checkReadiness()).status === 'ready') {
             // If the transfer failed, the request was dropped; don't fetch from the manager this round (matching
             // crawlee-python behaviour). The next `fetchNextRequest()` call will pick up where we left off.
             if (!(await this.transferNextRequestToQueue())) {
@@ -90,20 +94,16 @@ export class RequestManagerTandem {
         return (await this.getRequestManager()).fetchNextRequest();
     }
     /**
+     * The loader and the manager read as one source.
      * @inheritdoc
      */
-    async isFinished() {
+    async checkReadiness() {
         const requestManager = await this.getRequestManager();
-        const storagesFinished = await Promise.all([this.#requestLoader.isFinished(), requestManager.isFinished()]);
-        return storagesFinished.every(Boolean);
-    }
-    /**
-     * @inheritdoc
-     */
-    async isEmpty() {
-        const requestManager = await this.getRequestManager();
-        const storagesEmpty = await Promise.all([this.#requestLoader.isEmpty(), requestManager.isEmpty()]);
-        return storagesEmpty.every(Boolean);
+        const [loaderStatus, managerStatus] = await Promise.all([
+            this.#requestLoader.checkReadiness(),
+            requestManager.checkReadiness(),
+        ]);
+        return joinRequestSourceStatuses(loaderStatus, managerStatus);
     }
     /**
      * @inheritdoc
@@ -194,4 +194,13 @@ export class RequestManagerTandem {
         this.#expectedRequestProcessingSecs = secs;
         await this.#resolvedRequestManager?.setExpectedRequestProcessingTimeSecs?.(secs);
     }
+    /**
+     * Forwards a pacing signal to the writable manager - the loader side is read-only and dispatches nothing of
+     * its own. Only a resolved manager is signalled; the tandem will not open a queue to answer a question about
+     * pacing.
+     * @inheritdoc
+     */
+    recordPacingSignal(signal) {
+        return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;
+    }
 }

package/storages/request_queue.d.ts

@@ -5,7 +5,8 @@ import type { IProxyConfiguration } from '../proxy_configuration.js';
 import type { Source } from '../request.js';
 import { Request } from '../request.js';
 import type { JournalEntry } from './transaction.js';
-import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { RequestLoaderStatus } from './request_loader.js';
+import type { IRequestManager, PacingSignal, RequestsLike } from './request_manager.js';
 import type { RequestQueueStats } from './storage_stats.js';
 import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
 import type { StorageOpenOptions } from './utils.js';
@@ -147,7 +148,7 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * Note that the `null` return value doesn't mean the queue processing finished,
      * it means there are currently no pending requests.
      * To check whether all requests in queue were finished,
-     * use {@link RequestQueue.isFinished} instead.
+     * use {@link RequestQueue.checkReadiness} instead.
      *
      * @returns
      *   Returns the request object or `null` if there are no more pending requests.
@@ -168,25 +169,21 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      */
     reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
     /**
-     * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest} would return
-     * `null`, i.e. there are no pending requests to fetch right now. Otherwise it resolves to `false`.
-     *
-     * Note that even if the queue is empty, there might be some requests currently being processed
-     * (fetched but not yet handled or reclaimed). An empty queue therefore does not mean crawling is
-     * finished — those in-progress requests may still be reclaimed, and background tasks may still be
-     * adding more requests. To check whether all activity in the queue has finished, use
-     * {@link RequestQueue.isFinished}.
+     * A queue hands requests out as fast as they are asked for; pacing is a job for a manager wrapped around it,
+     * such as {@link ThrottlingRequestManager}.
+     * @inheritdoc
      */
-    isEmpty(): Promise<boolean>;
+    recordPacingSignal(_signal: PacingSignal): boolean;
     /**
-     * Resolves to `true` if all requests were already handled and there are no more left — including no
-     * requests currently in progress (fetched but not yet handled or reclaimed, including requests
-     * locked by other clients sharing the same queue) and no background add operations still in flight.
+     * Reports whether the queue has a request to hand over, is waiting on one, or is done.
+     *
+     * `waiting` means requests are in progress (fetched but not yet handled or reclaimed, possibly by another
+     * client sharing the queue) or a background add is still landing; neither has a clock, so no `readyAt`.
      *
-     * Due to the nature of distributed storage used by the queue, the function may occasionally return
-     * a false negative, but it shall never return a false positive.
+     * Due to the nature of distributed storage used by the queue, `finished` may occasionally arrive a probe or
+     * two late, but it is never reported early.
      */
-    isFinished(): Promise<boolean>;
+    checkReadiness(): Promise<RequestLoaderStatus>;
     /**
      * Tells the queue how long a consumer expects to hold a fetched request before marking it handled
      * or reclaiming it (typically the request-handler timeout plus padding), so that a storage backend

package/storages/request_queue.js

@@ -553,7 +553,7 @@ export class RequestQueue {
      * Note that the `null` return value doesn't mean the queue processing finished,
      * it means there are currently no pending requests.
      * To check whether all requests in queue were finished,
-     * use {@link RequestQueue.isFinished} instead.
+     * use {@link RequestQueue.checkReadiness} instead.
      *
      * @returns
      *   Returns the request object or `null` if there are no more pending requests.
@@ -625,42 +625,37 @@ export class RequestQueue {
         return queueOperationInfo;
     }
     /**
-     * Resolves to `true` if the next call to {@link RequestQueue.fetchNextRequest} would return
-     * `null`, i.e. there are no pending requests to fetch right now. Otherwise it resolves to `false`.
-     *
-     * Note that even if the queue is empty, there might be some requests currently being processed
-     * (fetched but not yet handled or reclaimed). An empty queue therefore does not mean crawling is
-     * finished — those in-progress requests may still be reclaimed, and background tasks may still be
-     * adding more requests. To check whether all activity in the queue has finished, use
-     * {@link RequestQueue.isFinished}.
+     * A queue hands requests out as fast as they are asked for; pacing is a job for a manager wrapped around it,
+     * such as {@link ThrottlingRequestManager}.
+     * @inheritdoc
      */
-    async isEmpty() {
-        const transaction = activeStorageTransaction();
-        // Requests buffered by the active transaction count as pending from its point of view.
-        if (transaction && this.bufferedRequests(transaction).size > 0) {
-            return false;
-        }
-        return this.backend.isEmpty();
+    recordPacingSignal(_signal) {
+        return false;
     }
     /**
-     * Resolves to `true` if all requests were already handled and there are no more left — including no
-     * requests currently in progress (fetched but not yet handled or reclaimed, including requests
-     * locked by other clients sharing the same queue) and no background add operations still in flight.
+     * Reports whether the queue has a request to hand over, is waiting on one, or is done.
+     *
+     * `waiting` means requests are in progress (fetched but not yet handled or reclaimed, possibly by another
+     * client sharing the queue) or a background add is still landing; neither has a clock, so no `readyAt`.
      *
-     * Due to the nature of distributed storage used by the queue, the function may occasionally return
-     * a false negative, but it shall never return a false positive.
+     * Due to the nature of distributed storage used by the queue, `finished` may occasionally arrive a probe or
+     * two late, but it is never reported early.
      */
-    async isFinished() {
+    async checkReadiness() {
         const transaction = activeStorageTransaction();
-        // We are not finished if we're still adding new requests in the background.
-        if (this.#inProgressRequestBatchCount > 0) {
-            return false;
-        }
         // Requests buffered by the active transaction count as pending from its point of view.
         if (transaction && this.bufferedRequests(transaction).size > 0) {
-            return false;
+            return { status: 'ready' };
+        }
+        // Something fetchable outranks everything below, so this is the only backend call a probe needs.
+        if (!(await this.backend.isEmpty())) {
+            return { status: 'ready' };
+        }
+        // We are not finished if we're still adding new requests in the background.
+        if (this.#inProgressRequestBatchCount > 0) {
+            return { status: 'waiting' };
         }
-        return this.backend.isFinished();
+        return (await this.backend.isFinished()) ? { status: 'finished' } : { status: 'waiting' };
     }
     /**
      * Tells the queue how long a consumer expects to hold a fetched request before marking it handled

package/storages/sitemap_request_loader.d.ts

@@ -2,7 +2,7 @@ import type { BaseHttpClient } from '@crawlee/http-client';
 import { EnqueueStrategy, type ParseSitemapOptions } from '@crawlee/utils';
 import type { UrlPatternInput } from '../enqueue_links/shared.js';
 import { Request } from '../request.js';
-import type { IRequestLoader } from './request_loader.js';
+import type { IRequestLoader, RequestLoaderStatus } from './request_loader.js';
 import type { IRequestManager } from './request_manager.js';
 interface UrlConstraints {
     /**
@@ -163,11 +163,7 @@ export declare class SitemapRequestLoader implements IRequestLoader {
     /**
      * @inheritDoc
      */
-    isFinished(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    isEmpty(): Promise<boolean>;
+    checkReadiness(): Promise<RequestLoaderStatus>;
     /**
      * @inheritDoc
      */

package/storages/sitemap_request_loader.js

@@ -72,7 +72,7 @@ export class SitemapRequestLoader {
      * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
      * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
      *
-     * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
+     * If the loading is aborted and all the requests are handled, `checkReadiness()` will report `finished`.
      */
     #abortLoading = false;
     /** Number of URLs that were marked as handled */
@@ -300,14 +300,15 @@ export class SitemapRequestLoader {
     /**
      * @inheritDoc
      */
-    async isFinished() {
-        return ((await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.#abortLoading));
-    }
-    /**
-     * @inheritDoc
-     */
-    async isEmpty() {
-        return this.#urlQueueStream.readableLength === 0;
+    async checkReadiness() {
+        if (this.#urlQueueStream.readableLength > 0) {
+            return { status: 'ready' };
+        }
+        // The parser is still running, so more URLs may yet arrive - on no schedule, hence no `readyAt`.
+        if (!this.isSitemapFullyLoaded() && !this.#abortLoading) {
+            return { status: 'waiting' };
+        }
+        return this.inProgress.size === 0 ? { status: 'finished' } : { status: 'waiting' };
     }
     /**
      * @inheritDoc
@@ -402,7 +403,7 @@ export class SitemapRequestLoader {
      * @inheritDoc
      */
     async *[Symbol.asyncIterator]() {
-        while (!(await this.isFinished())) {
+        while ((await this.checkReadiness()).status !== 'finished') {
             const request = await this.fetchNextRequest();
             if (!request)
                 break;