@crawlee/core 4.0.0-beta.63 → 4.0.0-beta.65

package/storages/request_loader.d.ts

@@ -0,0 +1,97 @@
+import type { Dictionary } from '@crawlee/types';
+import type { Request } from '../request.js';
+import type { IRequestManager } from './request_manager.js';
+import type { RequestQueueOperationInfo } from './request_provider.js';
+/**
+ * An abstract interface defining a read-only stream of requests to crawl.
+ *
+ * Request loaders are used to manage and provide access to a storage of crawling requests.
+ *
+ * Key responsibilities:
+ * - Fetching the next request to be processed.
+ * - Marking requests as handled once they are no longer in progress.
+ * - Managing state information such as the total and handled request counts.
+ *
+ * ## Request lifecycle contract
+ *
+ * Every request returned by {@link IRequestLoader.fetchNextRequest} is considered **in progress**
+ * until it is passed to {@link IRequestLoader.markRequestHandled}. Once you fetch a request, you are
+ * obligated to eventually mark it as handled — there is no way to hand a request back to a loader
+ * (only an {@link IRequestManager} can reclaim requests for a retry). "Handled" therefore means
+ * "finished with this request", whether processing succeeded or was abandoned after exhausting retries.
+ *
+ * Honoring this contract matters for three reasons:
+ * - **Restarts and migrations:** loaders that persist their state (see {@link IRequestLoader.persistState})
+ *   treat in-progress requests as interrupted and re-serve them after a restart. A request that is fetched
+ *   but never marked handled will be crawled again.
+ * - **Termination detection:** {@link IRequestLoader.isFinished} only resolves to `true` once nothing is
+ *   in progress. Leaving a request unmarked keeps the crawler running indefinitely.
+ * - **Bookkeeping:** the handled and pending counts are derived from the set of in-progress requests, so
+ *   skipping {@link IRequestLoader.markRequestHandled} corrupts {@link IRequestLoader.getHandledCount}
+ *   and {@link IRequestLoader.getPendingCount}.
+ *
+ * Concrete implementations such as {@link RequestList} or {@link SitemapRequestLoader} build on this interface.
+ * The {@link IRequestManager} interface extends it with the capability to enqueue and reclaim requests.
+ */
+export interface IRequestLoader {
+    /**
+     * Returns an approximation of the total number of requests in the loader (i.e. pending + handled).
+     */
+    getTotalCount(): Promise<number>;
+    /**
+     * Returns an approximation of the number of pending requests in the loader.
+     */
+    getPendingCount(): Promise<number>;
+    /**
+     * Returns the number of requests in the loader that have been handled.
+     */
+    getHandledCount(): Promise<number>;
+    /**
+     * Returns `true` if all requests were already handled and there are no more left.
+     */
+    isFinished(): Promise<boolean>;
+    /**
+     * Resolves to `true` if the next call to {@link IRequestLoader.fetchNextRequest} function
+     * would return `null`, otherwise it resolves to `false`.
+     * Note that even if the loader is empty, there might be some pending requests currently being processed.
+     */
+    isEmpty(): Promise<boolean>;
+    /**
+     * Gets the next {@link Request} to process, or `null` if there are no more pending requests.
+     *
+     * The returned request is marked as **in progress** and remains so until it is passed to
+     * {@link IRequestLoader.markRequestHandled}. The caller is responsible for eventually marking
+     * every fetched request as handled; otherwise the loader never considers itself finished and the
+     * request may be re-served after a restart. See the request lifecycle contract on {@link IRequestLoader}.
+     */
+    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
+    /**
+     * Can be used to iterate over the loader instance in a `for await .. of` loop.
+     * Provides an alternative for the repeated use of `fetchNextRequest`.
+     */
+    [Symbol.asyncIterator](): AsyncGenerator<Request>;
+    /**
+     * Marks a request previously returned by {@link IRequestLoader.fetchNextRequest} as handled,
+     * removing it from the set of in-progress requests.
+     *
+     * Call this once you are done with the request — whether processing succeeded or was abandoned after
+     * exhausting retries. Because a loader cannot take a request back, marking it handled is the only way to
+     * signal completion; failing to do so prevents {@link IRequestLoader.isFinished} from ever resolving to
+     * `true` and skews the handled and pending counts. See the request lifecycle contract on {@link IRequestLoader}.
+     */
+    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
+    /**
+     * Persists the current state of the loader into the default {@link KeyValueStore}.
+     *
+     * Not all loaders support persistence; implementations that do not should leave this `undefined`.
+     */
+    persistState?(): Promise<void>;
+    /**
+     * Combines the loader with a request manager to support adding and reclaiming requests.
+     *
+     * @param requestManager Request manager to combine the loader with. If not provided, the default
+     *  {@link RequestQueue} is used.
+     */
+    toTandem?(requestManager?: IRequestManager): Promise<IRequestManager>;
+}
+//# sourceMappingURL=request_loader.d.ts.map

package/storages/request_loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request_loader.d.ts","sourceRoot":"","sources":["../../src/storages/request_loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAEvE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjC;;OAEG;IACH,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnC;;OAEG;IACH,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnC;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;;;;;;OAOG;IACH,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAElF;;;OAGG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAElD;;;;;;;;OAQG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAEvF;;;;OAIG;IACH,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B;;;;;OAKG;IACH,QAAQ,CAAC,CAAC,cAAc,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACzE"}

package/storages/request_loader.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=request_loader.js.map

package/storages/request_loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request_loader.js","sourceRoot":"","sources":["../../src/storages/request_loader.ts"],"names":[],"mappings":""}

package/storages/request_manager.d.ts

@@ -0,0 +1,25 @@
+import type { Request, Source } from '../request.js';
+import type { IRequestLoader } from './request_loader.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_provider.js';
+export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
+/**
+ * Extends the read-only {@link IRequestLoader} interface with the capability to enqueue new requests
+ * and reclaim failed ones.
+ */
+export interface IRequestManager extends IRequestLoader {
+    /**
+     * Reclaims request to the provider if its processing failed.
+     * The request will be returned by some subsequent `fetchNextRequest()` call.
+     */
+    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
+    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
+    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Remove all requests from the queue but keep the queue itself, resetting it
+     * so it can be reused (e.g. across multiple `crawler.run()` calls).
+     *
+     * Implementations that do not support purging may leave this `undefined`.
+     */
+    purge?(): Promise<void>;
+}
+//# sourceMappingURL=request_manager.d.ts.map

package/storages/request_manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request_manager.d.ts","sourceRoot":"","sources":["../../src/storages/request_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,yBAAyB,EACzB,4BAA4B,EAC/B,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE5G;;;GAGG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc;IACnD;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC;IAEpH,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE5G,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;IAEnH;;;;;OAKG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B"}

package/storages/request_manager.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=request_manager.js.map

package/storages/request_manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"request_manager.js","sourceRoot":"","sources":["../../src/storages/request_manager.ts"],"names":[],"mappings":""}

package/storages/request_manager_tandem.d.ts

@@ -1,26 +1,43 @@
 import type { Dictionary } from '@crawlee/types';
 import type { Request, Source } from '../request.js';
-import type { IRequestList } from './request_list.js';
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, IRequestManager, RequestQueueOperationInfo, RequestQueueOperationOptions, RequestsLike } from './request_provider.js';
+import type { IRequestLoader } from './request_loader.js';
+import type { IRequestManager, RequestsLike } from './request_manager.js';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request_provider.js';
 /**
- * A request manager that combines a RequestList and a RequestQueue.
- * It first reads requests from the RequestList and then, when needed,
- * transfers them in batches to the RequestQueue.
+ * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
+ * {@link IRequestManager} (such as a `RequestQueue`).
+ * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
  */
 export declare class RequestManagerTandem implements IRequestManager {
     private log;
-    private requestList;
-    private requestQueue;
-    constructor(requestList: IRequestList, requestQueue: IRequestManager);
+    private requestLoader;
+    private requestManagerPromise?;
+    private resolvedRequestManager?;
+    private requestManagerFactory;
     /**
-     * Transfers a batch of requests from the RequestList to the RequestQueue.
-     * Handles both successful transfers and failures appropriately.
+     * @param requestLoader The read-only loader to read requests from first.
+     * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
+     *  factory function so that the tandem can be constructed synchronously and the manager opened lazily on first use
+     *  (e.g. a lazily-opened default {@link RequestQueue}).
+     */
+    constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
+    /**
+     * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
+     * @private
+     */
+    private getRequestManager;
+    /**
+     * Transfers a single request from the read-only loader to the writable manager.
+     * If the transfer fails, the request is dropped (and logged) rather than reclaimed.
+     *
+     * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
+     *  transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
      * @private
      */
-    private transferNextBatchToQueue;
+    private transferNextRequestToQueue;
     /**
-     * Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
-     * is not finished, it will transfer a batch of requests from the list to the queue first.
+     * Fetches the next request from the request manager. If the manager is empty and the loader
+     * is not finished, it will transfer a request from the loader to the manager first.
      * @inheritdoc
      */
     fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
@@ -35,15 +52,15 @@ export declare class RequestManagerTandem implements IRequestManager {
     /**
      * @inheritdoc
      */
-    handledCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
     /**
      * @inheritdoc
      */
-    getTotalCount(): number;
+    getTotalCount(): Promise<number>;
     /**
      * @inheritdoc
      */
-    getPendingCount(): number;
+    getPendingCount(): Promise<number>;
     /**
      * @inheritdoc
      */
@@ -64,5 +81,16 @@ export declare class RequestManagerTandem implements IRequestManager {
      * @inheritdoc
      */
     addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+    /**
+     * Persists the state of the underlying read-only loader, if it supports persistence.
+     * @inheritdoc
+     */
+    persistState(): Promise<void>;
+    /**
+     * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
+     * The read-only loader is immutable and cannot be purged, so only the manager side is reset.
+     * @inheritdoc
+     */
+    purge(): Promise<void>;
 }
 //# sourceMappingURL=request_manager_tandem.d.ts.map

package/storages/request_manager_tandem.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"request_manager_tandem.d.ts","sourceRoot":"","sources":["../../src/storages/request_manager_tandem.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,eAAe,EACf,yBAAyB,EACzB,4BAA4B,EAC5B,YAAY,EACf,MAAM,uBAAuB,CAAC;AAE/B;;;;GAIG;AACH,qBAAa,oBAAqB,YAAW,eAAe;IACxD,OAAO,CAAC,GAAG,CAAgB;IAC3B,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,YAAY,CAAkB;gBAE1B,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe;IAMpE;;;;OAIG;YACW,wBAAwB;IAuBtC;;;;OAIG;IACG,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAevF;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAKpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAKrC;;OAEG;IACH,aAAa,IAAI,MAAM;IAIvB;;OAEG;IACH,eAAe,IAAI,MAAM;IAIzB;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC;IAI5F;;OAEG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,CAAC,EAAE,4BAA4B,GACvC,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAI5C;;OAEG;IACG,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAIjH;;OAEG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,EACtB,OAAO,CAAC,EAAE,yBAAyB,GACpC,OAAO,CAAC,wBAAwB,CAAC;CAGvC"}
+{"version":3,"file":"request_manager_tandem.d.ts","sourceRoot":"","sources":["../../src/storages/request_manager_tandem.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAErD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,KAAK,EACR,yBAAyB,EACzB,wBAAwB,EACxB,yBAAyB,EACzB,4BAA4B,EAC/B,MAAM,uBAAuB,CAAC;AAE/B;;;;GAIG;AACH,qBAAa,oBAAqB,YAAW,eAAe;IACxD,OAAO,CAAC,GAAG,CAAgB;IAC3B,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,qBAAqB,CAAC,CAA2B;IACzD,OAAO,CAAC,sBAAsB,CAAC,CAAkB;IAEjD,OAAO,CAAC,qBAAqB,CAAmD;IAEhF;;;;;OAKG;gBAEC,aAAa,EAAE,cAAc,EAC7B,cAAc,EAAE,eAAe,GAAG,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAOxF;;;OAGG;YACW,iBAAiB;IAQ/B;;;;;;;OAOG;YACW,0BAA0B;IAyBxC;;;;OAIG;IACG,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAmBvF;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAMpC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAMjC;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAKxC;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;IAStC;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IASxC;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;OAEG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC;IAI5F;;OAEG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,CAAC,EAAE,4BAA4B,GACvC,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAI5C;;OAEG;IACG,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAIjH;;OAEG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,EACtB,OAAO,CAAC,EAAE,yBAAyB,GACpC,OAAO,CAAC,wBAAwB,CAAC;IAIpC;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAInC;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/storages/request_manager_tandem.js

@@ -1,89 +1,129 @@
 import { serviceLocator } from '../service_locator.js';
 /**
- * A request manager that combines a RequestList and a RequestQueue.
- * It first reads requests from the RequestList and then, when needed,
- * transfers them in batches to the RequestQueue.
+ * A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
+ * {@link IRequestManager} (such as a `RequestQueue`).
+ * It first reads requests from the loader and then, when needed, transfers them in batches to the manager.
  */
 export class RequestManagerTandem {
     log;
-    requestList;
-    requestQueue;
-    constructor(requestList, requestQueue) {
+    requestLoader;
+    requestManagerPromise;
+    resolvedRequestManager;
+    requestManagerFactory;
+    /**
+     * @param requestLoader The read-only loader to read requests from first.
+     * @param requestManager The writable manager to transfer requests into and enqueue new ones. May be passed as a
+     *  factory function so that the tandem can be constructed synchronously and the manager opened lazily on first use
+     *  (e.g. a lazily-opened default {@link RequestQueue}).
+     */
+    constructor(requestLoader, requestManager) {
         this.log = serviceLocator.getLogger().child({ prefix: 'RequestManagerTandem' });
-        this.requestList = requestList;
-        this.requestQueue = requestQueue;
+        this.requestLoader = requestLoader;
+        this.requestManagerFactory = typeof requestManager === 'function' ? requestManager : () => requestManager;
     }
     /**
-     * Transfers a batch of requests from the RequestList to the RequestQueue.
-     * Handles both successful transfers and failures appropriately.
+     * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
      * @private
      */
-    async transferNextBatchToQueue() {
-        const request = await this.requestList.fetchNextRequest();
+    async getRequestManager() {
+        if (this.resolvedRequestManager === undefined) {
+            this.requestManagerPromise ??= Promise.resolve(this.requestManagerFactory());
+            this.resolvedRequestManager = await this.requestManagerPromise;
+        }
+        return this.resolvedRequestManager;
+    }
+    /**
+     * Transfers a single request from the read-only loader to the writable manager.
+     * If the transfer fails, the request is dropped (and logged) rather than reclaimed.
+     *
+     * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
+     *  transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
+     * @private
+     */
+    async transferNextRequestToQueue() {
+        const request = await this.requestLoader.fetchNextRequest();
         if (request === null) {
-            return;
+            return true;
         }
+        const requestManager = await this.getRequestManager();
         try {
-            await this.requestQueue.addRequest(request, { forefront: true });
+            await requestManager.addRequest(request, { forefront: true });
+            return true;
         }
         catch (error) {
-            // If requestQueue.addRequest() fails here then we must reclaim it back to
-            // the RequestList because probably it's not yet in the queue!
-            this.log.error('Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.', { request });
-            await this.requestList.reclaimRequest(request);
-            return;
+            this.log.exception(error, 'Adding request from the RequestLoader to the RequestManager failed, the request has been dropped.', { url: request.url, uniqueKey: request.uniqueKey });
+            return false;
+        }
+        finally {
+            // Mark it as handled so that the request doesn't get stuck in the `inProgress` state in the loader.
+            await this.requestLoader.markRequestHandled(request);
         }
-        await this.requestList.markRequestHandled(request);
     }
     /**
-     * Fetches the next request from the RequestQueue. If the queue is empty and the RequestList
-     * is not finished, it will transfer a batch of requests from the list to the queue first.
+     * Fetches the next request from the request manager. If the manager is empty and the loader
+     * is not finished, it will transfer a request from the loader to the manager first.
      * @inheritdoc
      */
     async fetchNextRequest() {
         // First, try to transfer a request from the requestList
         const [listEmpty, listFinished] = await Promise.all([
-            this.requestList.isEmpty(),
-            this.requestList.isFinished(),
+            this.requestLoader.isEmpty(),
+            this.requestLoader.isFinished(),
         ]);
         if (!listEmpty && !listFinished) {
-            await this.transferNextBatchToQueue();
+            // If the transfer failed, the request was dropped; don't fetch from the manager this round (matching
+            // crawlee-python behaviour). The next `fetchNextRequest()` call will pick up where we left off.
+            if (!(await this.transferNextRequestToQueue())) {
+                return null;
+            }
         }
-        // Try to fetch from queue after potential transfer
-        return this.requestQueue.fetchNextRequest();
+        // Try to fetch from manager after the transfer
+        return (await this.getRequestManager()).fetchNextRequest();
     }
     /**
      * @inheritdoc
      */
     async isFinished() {
-        const storagesFinished = await Promise.all([this.requestList.isFinished(), this.requestQueue.isFinished()]);
+        const requestManager = await this.getRequestManager();
+        const storagesFinished = await Promise.all([this.requestLoader.isFinished(), requestManager.isFinished()]);
         return storagesFinished.every(Boolean);
     }
     /**
      * @inheritdoc
      */
     async isEmpty() {
-        const storagesEmpty = await Promise.all([this.requestList.isEmpty(), this.requestQueue.isEmpty()]);
+        const requestManager = await this.getRequestManager();
+        const storagesEmpty = await Promise.all([this.requestLoader.isEmpty(), requestManager.isEmpty()]);
         return storagesEmpty.every(Boolean);
     }
     /**
      * @inheritdoc
      */
-    async handledCount() {
-        // Since one of the stores needs to have priority when both are present, we query the request queue - the request list will first be dumped into the queue and then left empty.
-        return await this.requestQueue.handledCount();
+    async getHandledCount() {
+        // Since one of the stores needs to have priority when both are present, we query the request manager - the request loader will first be dumped into the manager and then left empty.
+        return (await this.getRequestManager()).getHandledCount();
     }
     /**
      * @inheritdoc
      */
-    getTotalCount() {
-        return this.requestQueue.getTotalCount();
+    async getTotalCount() {
+        const requestManager = await this.getRequestManager();
+        const [managerTotal, loaderTotal] = await Promise.all([
+            requestManager.getTotalCount(),
+            this.requestLoader.getTotalCount(),
+        ]);
+        return managerTotal + loaderTotal;
     }
     /**
      * @inheritdoc
      */
-    getPendingCount() {
-        return this.requestQueue.getPendingCount() + this.requestList.length() - this.requestList.handledCount();
+    async getPendingCount() {
+        const requestManager = await this.getRequestManager();
+        const [managerPending, loaderPending] = await Promise.all([
+            requestManager.getPendingCount(),
+            this.requestLoader.getPendingCount(),
+        ]);
+        return managerPending + loaderPending;
     }
     /**
      * @inheritdoc
@@ -100,25 +140,40 @@ export class RequestManagerTandem {
      * @inheritdoc
      */
     async markRequestHandled(request) {
-        return this.requestQueue.markRequestHandled(request);
+        return (await this.getRequestManager()).markRequestHandled(request);
     }
     /**
      * @inheritdoc
      */
     async reclaimRequest(request, options) {
-        return await this.requestQueue.reclaimRequest(request, options);
+        return (await this.getRequestManager()).reclaimRequest(request, options);
     }
     /**
      * @inheritdoc
      */
     async addRequest(requestLike, options) {
-        return await this.requestQueue.addRequest(requestLike, options);
+        return (await this.getRequestManager()).addRequest(requestLike, options);
     }
     /**
      * @inheritdoc
      */
     async addRequestsBatched(requests, options) {
-        return await this.requestQueue.addRequestsBatched(requests, options);
+        return (await this.getRequestManager()).addRequestsBatched(requests, options);
+    }
+    /**
+     * Persists the state of the underlying read-only loader, if it supports persistence.
+     * @inheritdoc
+     */
+    async persistState() {
+        await this.requestLoader.persistState?.();
+    }
+    /**
+     * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
+     * The read-only loader is immutable and cannot be purged, so only the manager side is reset.
+     * @inheritdoc
+     */
+    async purge() {
+        await (await this.getRequestManager()).purge?.();
     }
 }
 //# sourceMappingURL=request_manager_tandem.js.map

package/storages/request_manager_tandem.js.map

@@ -1 +1 @@
-{"version":3,"file":"request_manager_tandem.js","sourceRoot":"","sources":["../../src/storages/request_manager_tandem.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAWvD;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IACrB,GAAG,CAAgB;IACnB,WAAW,CAAe;IAC1B,YAAY,CAAkB;IAEtC,YAAY,WAAyB,EAAE,YAA6B;QAChE,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACrC,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,wBAAwB;QAClC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,CAAC;QAE1D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,0EAA0E;YAC1E,8DAA8D;YAC9D,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,yGAAyG,EACzG,EAAE,OAAO,EAAE,CACd,CAAC;YACF,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC/C,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QAClB,wDAAwD;QACxD,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChD,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE;YAC1B,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAC1C,CAAC;QAED,mDAAmD;QACnD,OAAO,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAK,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC5G,OAAO,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnG,OAAO,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,+KAA+K;QAC/K,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,aAAa;QACT,OAAO,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,eAAe;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;IAC7G,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,OAAO,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,OAAsC;QAEtC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,OAAsC;QACxE,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAAsB,EACtB,OAAmC;QAEnC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;CACJ"}
+{"version":3,"file":"request_manager_tandem.js","sourceRoot":"","sources":["../../src/storages/request_manager_tandem.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAUvD;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IACrB,GAAG,CAAgB;IACnB,aAAa,CAAiB;IAC9B,qBAAqB,CAA4B;IACjD,sBAAsB,CAAmB;IAEzC,qBAAqB,CAAmD;IAEhF;;;;;OAKG;IACH,YACI,aAA6B,EAC7B,cAAoF;QAEpF,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,qBAAqB,GAAG,OAAO,cAAc,KAAK,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,cAAc,CAAC;IAC9G,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB;QAC3B,IAAI,IAAI,CAAC,sBAAsB,KAAK,SAAS,EAAE,CAAC;YAC5C,IAAI,CAAC,qBAAqB,KAAK,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAC;YAC7E,IAAI,CAAC,sBAAsB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;QACnE,CAAC;QACD,OAAO,IAAI,CAAC,sBAAsB,CAAC;IACvC,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,0BAA0B;QACpC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE,CAAC;QAE5D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEtD,IAAI,CAAC;YACD,MAAM,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9D,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,SAAS,CACd,KAAc,EACd,mGAAmG,EACnG,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CACrD,CAAC;YACF,OAAO,KAAK,CAAC;QACjB,CAAC;gBAAS,CAAC;YACP,oGAAoG;YACpG,MAAM,IAAI,CAAC,aAAa,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QAClB,wDAAwD;QACxD,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChD,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;YAC5B,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,qGAAqG;YACrG,gGAAgG;YAChG,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,0BAA0B,EAAE,CAAC,EAAE,CAAC;gBAC7C,OAAO,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;QAED,+CAA+C;QAC/C,OAAO,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,gBAAgB,EAAK,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC3G,OAAO,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACT,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClG,OAAO,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACjB,qLAAqL;QACrL,OAAO,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,eAAe,EAAE,CAAC;IAC9D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtD,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClD,cAAc,CAAC,aAAa,EAAE;YAC9B,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE;SACrC,CAAC,CAAC;QACH,OAAO,YAAY,GAAG,WAAW,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACjB,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtD,MAAM,CAAC,cAAc,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACtD,cAAc,CAAC,eAAe,EAAE;YAChC,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE;SACvC,CAAC,CAAC;QACH,OAAO,cAAc,GAAG,aAAa,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACzB,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,IAAI,CAAC,GAAG;gBAAE,MAAM;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,OAAO,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAChB,OAAgB,EAChB,OAAsC;QAEtC,OAAO,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,OAAsC;QACxE,OAAO,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CACpB,QAAsB,EACtB,OAAmC;QAEnC,OAAO,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClF,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QACd,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,EAAE,CAAC;IAC9C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,CAAC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;IACrD,CAAC;CACJ"}

package/storages/request_provider.d.ts

@@ -7,66 +7,9 @@ import type { CrawleeLogger } from '../log.js';
 import type { ProxyConfiguration } from '../proxy_configuration.js';
 import type { InternalSource, RequestOptions, Source } from '../request.js';
 import { Request } from '../request.js';
+import type { IRequestManager, RequestsLike } from './request_manager.js';
 import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
 import type { StorageOpenOptions } from './utils.js';
-export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
-/**
- * Represents a provider of requests/URLs to crawl.
- */
-export interface IRequestManager {
-    /**
-     * Returns `true` if all requests were already handled and there are no more left.
-     */
-    isFinished(): Promise<boolean>;
-    /**
-     * Resolves to `true` if the next call to {@link IRequestManager.fetchNextRequest} function
-     * would return `null`, otherwise it resolves to `false`.
-     * Note that even if the provider is empty, there might be some pending requests currently being processed.
-     */
-    isEmpty(): Promise<boolean>;
-    /**
-     * Returns number of handled requests.
-     */
-    handledCount(): Promise<number>;
-    /**
-     * Get the total number of requests known to the request manager.
-     */
-    getTotalCount(): number;
-    /**
-     * Get an offline approximation of the number of pending requests.
-     */
-    getPendingCount(): number;
-    /**
-     * Gets the next {@link Request} to process.
-     *
-     * The function's `Promise` resolves to `null` if there are no more
-     * requests to process.
-     */
-    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request<T> | null>;
-    /**
-     * Can be used to iterate over the `RequestManager` instance in a `for await .. of` loop.
-     * Provides an alternative for the repeated use of `fetchNextRequest`.
-     */
-    [Symbol.asyncIterator](): AsyncGenerator<Request>;
-    /**
-     * Marks request as handled after successful processing.
-     */
-    markRequestHandled(request: Request): Promise<RequestQueueOperationInfo | void | null>;
-    /**
-     * Reclaims request to the provider if its processing failed.
-     * The request will become available in the next `fetchNextRequest()`.
-     */
-    reclaimRequest(request: Request, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
-    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
-    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
-    /**
-     * Remove all requests from the queue but keep the queue itself, resetting it
-     * so it can be reused (e.g. across multiple `crawler.run()` calls).
-     *
-     * Implementations that do not support purging may leave this `undefined`.
-     */
-    purge?(): Promise<void>;
-}
 export declare abstract class RequestProvider implements IStorage, IRequestManager {
     protected readonly config: Configuration;
     id: string;
@@ -80,8 +23,6 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
     requestLockSecs: number;
     assumedTotalCount: number;
     assumedHandledCount: number;
-    private initialCount;
-    private initialHandledCount;
     private isInitialized;
     protected queueHeadIds: ListDictionary<string>;
     protected requestCache: LruCache<RequestLruItem>;
@@ -94,17 +35,17 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
     protected readonly events: EventManager;
     constructor(options: InternalRequestProviderOptions, config?: Configuration);
     /**
-     * Returns an offline approximation of the total number of requests in the queue (i.e. pending + handled).
+     * Returns the total number of requests in the queue (i.e. pending + handled).
      *
      * Survives restarts and actor migrations.
      */
-    getTotalCount(): number;
+    getTotalCount(): Promise<number>;
     /**
-     * Returns an offline approximation of the total number of pending requests in the queue.
+     * Returns the total number of pending requests in the queue.
      *
      * Survives restarts and Actor migrations.
      */
-    getPendingCount(): number;
+    getPendingCount(): Promise<number>;
     /**
      * Adds a request to the queue.
      *
@@ -232,7 +173,7 @@ export declare abstract class RequestProvider implements IStorage, IRequestManag
      * ```
      * @inheritdoc
      */
-    handledCount(): Promise<number>;
+    getHandledCount(): Promise<number>;
     /**
      * Returns an object containing general information about the request queue.
      *

package/storages/request_provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,cAAc,EACd,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EACnB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGjE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAIrD,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AAE5G;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhC;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,IAAI,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAElF;;;OAGG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;IAElD;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAEvF;;;OAGG;IACH,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC;IAEpH,UAAU,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAE5G,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;IAEnH;;;;;OAKG;IACH,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAED,8BAAsB,eAAgB,YAAW,QAAQ,EAAE,eAAe;IAwClE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa;IAvC5C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,aAAa,CAAC;IACnB,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,aAAa,CAAS;IAE9B,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;IAE1C,SAAS,CAAC,UAAU,CAAC,EAAE,cAAc,CAAC;IAEtC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;gBAGpC,OAAO,EAAE,8BAA8B,EACpB,MAAM,GAAE,aAA+C;IAoB9E;;;;OAIG;IACH,aAAa;IAIb;;;;OAIG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IA4GlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,CAAC,YAAY,CAAC,EACpC,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IAmIpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAE1F;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB5B;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;;;;;;;;OASG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;OAkBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAM1C;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAWjC;;;;;;;;;;;;;;;;OAgBG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,eAAe,CAAC;CAmD9B;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,kBAAkB,CAAC;IAE3B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D"}
+{"version":3,"file":"request_provider.d.ts","sourceRoot":"","sources":["../../src/storages/request_provider.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACR,cAAc,EACd,sBAAsB,EACtB,UAAU,EACV,gBAAgB,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EACnB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGjE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACjF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAIrD,8BAAsB,eAAgB,YAAW,QAAQ,EAAE,eAAe;IAsClE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa;IArC5C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAElD,GAAG,EAAE,aAAa,CAAC;IACnB,qBAAqB,SAAc;IACnC,eAAe,SAAU;IAIzB,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAExB,OAAO,CAAC,aAAa,CAAS;IAE9B,SAAS,CAAC,YAAY,yBAAgC;IACtD,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;IAEjD,SAAS,CAAC,4BAA4B,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE1D,SAAS,CAAC,uBAAuB,UAAS;IAE1C,SAAS,CAAC,YAAY,OAAc;IAEpC,SAAS,CAAC,oCAAoC,SAAK;IAEnD,SAAS,CAAC,2BAA2B,SAAK;IAE1C,SAAS,CAAC,UAAU,CAAC,EAAE,cAAc,CAAC;IAEtC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;gBAGpC,OAAO,EAAE,8BAA8B,EACpB,MAAM,GAAE,aAA+C;IAoB9E;;;;OAIG;IACG,aAAa;IAKnB;;;;OAIG;IACG,eAAe;IAKrB;;;;;;;;;;;;;OAaG;IACG,UAAU,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,CAAC;IAmErC;;;;;;;;;;;;;OAaG;IACG,WAAW,CACb,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IA4GlC;;;;;;;;OAQG;IACG,kBAAkB,CACpB,QAAQ,EAAE,YAAY,CAAC,YAAY,CAAC,EACpC,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,wBAAwB,CAAC;IAmIpC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAW3F;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAE1F;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAwCrF;;;;;OAKG;IACG,cAAc,CAChB,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;IAmC5C,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAExD;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEvC,SAAS,CAAC,MAAM;IAShB;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,yBAAyB,GAAG,IAAI;IAc9F;;OAEG;IACH,SAAS,CAAC,2BAA2B,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAQlF;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB5B;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC;IAQ7B;;;;;;;;;OASG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAMxC;;;;;;;;;;;;;;;;;;OAkBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAM1C;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwBxF;;OAEG;cACa,mBAAmB,CAC/B,MAAM,EAAE,cAAc,EACtB,eAAe,EAAE,cAAc,EAAE,EACjC,OAAO,EAAE,4BAA4B;IAiBzC;;OAEG;YACW,mBAAmB;IAWjC;;;;;;;;;;;;;;;;OAgBG;WACU,IAAI,CACb,UAAU,CAAC,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,EAC9C,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,eAAe,CAAC;CA+C9B;AAYD,UAAU,cAAc;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,OAAO,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,sBAAsB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,kBAAkB,CAAC;IAE3B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,sBAAsB;CAAG;AAEtE;;GAEG;AACH,MAAM,WAAW,8BAA+B,SAAQ,sBAAsB;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,kBAAkB;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,yBAA0B,SAAQ,4BAA4B;IAC3E;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,gBAAgB,EAAE,CAAC;IAClC;;;;;;;;;;;;;;;OAeG;IACH,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;CAC5D"}