@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/storages/request_list.d.ts

@@ -1,8 +1,9 @@
-import type { BaseHttpClient, Dictionary } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary } from '@crawlee/types';
 import type { Configuration } from '../configuration.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 import { Request, type RequestOptions, type Source } from '../request.js';
-import type { IRequestLoader } from './request_loader.js';
+import type { IRequestLoader, RequestLoaderStatus } from './request_loader.js';
 import type { IRequestManager } from './request_manager.js';
 /** @internal */
 export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
@@ -126,7 +127,8 @@ export interface RequestListOptions {
      * ```
      *
      * Note that the preferred (and simpler) way to persist the state of crawling of the `RequestList`
-     * is to use the `stateKeyPrefix` parameter instead.
+     * is to use the `stateKeyPrefix` parameter instead. When both are set, a record persisted under
+     * {@link RequestListOptions.persistStateKey} takes precedence over this option.
      */
     state?: RequestListState;
     /**
@@ -216,73 +218,29 @@ export interface RequestListOptions {
  * @category Sources
  */
 export declare class RequestList implements IRequestLoader {
-    private log;
+    #private;
     /**
      * Array of all requests from all sources, in the order as they appeared in sources.
      * All requests in the array have distinct uniqueKey!
      * @internal
      */
     readonly requests: (Request | RequestOptions)[];
-    /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
-    private nextIndex;
-    /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
-    private uniqueKeyToIndex;
     /**
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
-    inProgress: Set<string>;
-    /**
-     * `uniqueKey`s of requests that were in progress when the state was last persisted and thus need to be
-     * re-crawled after a restart. They are served before advancing through the rest of the sources.
-     * @internal
-     */
-    private requestsToRetry;
-    /**
-     * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
-     * @internal
-     */
-    isStatePersisted: boolean;
+    get inProgress(): Set<string>;
     /**
      * Starts as false because we don't know yet and sources might change in the meantime (eg. download from live list).
      * @internal
      */
     areRequestsPersisted: boolean;
-    private isLoading;
-    private isInitialized;
-    private persistStateKey?;
-    private persistRequestsKey?;
-    private initialState?;
-    private store?;
-    private keepDuplicateUrls;
-    private sources;
-    private sourcesFunction?;
-    private proxyConfiguration?;
-    private httpClient?;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
      * @internal
      */
     private constructor();
-    /**
-     * Loads all remote sources of URLs and potentially starts periodic state persistence.
-     * This function must be called before you can start using the instance in a meaningful way.
-     */
-    private initialize;
-    /**
-     * Adds previously persisted Requests, as retrieved from the key-value store.
-     * This needs to be done in a memory efficient way. We should update the input
-     * to a Stream once apify-client supports streams.
-     */
-    private addPersistedRequests;
-    /**
-     * Add Requests from both options.sources and options.sourcesFunction.
-     * This function is called only when persisted sources were not loaded.
-     * We need to avoid keeping both sources and requests in memory
-     * to reduce memory footprint with very large sources.
-     */
-    private addRequestsFromSources;
     /**
      * @inheritDoc
      */
@@ -293,21 +251,6 @@ export declare class RequestList implements IRequestLoader {
      * leaking the listener (and the requests it retains) on the shared event manager.
      */
     teardown(): Promise<void>;
-    /**
-     * Unlike persistState(), this is used only internally, since the sources
-     * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
-     * but there's no reason to persist it again afterwards, because RequestList is immutable.
-     */
-    private persistRequests;
-    /**
-     * Restores RequestList state from a state object.
-     */
-    private restoreState;
-    /**
-     * Attempts to load state and requests using the `RequestList` configuration
-     * and returns a tuple of [state, requests] where each may be null if not loaded.
-     */
-    private loadStateAndPersistedRequests;
     /**
      * Returns an object representing the internal state of the `RequestList` instance.
      * Note that the object's fields can change in future releases.
@@ -316,11 +259,7 @@ export declare class RequestList implements IRequestLoader {
     /**
      * @inheritDoc
      */
-    isEmpty(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    isFinished(): Promise<boolean>;
+    checkReadiness(): Promise<RequestLoaderStatus>;
     /**
      * @inheritDoc
      */
@@ -329,39 +268,10 @@ export declare class RequestList implements IRequestLoader {
      * @inheritDoc
      */
     [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
-    private ensureRequest;
     /**
      * @inheritDoc
      */
     markRequestAsHandled(request: Request): Promise<void>;
-    /**
-     * Adds all fetched requests from a URL from a remote resource.
-     */
-    private addFetchedRequests;
-    private getPersistedState;
-    /**
-     * Fetches URLs from requestsFromUrl and returns them in format of list of requests
-     */
-    private fetchRequestsFromUrl;
-    /**
-     * Adds given request.
-     * If the `source` parameter is a string or plain object and not an instance
-     * of a `Request`, then the function creates a `Request` instance.
-     */
-    private addRequest;
-    /**
-     * Helper function that validates unique key.
-     * Throws an error if uniqueKey is not a non-empty string.
-     */
-    private ensureUniqueKeyValid;
-    /**
-     * Checks that a request is currently being processed and throws an error if not.
-     */
-    private ensureInProgress;
-    /**
-     * Throws an error if request list wasn't initialized.
-     */
-    private ensureIsInitialized;
     /**
      * Returns the total number of unique requests present in the `RequestList`.
      */
@@ -434,10 +344,6 @@ export declare class RequestList implements IRequestLoader {
      *   options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
      */
     static open(listNameOrOptions: string | null | RequestListOptions, sources?: RequestListSource[], options?: RequestListOptions): Promise<RequestList>;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private _downloadListOfUrls;
 }
 /**
  * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.
@@ -463,6 +369,5 @@ export interface RequestListState {
     /** Array of request keys representing those that being processed at the moment. */
     inProgress: string[];
 }
-type RequestListSource = string | Source;
+export type RequestListSource = string | Source;
 export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;
-export {};