@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/storages/request_dedup_cache.js

@@ -12,37 +12,37 @@
  * @internal
  */
 export class RequestDeduplicationCache {
-    size;
     #keys;
     #ids;
+    #size;
     // The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.
     constructor(size = 1_000_000) {
-        this.size = size;
+        this.#size = size;
         this.#keys = new Array(size);
         this.#ids = new Array(size);
     }
     get(cacheKey) {
-        const index = this.indexOf(cacheKey);
+        const index = this.#indexOf(cacheKey);
         return this.#keys[index] === cacheKey ? this.#ids[index] : null;
     }
     add(cacheKey, requestId) {
-        const index = this.indexOf(cacheKey);
+        const index = this.#indexOf(cacheKey);
         this.#keys[index] = cacheKey;
         this.#ids[index] = requestId;
     }
     clear() {
-        this.#keys = new Array(this.size);
-        this.#ids = new Array(this.size);
+        this.#keys = new Array(this.#size);
+        this.#ids = new Array(this.#size);
     }
     // A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.
-    indexOf(cacheKey) {
+    #indexOf(cacheKey) {
         /* eslint-disable no-bitwise */
         let hash = 0x811c9dc5;
         for (let i = 0; i < cacheKey.length; i++) {
             hash ^= cacheKey.charCodeAt(i);
             hash = Math.imul(hash, 0x01000193);
         }
-        return (hash >>> 0) % this.size;
+        return (hash >>> 0) % this.#size;
         /* eslint-enable no-bitwise */
     }
 }

package/storages/request_list.d.ts

@@ -3,7 +3,7 @@ import type { Dictionary } from '@crawlee/types';
 import type { Configuration } from '../configuration.js';
 import type { IProxyConfiguration } from '../proxy_configuration.js';
 import { Request, type RequestOptions, type Source } from '../request.js';
-import type { IRequestLoader } from './request_loader.js';
+import type { IRequestLoader, RequestLoaderStatus } from './request_loader.js';
 import type { IRequestManager } from './request_manager.js';
 /** @internal */
 export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
@@ -127,7 +127,8 @@ export interface RequestListOptions {
      * ```
      *
      * Note that the preferred (and simpler) way to persist the state of crawling of the `RequestList`
-     * is to use the `stateKeyPrefix` parameter instead.
+     * is to use the `stateKeyPrefix` parameter instead. When both are set, a record persisted under
+     * {@link RequestListOptions.persistStateKey} takes precedence over this option.
      */
     state?: RequestListState;
     /**
@@ -228,42 +229,18 @@ export declare class RequestList implements IRequestLoader {
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
-    inProgress: Set<string>;
-    /**
-     * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
-     * @internal
-     */
-    isStatePersisted: boolean;
+    get inProgress(): Set<string>;
     /**
      * Starts as false because we don't know yet and sources might change in the meantime (eg. download from live list).
      * @internal
      */
     areRequestsPersisted: boolean;
-    private sources;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
      * @internal
      */
     private constructor();
-    /**
-     * Loads all remote sources of URLs and potentially starts periodic state persistence.
-     * This function must be called before you can start using the instance in a meaningful way.
-     */
-    private initialize;
-    /**
-     * Adds previously persisted Requests, as retrieved from the key-value store.
-     * This needs to be done in a memory efficient way. We should update the input
-     * to a Stream once apify-client supports streams.
-     */
-    private addPersistedRequests;
-    /**
-     * Add Requests from both options.sources and options.sourcesFunction.
-     * This function is called only when persisted sources were not loaded.
-     * We need to avoid keeping both sources and requests in memory
-     * to reduce memory footprint with very large sources.
-     */
-    private addRequestsFromSources;
     /**
      * @inheritDoc
      */
@@ -274,21 +251,6 @@ export declare class RequestList implements IRequestLoader {
      * leaking the listener (and the requests it retains) on the shared event manager.
      */
     teardown(): Promise<void>;
-    /**
-     * Unlike persistState(), this is used only internally, since the sources
-     * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
-     * but there's no reason to persist it again afterwards, because RequestList is immutable.
-     */
-    private persistRequests;
-    /**
-     * Restores RequestList state from a state object.
-     */
-    private restoreState;
-    /**
-     * Attempts to load state and requests using the `RequestList` configuration
-     * and returns a tuple of [state, requests] where each may be null if not loaded.
-     */
-    private loadStateAndPersistedRequests;
     /**
      * Returns an object representing the internal state of the `RequestList` instance.
      * Note that the object's fields can change in future releases.
@@ -297,11 +259,7 @@ export declare class RequestList implements IRequestLoader {
     /**
      * @inheritDoc
      */
-    isEmpty(): Promise<boolean>;
-    /**
-     * @inheritDoc
-     */
-    isFinished(): Promise<boolean>;
+    checkReadiness(): Promise<RequestLoaderStatus>;
     /**
      * @inheritDoc
      */
@@ -310,39 +268,10 @@ export declare class RequestList implements IRequestLoader {
      * @inheritDoc
      */
     [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
-    private ensureRequest;
     /**
      * @inheritDoc
      */
     markRequestAsHandled(request: Request): Promise<void>;
-    /**
-     * Adds all fetched requests from a URL from a remote resource.
-     */
-    private addFetchedRequests;
-    private getPersistedState;
-    /**
-     * Fetches URLs from requestsFromUrl and returns them in format of list of requests
-     */
-    private fetchRequestsFromUrl;
-    /**
-     * Adds given request.
-     * If the `source` parameter is a string or plain object and not an instance
-     * of a `Request`, then the function creates a `Request` instance.
-     */
-    private addRequest;
-    /**
-     * Helper function that validates unique key.
-     * Throws an error if uniqueKey is not a non-empty string.
-     */
-    private ensureUniqueKeyValid;
-    /**
-     * Checks that a request is currently being processed and throws an error if not.
-     */
-    private ensureInProgress;
-    /**
-     * Throws an error if request list wasn't initialized.
-     */
-    private ensureIsInitialized;
     /**
      * Returns the total number of unique requests present in the `RequestList`.
      */
@@ -415,10 +344,6 @@ export declare class RequestList implements IRequestLoader {
      *   options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
      */
     static open(listNameOrOptions: string | null | RequestListOptions, sources?: RequestListSource[], options?: RequestListOptions): Promise<RequestList>;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private downloadListOfUrls;
 }
 /**
  * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.
@@ -444,6 +369,5 @@ export interface RequestListState {
     /** Array of request keys representing those that being processed at the moment. */
     inProgress: string[];
 }
-type RequestListSource = string | Source;
+export type RequestListSource = string | Source;
 export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;
-export {};