@crawlee/core 4.0.0-beta.80 → 4.0.0-beta.82

package/storages/request_list.js

@@ -162,19 +162,19 @@ export class RequestList {
         }
         this.isLoading = true;
         await purgeDefaultStorages({ onlyPurgeOnce: true });
-        const [state, persistedRequests] = await this._loadStateAndPersistedRequests();
+        const [state, persistedRequests] = await this.loadStateAndPersistedRequests();
         // Add persisted requests / new sources in a memory efficient way because with very
         // large lists, we were running out of memory.
         if (persistedRequests) {
-            await this._addPersistedRequests(persistedRequests);
+            await this.addPersistedRequests(persistedRequests);
         }
         else {
-            await this._addRequestsFromSources();
+            await this.addRequestsFromSources();
         }
-        this._restoreState(state);
+        this.restoreState(state);
         this.isInitialized = true;
         if (this.persistRequestsKey && !this.areRequestsPersisted)
-            await this._persistRequests();
+            await this.persistRequests();
         if (this.persistStateKey) {
             serviceLocator.getEventManager().on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
         }
@@ -185,7 +185,7 @@ export class RequestList {
      * This needs to be done in a memory efficient way. We should update the input
      * to a Stream once apify-client supports streams.
      */
-    async _addPersistedRequests(persistedRequests) {
+    async addPersistedRequests(persistedRequests) {
         // We don't need the sources so we purge them to
         // prevent them from hanging in memory.
         for (let i = 0; i < this.sources.length; i++) {
@@ -196,7 +196,7 @@ export class RequestList {
         this.areRequestsPersisted = true;
         const requestStream = createDeserialize(persistedRequests);
         for await (const request of requestStream) {
-            this._addRequest(request);
+            this.addRequest(request);
         }
     }
     /**
@@ -205,7 +205,7 @@ export class RequestList {
      * We need to avoid keeping both sources and requests in memory
      * to reduce memory footprint with very large sources.
      */
-    async _addRequestsFromSources() {
+    async addRequestsFromSources() {
         // We'll load all sources in sequence to ensure that they get loaded in the right order.
         const sourcesCount = this.sources.length;
         for (let i = 0; i < sourcesCount; i++) {
@@ -215,11 +215,11 @@ export class RequestList {
             // oxlint-disable-next-line typescript/no-array-delete
             delete this.sources[i];
             if (typeof source === 'object' && source.requestsFromUrl) {
-                const fetchedRequests = await this._fetchRequestsFromUrl(source);
-                await this._addFetchedRequests(source, fetchedRequests);
+                const fetchedRequests = await this.fetchRequestsFromUrl(source);
+                await this.addFetchedRequests(source, fetchedRequests);
             }
             else {
-                this._addRequest(source);
+                this.addRequest(source);
             }
         }
         // Drop the original array full of empty indexes.
@@ -232,7 +232,7 @@ export class RequestList {
                     const source = sourcesFromFunction[i];
                     // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
                     delete sourcesFromFunction[i];
-                    this._addRequest(source);
+                    this.addRequest(source);
                 }
                 sourcesFromFunction.length = 0;
             }
@@ -277,7 +277,7 @@ export class RequestList {
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
      * but there's no reason to persist it again afterwards, because RequestList is immutable.
      */
-    async _persistRequests() {
+    async persistRequests() {
         const serializedRequests = await serializeArray(this.requests);
         this.store ??= await KeyValueStore.open();
         await this.store.setValue(this.persistRequestsKey, serializedRequests, { contentType: CONTENT_TYPE_BINARY });
@@ -286,7 +286,7 @@ export class RequestList {
     /**
      * Restores RequestList state from a state object.
      */
-    _restoreState(state) {
+    restoreState(state) {
         // If there's no state it means we've not persisted any (yet).
         if (!state)
             return;
@@ -341,7 +341,7 @@ export class RequestList {
      * Attempts to load state and requests using the `RequestList` configuration
      * and returns a tuple of [state, requests] where each may be null if not loaded.
      */
-    async _loadStateAndPersistedRequests() {
+    async loadStateAndPersistedRequests() {
         let state;
         let persistedRequests;
         if (this.initialState) {
@@ -349,12 +349,12 @@ export class RequestList {
             this.log.debug('Loaded state from options.state argument.');
         }
         else if (this.persistStateKey) {
-            state = await this._getPersistedState(this.persistStateKey);
+            state = await this.getPersistedState(this.persistStateKey);
             if (state)
                 this.log.debug('Loaded state from key value store using the persistStateKey.');
         }
         if (this.persistRequestsKey) {
-            persistedRequests = await this._getPersistedState(this.persistRequestsKey);
+            persistedRequests = await this.getPersistedState(this.persistRequestsKey);
             if (persistedRequests)
                 this.log.debug('Loaded requests from key value store using the persistRequestsKey.');
         }
@@ -365,7 +365,7 @@ export class RequestList {
      * Note that the object's fields can change in future releases.
      */
     getState() {
-        this._ensureIsInitialized();
+        this.ensureIsInitialized();
         return {
             nextIndex: this.nextIndex,
             nextUniqueKey: this.nextIndex < this.requests.length ? this.requests[this.nextIndex].uniqueKey : null,
@@ -376,21 +376,21 @@ export class RequestList {
      * @inheritDoc
      */
     async isEmpty() {
-        this._ensureIsInitialized();
+        this.ensureIsInitialized();
         return this.requestsToRetry.length === 0 && this.nextIndex >= this.requests.length;
     }
     /**
      * @inheritDoc
      */
     async isFinished() {
-        this._ensureIsInitialized();
+        this.ensureIsInitialized();
         return this.inProgress.size === 0 && this.nextIndex >= this.requests.length;
     }
     /**
      * @inheritDoc
      */
     async fetchNextRequest() {
-        this._ensureIsInitialized();
+        this.ensureIsInitialized();
         // First re-serve any requests that were interrupted before the last state persist.
         const uniqueKey = this.requestsToRetry.shift();
         if (uniqueKey) {
@@ -431,19 +431,19 @@ export class RequestList {
      */
     async markRequestAsHandled(request) {
         const { uniqueKey } = request;
-        this._ensureUniqueKeyValid(uniqueKey);
-        this._ensureInProgress(uniqueKey);
-        this._ensureIsInitialized();
+        this.ensureUniqueKeyValid(uniqueKey);
+        this.ensureInProgress(uniqueKey);
+        this.ensureIsInitialized();
         this.inProgress.delete(uniqueKey);
         this.isStatePersisted = false;
     }
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    async _addFetchedRequests(source, fetchedRequests) {
+    async addFetchedRequests(source, fetchedRequests) {
         const { requestsFromUrl, regex } = source;
         const originalLength = this.requests.length;
-        fetchedRequests.forEach((request) => this._addRequest(request));
+        fetchedRequests.forEach((request) => this.addRequest(request));
         const fetchedCount = fetchedRequests.length;
         const importedCount = this.requests.length - originalLength;
         this.log.info('Fetched and loaded Requests from a remote resource.', {
@@ -455,7 +455,7 @@ export class RequestList {
             sample: JSON.stringify(fetchedRequests.slice(0, 5)),
         });
     }
-    async _getPersistedState(key) {
+    async getPersistedState(key) {
         this.store ??= await KeyValueStore.open();
         const state = await this.store.getValue(key);
         return state;
@@ -463,7 +463,7 @@ export class RequestList {
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    async _fetchRequestsFromUrl(source) {
+    async fetchRequestsFromUrl(source) {
         const { requestsFromUrl, regex, ...sharedOpts } = source;
         // Download remote resource and parse URLs.
         let urlsArr;
@@ -471,7 +471,7 @@ export class RequestList {
             urlsArr = await this._downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
-                proxyUrl: await this.proxyConfiguration?.newUrl(),
+                proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,
             });
         }
         catch (err) {
@@ -489,7 +489,7 @@ export class RequestList {
      * If the `source` parameter is a string or plain object and not an instance
      * of a `Request`, then the function creates a `Request` instance.
      */
-    _addRequest(source) {
+    addRequest(source) {
         let request;
         const type = typeof source;
         if (type === 'string') {
@@ -511,7 +511,7 @@ export class RequestList {
             request.uniqueKey += `-${this.requests.length}`;
         }
         const { uniqueKey } = request;
-        this._ensureUniqueKeyValid(uniqueKey);
+        this.ensureUniqueKeyValid(uniqueKey);
         // Skip requests with duplicate uniqueKey
         if (!Object.hasOwn(this.uniqueKeyToIndex, uniqueKey)) {
             this.uniqueKeyToIndex[uniqueKey] = this.requests.length;
@@ -525,7 +525,7 @@ export class RequestList {
      * Helper function that validates unique key.
      * Throws an error if uniqueKey is not a non-empty string.
      */
-    _ensureUniqueKeyValid(uniqueKey) {
+    ensureUniqueKeyValid(uniqueKey) {
         if (typeof uniqueKey !== 'string' || !uniqueKey) {
             throw new Error("Request object's uniqueKey must be a non-empty string");
         }
@@ -533,7 +533,7 @@ export class RequestList {
     /**
      * Checks that a request is currently being processed and throws an error if not.
      */
-    _ensureInProgress(uniqueKey) {
+    ensureInProgress(uniqueKey) {
         if (!this.inProgress.has(uniqueKey)) {
             throw new Error(`The request is not being processed (uniqueKey: ${uniqueKey})`);
         }
@@ -541,7 +541,7 @@ export class RequestList {
     /**
      * Throws an error if request list wasn't initialized.
      */
-    _ensureIsInitialized() {
+    ensureIsInitialized() {
         if (!this.isInitialized) {
             throw new Error('RequestList is not initialized; you must call "await requestList.initialize()" before using it!');
         }
@@ -550,14 +550,14 @@ export class RequestList {
      * Returns the total number of unique requests present in the `RequestList`.
      */
     async getTotalCount() {
-        this._ensureIsInitialized();
+        this.ensureIsInitialized();
         return this.requests.length;
     }
     /**
      * Returns the number of pending requests in the `RequestList`.
      */
     async getPendingCount() {
-        this._ensureIsInitialized();
+        this.ensureIsInitialized();
         return this.requests.length - (this.nextIndex - this.inProgress.size);
     }
     /**
@@ -575,7 +575,7 @@ export class RequestList {
      * @inheritDoc
      */
     async getHandledCount() {
-        this._ensureIsInitialized();
+        this.ensureIsInitialized();
         return this.nextIndex - this.inProgress.size;
     }
     /**

package/storages/request_queue.d.ts

@@ -1,17 +1,13 @@
-import type { BaseHttpClient, BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueBackend, RequestQueueInfo } from '@crawlee/types';
+import type { BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueBackend, RequestQueueInfo } from '@crawlee/types';
 import type { ReadonlyDeep } from 'type-fest';
-import { LruCache } from '@apify/datastructures';
-import { Configuration } from '../configuration.js';
-import type { EventManager } from '../events/event_manager.js';
 import type { CrawleeLogger } from '../log.js';
-import type { ProxyConfiguration } from '../proxy_configuration.js';
-import type { InternalSource, RequestOptions, Source } from '../request.js';
+import type { IProxyConfiguration } from '../proxy_configuration.js';
+import type { Source } from '../request.js';
 import { Request } from '../request.js';
 import type { IRequestManager, RequestsLike } from './request_manager.js';
 import type { RequestQueueStats } from './storage_stats.js';
 import type { IStorage, StorageIdentifier } from './storage_instance_manager.js';
 import type { StorageOpenOptions } from './utils.js';
-import { RequestDeduplicationCache } from './request_dedup_cache.js';
 /**
  * Represents a queue of URLs to crawl, which is used for deep crawling of websites
  * where you start with several URLs and then recursively
@@ -47,29 +43,28 @@ import { RequestDeduplicationCache } from './request_dedup_cache.js';
  * @category Sources
  */
 export declare class RequestQueue implements IStorage, IRequestManager {
-    protected readonly config: Configuration;
-    id: string;
-    name?: string;
-    backend: RequestQueueBackend;
-    protected proxyConfiguration?: ProxyConfiguration;
-    log: CrawleeLogger;
-    protected requestCache: LruCache<RequestLruItem>;
+    readonly id: string;
+    readonly name?: string;
+    readonly backend: RequestQueueBackend;
+    private proxyConfiguration?;
+    readonly log: CrawleeLogger;
+    private requestCache;
     /**
      * Remembers the `requestId` of every request already submitted to the client — including background
      * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
      * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
      */
-    protected requestSeenCache: RequestDeduplicationCache;
-    protected queuePausedForMigration: boolean;
-    protected inProgressRequestBatchCount: number;
+    private requestSeenCache;
+    private queuePausedForMigration;
+    private inProgressRequestBatchCount;
     /**
      * The largest expected request-processing time (in seconds) seen so far via
      * {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never
      * lowered, before being forwarded to the storage backend.
      */
-    protected expectedRequestProcessingSecs: number;
-    protected httpClient?: BaseHttpClient;
-    protected readonly events: EventManager;
+    private expectedRequestProcessingSecs;
+    private httpClient?;
+    private readonly events;
     private readonly statsTracker;
     /**
      * Backend-independent usage counters tracked for this request queue (write operations and
@@ -79,7 +74,7 @@ export declare class RequestQueue implements IStorage, IRequestManager {
     /**
      * @internal
      */
-    constructor(options: RequestQueueOptions, config?: Configuration);
+    constructor(options: RequestQueueOptions);
     /**
      * Returns the total number of requests in the queue (i.e. pending + handled).
      *
@@ -205,7 +200,7 @@ export declare class RequestQueue implements IStorage, IRequestManager {
     /**
      * Caches information about request to beware of unneeded addRequest() calls.
      */
-    protected _cacheRequest(cacheKey: string, queueOperationInfo: RequestQueueOperationInfo): void;
+    private cacheRequest;
     /**
      * Removes the queue either from the Apify Cloud storage or from the local database,
      * depending on the mode of operation.
@@ -254,11 +249,11 @@ export declare class RequestQueue implements IStorage, IRequestManager {
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;
+    private fetchRequestsFromUrl;
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[], options: RequestQueueOperationOptions): Promise<ProcessedRequest[]>;
+    private addFetchedRequests;
     /**
      * @internal wraps public utility for mocking purposes
      */
@@ -282,14 +277,6 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      */
     static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;
 }
-interface RequestLruItem {
-    uniqueKey: string;
-    isHandled: boolean;
-    id: string;
-    hydrated: Request | null;
-    lockExpiresAt: number | null;
-    forefront: boolean;
-}
 export interface RequestQueueOptions {
     /** Resolved metadata for the request queue, as returned by the backend's `getMetadata()`. */
     metadata: RequestQueueInfo;
@@ -299,7 +286,7 @@ export interface RequestQueueOptions {
      * Takes advantage of the internal address rotation and authentication process.
      * If undefined, the `requestsFromUrl` requests will be made without proxy.
      */
-    proxyConfiguration?: ProxyConfiguration;
+    proxyConfiguration?: IProxyConfiguration;
 }
 export interface RequestQueueOperationOptions {
     /**
@@ -382,4 +369,3 @@ export interface AddRequestsBatchedResult {
      */
     requestsOverLimit?: Source[];
 }
-export {};

package/storages/request_queue.js

@@ -55,7 +55,6 @@ const MAX_UNPROCESSED_REQUESTS_RETRIES = 3;
  * @category Sources
  */
 export class RequestQueue {
-    config;
     id;
     name;
     backend;
@@ -92,8 +91,7 @@ export class RequestQueue {
     /**
      * @internal
      */
-    constructor(options, config = Configuration.getGlobalConfig()) {
-        this.config = config;
+    constructor(options) {
         this.id = options.metadata.id;
         this.name = options.metadata.name;
         this.events = serviceLocator.getEventManager();
@@ -146,8 +144,8 @@ export class RequestQueue {
         }));
         const { forefront = false } = options;
         if ('requestsFromUrl' in requestLike) {
-            const requests = await this._fetchRequestsFromUrl(requestLike);
-            const processedRequests = await this._addFetchedRequests(requestLike, requests, options);
+            const requests = await this.fetchRequestsFromUrl(requestLike);
+            const processedRequests = await this.addFetchedRequests(requestLike, requests, options);
             return { ...processedRequests[0], forefront };
         }
         ow(requestLike, ow.object.partialShape({
@@ -176,7 +174,7 @@ export class RequestQueue {
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this._cacheRequest(cacheKey, queueOperationInfo);
+        this.cacheRequest(cacheKey, queueOperationInfo);
         this.requestSeenCache.add(cacheKey, request.id);
         return queueOperationInfo;
     }
@@ -223,8 +221,8 @@ export class RequestQueue {
                 requests.push(new Request({ url: requestLike }));
             }
             else if ('requestsFromUrl' in requestLike) {
-                const fetchedRequests = await this._fetchRequestsFromUrl(requestLike);
-                await this._addFetchedRequests(requestLike, fetchedRequests, options);
+                const fetchedRequests = await this.fetchRequestsFromUrl(requestLike);
+                await this.addFetchedRequests(requestLike, fetchedRequests, options);
             }
             else {
                 requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike));
@@ -264,7 +262,7 @@ export class RequestQueue {
             results.processedRequests.push(newRequest);
             const cacheKey = getCachedRequestId(newRequest.uniqueKey);
             if (cache) {
-                this._cacheRequest(cacheKey, { ...newRequest, forefront });
+                this.cacheRequest(cacheKey, { ...newRequest, forefront });
             }
             // Unlike `requestCache`, populate this on every batch (including background ones).
             this.requestSeenCache.add(cacheKey, newRequest.requestId);
@@ -474,7 +472,7 @@ export class RequestQueue {
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
+        this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
         return queueOperationInfo;
     }
     /**
@@ -504,7 +502,7 @@ export class RequestQueue {
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this._cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
+        this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
         return queueOperationInfo;
     }
     /**
@@ -557,7 +555,7 @@ export class RequestQueue {
     /**
      * Caches information about request to beware of unneeded addRequest() calls.
      */
-    _cacheRequest(cacheKey, queueOperationInfo) {
+    cacheRequest(cacheKey, queueOperationInfo) {
         // Remove the previous entry, as otherwise our cache will never update 👀
         this.requestCache.remove(cacheKey);
         this.requestCache.add(cacheKey, {
@@ -646,7 +644,7 @@ export class RequestQueue {
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    async _fetchRequestsFromUrl(source) {
+    async fetchRequestsFromUrl(source) {
         const { requestsFromUrl, regex, ...sharedOpts } = source;
         // Download remote resource and parse URLs.
         let urlsArr;
@@ -654,7 +652,7 @@ export class RequestQueue {
             urlsArr = await this._downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
-                proxyUrl: await this.proxyConfiguration?.newUrl(),
+                proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,
             });
         }
         catch (err) {
@@ -670,7 +668,7 @@ export class RequestQueue {
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    async _addFetchedRequests(source, fetchedRequests, options) {
+    async addFetchedRequests(source, fetchedRequests, options) {
         const { requestsFromUrl, regex } = source;
         const { addedRequests } = await this.addRequestsBatched(fetchedRequests, options);
         this.log.info('Fetched and loaded Requests from a remote resource.', {

package/storages/utils.d.ts

@@ -1,6 +1,6 @@
 import type { BaseHttpClient, Dictionary, StorageBackend } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
-import type { ProxyConfiguration } from '../proxy_configuration.js';
+import type { IProxyConfiguration } from '../proxy_configuration.js';
 /**
  * Options for purging default storage.
  */
@@ -129,7 +129,7 @@ export interface StorageOpenOptions {
      * Takes advantage of the internal address rotation and authentication process.
      * If undefined, the `requestsFromUrl` requests will be made without proxy.
      */
-    proxyConfiguration?: ProxyConfiguration;
+    proxyConfiguration?: IProxyConfiguration;
     /**
      * HTTP client to be used to download the list of URLs in `RequestQueue`.
      */

package/validators.js

@@ -7,8 +7,8 @@ export const validators = {
         message: (label) => `Expected argument '${label}' to be a Puppeteer Page, got something else.`,
     }),
     proxyConfiguration: (value) => ({
-        validator: ow.isValid(value, ow.object.hasKeys('newUrl', 'newProxyInfo')),
-        message: (label) => `Expected argument '${label}' to be a ProxyConfiguration, got something else.`,
+        validator: ow.isValid(value, ow.object.hasKeys('newProxyInfo')),
+        message: (label) => `Expected argument '${label}' to implement the IProxyConfiguration interface (missing 'newProxyInfo'), got something else.`,
     }),
     requestList: (value) => ({
         validator: ow.isValid(value, ow.object.hasKeys('fetchNextRequest', 'persistState')),