@crawlee/core 4.0.0-beta.210 → 4.0.0-beta.212

package/storages/request_list.js

@@ -245,7 +245,7 @@ export class RequestList {
      * Loads all remote sources of URLs and potentially starts periodic state persistence.
      * This function must be called before you can start using the instance in a meaningful way.
      */
-    async initialize() {
+    async #initialize() {
         if (this.#isLoading) {
             throw new Error('RequestList sources are already loading or were loaded.');
         }
@@ -253,17 +253,17 @@ export class RequestList {
         await purgeDefaultStorages({ onlyPurgeOnce: true });
         let persistedRequests;
         if (this.#persistRequestsKey) {
-            persistedRequests = await this.getPersistedState(this.#persistRequestsKey);
+            persistedRequests = await this.#getPersistedState(this.#persistRequestsKey);
             if (persistedRequests)
                 this.#log.debug('Loaded requests from key value store using the persistRequestsKey.');
         }
         // Add persisted requests / new sources in a memory efficient way because with very
         // large lists, we were running out of memory.
         if (persistedRequests) {
-            await this.addPersistedRequests(persistedRequests);
+            await this.#addPersistedRequests(persistedRequests);
         }
         else {
-            await this.addRequestsFromSources();
+            await this.#addRequestsFromSources();
         }
         // Restores the crawling position - the requests have to be loaded first, as the record is validated
         // against them. Also starts the periodic state persistence when `persistStateKey` is set.
@@ -272,7 +272,7 @@ export class RequestList {
         this.#requestsToRetry = [...this.inProgress];
         this.#isInitialized = true;
         if (this.#persistRequestsKey && !this.areRequestsPersisted)
-            await this.persistRequests();
+            await this.#persistRequests();
         return this;
     }
     /**
@@ -280,7 +280,7 @@ export class RequestList {
      * This needs to be done in a memory efficient way. We should update the input
      * to a Stream once apify-client supports streams.
      */
-    async addPersistedRequests(persistedRequests) {
+    async #addPersistedRequests(persistedRequests) {
         // We don't need the sources so we purge them to
         // prevent them from hanging in memory.
         for (let i = 0; i < this.#sources.length; i++) {
@@ -291,7 +291,7 @@ export class RequestList {
         this.areRequestsPersisted = true;
         const requestStream = createDeserialize(persistedRequests);
         for await (const request of requestStream) {
-            this.addRequest(request);
+            this.#addRequest(request);
         }
     }
     /**
@@ -300,7 +300,7 @@ export class RequestList {
      * We need to avoid keeping both sources and requests in memory
      * to reduce memory footprint with very large sources.
      */
-    async addRequestsFromSources() {
+    async #addRequestsFromSources() {
         // We'll load all sources in sequence to ensure that they get loaded in the right order.
         const sourcesCount = this.#sources.length;
         for (let i = 0; i < sourcesCount; i++) {
@@ -310,11 +310,11 @@ export class RequestList {
             // oxlint-disable-next-line typescript/no-array-delete
             delete this.#sources[i];
             if (typeof source === 'object' && source.requestsFromUrl) {
-                const fetchedRequests = await this.fetchRequestsFromUrl(source);
-                await this.addFetchedRequests(source, fetchedRequests);
+                const fetchedRequests = await this.#fetchRequestsFromUrl(source);
+                await this.#addFetchedRequests(source, fetchedRequests);
             }
             else {
-                this.addRequest(source);
+                this.#addRequest(source);
             }
         }
         // Drop the original array full of empty indexes.
@@ -327,7 +327,7 @@ export class RequestList {
                     const source = sourcesFromFunction[i];
                     // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
                     delete sourcesFromFunction[i];
-                    this.addRequest(source);
+                    this.#addRequest(source);
                 }
                 sourcesFromFunction.length = 0;
             }
@@ -365,7 +365,7 @@ export class RequestList {
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
      * but there's no reason to persist it again afterwards, because RequestList is immutable.
      */
-    async persistRequests() {
+    async #persistRequests() {
         const serializedRequests = await serializeArray(this.requests);
         this.#store ??= await KeyValueStore.open();
         await this.#store.setValue(this.#persistRequestsKey, serializedRequests, { contentType: CONTENT_TYPE_BINARY });
@@ -376,14 +376,14 @@ export class RequestList {
      * Note that the object's fields can change in future releases.
      */
     getState() {
-        this.ensureIsInitialized();
+        this.#ensureIsInitialized();
         return this.#stateCodec.encode(this.#state.currentValue);
     }
     /**
      * @inheritDoc
      */
     async checkReadiness() {
-        this.ensureIsInitialized();
+        this.#ensureIsInitialized();
         if (this.#requestsToRetry.length > 0 || this.#state.currentValue.nextIndex < this.requests.length) {
             return { status: 'ready' };
         }
@@ -394,12 +394,12 @@ export class RequestList {
      * @inheritDoc
      */
     async fetchNextRequest() {
-        this.ensureIsInitialized();
+        this.#ensureIsInitialized();
         // First re-serve any requests that were interrupted before the last state persist.
         const uniqueKey = this.#requestsToRetry.shift();
         if (uniqueKey) {
             const index = this.#uniqueKeyToIndex[uniqueKey];
-            return this.ensureRequest(this.requests[index], index);
+            return this.#ensureRequest(this.requests[index], index);
         }
         // Otherwise return next request.
         const state = this.#state.currentValue;
@@ -408,7 +408,7 @@ export class RequestList {
             const request = this.requests[index];
             state.inProgress.add(request.uniqueKey);
             state.nextIndex++;
-            return this.ensureRequest(request, index);
+            return this.#ensureRequest(request, index);
         }
         return null;
     }
@@ -423,7 +423,7 @@ export class RequestList {
             yield req;
         }
     }
-    ensureRequest(requestLike, index) {
+    #ensureRequest(requestLike, index) {
         if (requestLike instanceof Request) {
             return requestLike;
         }
@@ -435,18 +435,18 @@ export class RequestList {
      */
     async markRequestAsHandled(request) {
         const { uniqueKey } = request;
-        this.ensureUniqueKeyValid(uniqueKey);
-        this.ensureInProgress(uniqueKey);
-        this.ensureIsInitialized();
+        this.#ensureUniqueKeyValid(uniqueKey);
+        this.#ensureInProgress(uniqueKey);
+        this.#ensureIsInitialized();
         this.inProgress.delete(uniqueKey);
     }
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    async addFetchedRequests(source, fetchedRequests) {
+    async #addFetchedRequests(source, fetchedRequests) {
         const { requestsFromUrl, regex } = source;
         const originalLength = this.requests.length;
-        fetchedRequests.forEach((request) => this.addRequest(request));
+        fetchedRequests.forEach((request) => this.#addRequest(request));
         const fetchedCount = fetchedRequests.length;
         const importedCount = this.requests.length - originalLength;
         this.#log.info('Fetched and loaded Requests from a remote resource.', {
@@ -458,7 +458,7 @@ export class RequestList {
             sample: JSON.stringify(fetchedRequests.slice(0, 5)),
         });
     }
-    async getPersistedState(key) {
+    async #getPersistedState(key) {
         this.#store ??= await KeyValueStore.open();
         const state = await this.#store.getValue(key);
         return state;
@@ -466,12 +466,12 @@ export class RequestList {
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    async fetchRequestsFromUrl(source) {
+    async #fetchRequestsFromUrl(source) {
         const { requestsFromUrl, regex, ...sharedOpts } = source;
         // Download remote resource and parse URLs.
         let urlsArr;
         try {
-            urlsArr = await this.downloadListOfUrls({
+            urlsArr = await this.#downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
                 proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,
@@ -492,7 +492,7 @@ export class RequestList {
      * If the `source` parameter is a string or plain object and not an instance
      * of a `Request`, then the function creates a `Request` instance.
      */
-    addRequest(source) {
+    #addRequest(source) {
         let request;
         const type = typeof source;
         if (type === 'string') {
@@ -514,7 +514,7 @@ export class RequestList {
             request.uniqueKey += `-${this.requests.length}`;
         }
         const { uniqueKey } = request;
-        this.ensureUniqueKeyValid(uniqueKey);
+        this.#ensureUniqueKeyValid(uniqueKey);
         // Skip requests with duplicate uniqueKey
         if (!Object.hasOwn(this.#uniqueKeyToIndex, uniqueKey)) {
             this.#uniqueKeyToIndex[uniqueKey] = this.requests.length;
@@ -528,7 +528,7 @@ export class RequestList {
      * Helper function that validates unique key.
      * Throws an error if uniqueKey is not a non-empty string.
      */
-    ensureUniqueKeyValid(uniqueKey) {
+    #ensureUniqueKeyValid(uniqueKey) {
         if (typeof uniqueKey !== 'string' || !uniqueKey) {
             throw new Error("Request object's uniqueKey must be a non-empty string");
         }
@@ -536,7 +536,7 @@ export class RequestList {
     /**
      * Checks that a request is currently being processed and throws an error if not.
      */
-    ensureInProgress(uniqueKey) {
+    #ensureInProgress(uniqueKey) {
         if (!this.inProgress.has(uniqueKey)) {
             throw new Error(`The request is not being processed (uniqueKey: ${uniqueKey})`);
         }
@@ -544,7 +544,7 @@ export class RequestList {
     /**
      * Throws an error if request list wasn't initialized.
      */
-    ensureIsInitialized() {
+    #ensureIsInitialized() {
         if (!this.#isInitialized) {
             throw new Error('RequestList is not initialized; you must call "await requestList.initialize()" before using it!');
         }
@@ -553,14 +553,14 @@ export class RequestList {
      * Returns the total number of unique requests present in the `RequestList`.
      */
     async getTotalCount() {
-        this.ensureIsInitialized();
+        this.#ensureIsInitialized();
         return this.requests.length;
     }
     /**
      * Returns the number of pending requests in the `RequestList`.
      */
     async getPendingCount() {
-        this.ensureIsInitialized();
+        this.#ensureIsInitialized();
         return this.requests.length - (this.#state.currentValue.nextIndex - this.inProgress.size);
     }
     /**
@@ -578,7 +578,7 @@ export class RequestList {
      * @inheritDoc
      */
     async getHandledCount() {
-        this.ensureIsInitialized();
+        this.#ensureIsInitialized();
         return this.#state.currentValue.nextIndex - this.inProgress.size;
     }
     /**
@@ -638,7 +638,7 @@ export class RequestList {
         if (listNameOrOptions != null && typeof listNameOrOptions === 'object') {
             options = { ...listNameOrOptions, ...options };
             const rl = new RequestList(options);
-            await rl.initialize();
+            await rl.#initialize();
             return rl;
         }
         const listName = listNameOrOptions;
@@ -651,13 +651,13 @@ export class RequestList {
             persistRequestsKey: listName ? `${listName}-${REQUESTS_PERSISTENCE_KEY}` : options.persistRequestsKey,
             sources: sources ?? options.sources,
         });
-        await rl.initialize();
+        await rl.#initialize();
         return rl;
     }
     /**
      * @internal wraps public utility for mocking purposes
      */
-    async downloadListOfUrls(options) {
+    async #downloadListOfUrls(options) {
         return downloadListOfUrls({
             ...options,
             httpClient: this.#httpClient,

package/storages/request_manager_tandem.d.ts

@@ -17,18 +17,6 @@ export declare class RequestManagerTandem implements IRequestManager {
      *  (e.g. a lazily-opened default {@link RequestQueue}).
      */
     constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
-    /**
-     * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
-     */
-    private getRequestManager;
-    /**
-     * Transfers a single request from the read-only loader to the writable manager.
-     * If the transfer fails, the request is dropped (and logged) rather than reclaimed.
-     *
-     * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
-     *  transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
-     */
-    private transferNextRequestToQueue;
     /**
      * Fetches the next request, transferring one from the loader first if the loader still has work.
      * @inheritdoc

package/storages/request_manager_tandem.js

@@ -37,7 +37,7 @@ export class RequestManagerTandem {
     /**
      * Resolves the writable request manager, opening it lazily (via the factory) on first use and memoizing the result.
      */
-    async getRequestManager() {
+    async #getRequestManager() {
         if (this.#resolvedRequestManager === undefined) {
             this.#requestManagerPromise ??= Promise.resolve(this.#requestManagerFactory());
             this.#resolvedRequestManager = await this.#requestManagerPromise;
@@ -55,12 +55,12 @@ export class RequestManagerTandem {
      * @returns `true` if a request was successfully transferred (or there was nothing to transfer), and `false` if a
      *  transfer was attempted but failed - in which case the caller should not fetch from the manager this round.
      */
-    async transferNextRequestToQueue() {
+    async #transferNextRequestToQueue() {
         const request = await this.#requestLoader.fetchNextRequest();
         if (request === null) {
             return true;
         }
-        const requestManager = await this.getRequestManager();
+        const requestManager = await this.#getRequestManager();
         try {
             await requestManager.addRequest(request, { forefront: true });
             return true;
@@ -84,19 +84,19 @@ export class RequestManagerTandem {
         if ((await this.#requestLoader.checkReadiness()).status === 'ready') {
             // If the transfer failed, the request was dropped; don't fetch from the manager this round (matching
             // crawlee-python behaviour). The next `fetchNextRequest()` call will pick up where we left off.
-            if (!(await this.transferNextRequestToQueue())) {
+            if (!(await this.#transferNextRequestToQueue())) {
                 return null;
             }
         }
         // Try to fetch from manager after the transfer
-        return (await this.getRequestManager()).fetchNextRequest();
+        return (await this.#getRequestManager()).fetchNextRequest();
     }
     /**
      * The loader and the manager read as one source.
      * @inheritdoc
      */
     async checkReadiness() {
-        const requestManager = await this.getRequestManager();
+        const requestManager = await this.#getRequestManager();
         const [loaderStatus, managerStatus] = await Promise.all([
             this.#requestLoader.checkReadiness(),
             requestManager.checkReadiness(),
@@ -108,13 +108,13 @@ export class RequestManagerTandem {
      */
     async getHandledCount() {
         // Since one of the stores needs to have priority when both are present, we query the request manager - the request loader will first be dumped into the manager and then left empty.
-        return (await this.getRequestManager()).getHandledCount();
+        return (await this.#getRequestManager()).getHandledCount();
     }
     /**
      * @inheritdoc
      */
     async getTotalCount() {
-        const requestManager = await this.getRequestManager();
+        const requestManager = await this.#getRequestManager();
         const [managerTotal, loaderTotal] = await Promise.all([
             requestManager.getTotalCount(),
             // count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager
@@ -126,7 +126,7 @@ export class RequestManagerTandem {
      * @inheritdoc
      */
     async getPendingCount() {
-        const requestManager = await this.getRequestManager();
+        const requestManager = await this.#getRequestManager();
         const [managerPending, loaderPending] = await Promise.all([
             requestManager.getPendingCount(),
             this.#requestLoader.getPendingCount(),
@@ -148,25 +148,25 @@ export class RequestManagerTandem {
      * @inheritdoc
      */
     async markRequestAsHandled(request) {
-        return (await this.getRequestManager()).markRequestAsHandled(request);
+        return (await this.#getRequestManager()).markRequestAsHandled(request);
     }
     /**
      * @inheritdoc
      */
     async reclaimRequest(request, options) {
-        return (await this.getRequestManager()).reclaimRequest(request, options);
+        return (await this.#getRequestManager()).reclaimRequest(request, options);
     }
     /**
      * @inheritdoc
      */
     async addRequest(requestLike, options) {
-        return (await this.getRequestManager()).addRequest(requestLike, options);
+        return (await this.#getRequestManager()).addRequest(requestLike, options);
     }
     /**
      * @inheritdoc
      */
     async addRequestsBatched(requests, options) {
-        return (await this.getRequestManager()).addRequestsBatched(requests, options);
+        return (await this.#getRequestManager()).addRequestsBatched(requests, options);
     }
     /**
      * Purges the writable request manager so the tandem can be reused (e.g. across repeated `crawler.run()` calls).
@@ -174,7 +174,7 @@ export class RequestManagerTandem {
      * @inheritdoc
      */
     async purge() {
-        await (await this.getRequestManager()).purge?.();
+        await (await this.#getRequestManager()).purge?.();
     }
     /**
      * Forwards the hint to the writable request manager — that is where requests are fetched from and
@@ -195,6 +195,6 @@ export class RequestManagerTandem {
         return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;
     }
     async extendRequestProcessingTimeSecs(request, secs) {
-        return (await this.getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;
+        return (await this.#getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;
     }
 }

package/storages/request_queue.d.ts

@@ -86,22 +86,6 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * @param [options] Request queue operation options.
      */
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
-    /**
-     * Journals an addition for introspection only; these entries are never replayed. A no-op unless the
-     * transaction is open, so detached and outliving writers stay out of the journal.
-     */
-    private recordRequestJournalEntry;
-    /**
-     * The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup
-     * index derived from the transaction journal.
-     */
-    private bufferedRequests;
-    /**
-     * Adds a request under the `deferred` policy: journaled now, really added by the commit replay.
-     * A new request's `requestId` is the local `uniqueKey` hash and is **provisional** — never write it
-     * to `request.id` or the dedup caches. Dedup is cheapest-first: buffer, caches, then a backend probe.
-     */
-    private addRequestDeferred;
     /** @internal */
     commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
@@ -201,10 +185,6 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * this only touches the one request it is given.
      */
     extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
-    /**
-     * Caches information about request to beware of unneeded addRequest() calls.
-     */
-    private cacheRequest;
     /**
      * Removes the queue either from the Apify Cloud storage or from the local database,
      * depending on the mode of operation.
@@ -250,18 +230,6 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     getInfo(): Promise<RequestQueueInfo>;
-    /**
-     * Fetches URLs from requestsFromUrl and returns them in format of list of requests
-     */
-    private fetchRequestsFromUrl;
-    /**
-     * Adds all fetched requests from a URL from a remote resource.
-     */
-    private addFetchedRequests;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private downloadListOfUrls;
     /**
      * Opens a request queue and returns a promise resolving to an instance
      * of the {@link RequestQueue} class.