@crawlee/core 4.0.0-beta.210 → 4.0.0-beta.212

package/storages/request_queue.js

@@ -180,20 +180,20 @@ export class RequestQueue {
         parseArgument(requestLike, schemas.anyObject);
         const { forefront } = parseArgument(options, operationOptionsSchema);
         if ('requestsFromUrl' in requestLike) {
-            const requests = await this.fetchRequestsFromUrl(requestLike);
-            const processedRequests = await this.addFetchedRequests(requestLike, requests, options);
+            const requests = await this.#fetchRequestsFromUrl(requestLike);
+            const processedRequests = await this.#addFetchedRequests(requestLike, requests, options);
             return { ...processedRequests[0], forefront };
         }
         parseArgument(requestLike, newRequestLikeSchema);
         const request = requestLike instanceof Request ? requestLike : new Request(requestLike);
         if (transaction?.policy.requestQueue === 'deferred') {
-            return this.addRequestDeferred(transaction, request, forefront);
+            return this.#addRequestDeferred(transaction, request, forefront);
         }
         const cacheKey = getRequestId(request.uniqueKey);
         const cachedInfo = this.#requestCache.get(cacheKey);
         if (cachedInfo) {
             request.id = cachedInfo.id;
-            this.recordRequestJournalEntry(transaction, [request], forefront, true);
+            this.#recordRequestJournalEntry(transaction, [request], forefront, true);
             return {
                 wasAlreadyPresent: true,
                 // We may assume that if request is in local cache then also the information if the
@@ -206,13 +206,13 @@ export class RequestQueue {
         }
         this.#statsTracker.add('writeCount');
         const { processedRequests } = await this.backend.addBatchOfRequests([request], { forefront });
-        this.recordRequestJournalEntry(transaction, [request], forefront, true);
+        this.#recordRequestJournalEntry(transaction, [request], forefront, true);
         const queueOperationInfo = {
             ...processedRequests[0],
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this.cacheRequest(cacheKey, queueOperationInfo);
+        this.#cacheRequest(cacheKey, queueOperationInfo);
         this.#requestSeenCache.add(cacheKey, request.id);
         return queueOperationInfo;
     }
@@ -220,7 +220,7 @@ export class RequestQueue {
      * Journals an addition for introspection only; these entries are never replayed. A no-op unless the
      * transaction is open, so detached and outliving writers stay out of the journal.
      */
-    recordRequestJournalEntry(transaction, requests, forefront, writeThrough) {
+    #recordRequestJournalEntry(transaction, requests, forefront, writeThrough) {
         if (!transaction?.isActive || requests.length === 0)
             return;
         transaction.recordJournalEntry({
@@ -239,7 +239,7 @@ export class RequestQueue {
      * The requests buffered by the given transaction for this queue, keyed by `uniqueKey` — a dedup
      * index derived from the transaction journal.
      */
-    bufferedRequests(transaction) {
+    #bufferedRequests(transaction) {
         const buffered = new Map();
         // Only `deferred` records snapshots, so scanning the journal under `writeThrough` never finds any.
         if (transaction.policy.requestQueue !== 'deferred')
@@ -259,10 +259,10 @@ export class RequestQueue {
      * A new request's `requestId` is the local `uniqueKey` hash and is **provisional** — never write it
      * to `request.id` or the dedup caches. Dedup is cheapest-first: buffer, caches, then a backend probe.
      */
-    async addRequestDeferred(transaction, request, forefront, buffered = this.bufferedRequests(transaction)) {
+    async #addRequestDeferred(transaction, request, forefront, buffered = this.#bufferedRequests(transaction)) {
         // This transaction's own buffered adds; the shared caches never see them (provisional ids).
         if (buffered.has(request.uniqueKey)) {
-            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            this.#recordRequestJournalEntry(transaction, [request], forefront, false);
             return {
                 wasAlreadyPresent: true,
                 wasAlreadyHandled: false,
@@ -277,7 +277,7 @@ export class RequestQueue {
         const cachedInfo = this.#requestCache.get(cacheKey);
         const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
         if (knownRequestId) {
-            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            this.#recordRequestJournalEntry(transaction, [request], forefront, false);
             return {
                 wasAlreadyPresent: true,
                 // The dedup cache doesn't track the handled state; only the full record does.
@@ -290,7 +290,7 @@ export class RequestQueue {
         // The caches are bounded, so a miss is not proof of absence - probe for an accurate answer.
         const existing = await this.backend.getRequest(request.uniqueKey);
         if (existing) {
-            this.recordRequestJournalEntry(transaction, [request], forefront, false);
+            this.#recordRequestJournalEntry(transaction, [request], forefront, false);
             return {
                 wasAlreadyPresent: true,
                 wasAlreadyHandled: existing.handledAt != null,
@@ -348,7 +348,7 @@ export class RequestQueue {
             // Only now, with the real backend-assigned ids, may the shared dedup caches be populated.
             for (const processed of processedRequests) {
                 const cacheKey = getRequestId(processed.uniqueKey);
-                this.cacheRequest(cacheKey, { ...processed, forefront });
+                this.#cacheRequest(cacheKey, { ...processed, forefront });
                 this.#requestSeenCache.add(cacheKey, processed.requestId);
             }
             if (unprocessedRequests.length > 0) {
@@ -398,21 +398,21 @@ export class RequestQueue {
                 requests.push(new Request({ url: requestLike }));
             }
             else if ('requestsFromUrl' in requestLike) {
-                const fetchedRequests = await this.fetchRequestsFromUrl(requestLike);
-                await this.addFetchedRequests(requestLike, fetchedRequests, options);
+                const fetchedRequests = await this.#fetchRequestsFromUrl(requestLike);
+                await this.#addFetchedRequests(requestLike, fetchedRequests, options);
             }
             else {
                 requests.push(requestLike instanceof Request ? requestLike : new Request(requestLike));
             }
         }
         if (transaction?.policy.requestQueue === 'deferred') {
-            const buffered = this.bufferedRequests(transaction);
+            const buffered = this.#bufferedRequests(transaction);
             for (const request of requests) {
-                results.processedRequests.push(await this.addRequestDeferred(transaction, request, forefront, buffered));
+                results.processedRequests.push(await this.#addRequestDeferred(transaction, request, forefront, buffered));
             }
             return results;
         }
-        this.recordRequestJournalEntry(transaction, requests, forefront, true);
+        this.#recordRequestJournalEntry(transaction, requests, forefront, true);
         const requestsToAdd = new Map();
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
@@ -447,7 +447,7 @@ export class RequestQueue {
             results.processedRequests.push(newRequest);
             const cacheKey = getCachedRequestId(newRequest.uniqueKey);
             if (cache) {
-                this.cacheRequest(cacheKey, { ...newRequest, forefront });
+                this.#cacheRequest(cacheKey, { ...newRequest, forefront });
             }
             // Unlike `requestCache`, populate this on every batch (including background ones).
             this.#requestSeenCache.add(cacheKey, newRequest.requestId);
@@ -533,7 +533,7 @@ export class RequestQueue {
         const transaction = activeStorageTransaction();
         parseArgument(uniqueKey, uniqueKeySchema);
         // Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.
-        const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);
+        const buffered = transaction && this.#bufferedRequests(transaction).get(uniqueKey);
         if (buffered) {
             return Request.fromSchema(buffered);
         }
@@ -596,7 +596,7 @@ export class RequestQueue {
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
+        this.#cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
         return queueOperationInfo;
     }
     /**
@@ -622,7 +622,7 @@ export class RequestQueue {
             uniqueKey: request.uniqueKey,
             forefront,
         };
-        this.cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
+        this.#cacheRequest(getRequestId(request.uniqueKey), queueOperationInfo);
         return queueOperationInfo;
     }
     /**
@@ -645,7 +645,7 @@ export class RequestQueue {
     async checkReadiness() {
         const transaction = activeStorageTransaction();
         // Requests buffered by the active transaction count as pending from its point of view.
-        if (transaction && this.bufferedRequests(transaction).size > 0) {
+        if (transaction && this.#bufferedRequests(transaction).size > 0) {
             return { status: 'ready' };
         }
         // Something fetchable outranks everything below, so this is the only backend call a probe needs.
@@ -689,7 +689,7 @@ export class RequestQueue {
     /**
      * Caches information about request to beware of unneeded addRequest() calls.
      */
-    cacheRequest(cacheKey, queueOperationInfo) {
+    #cacheRequest(cacheKey, queueOperationInfo) {
         // Remove the previous entry, as otherwise our cache will never update 👀
         this.#requestCache.remove(cacheKey);
         this.#requestCache.add(cacheKey, {
@@ -773,7 +773,7 @@ export class RequestQueue {
     async getInfo() {
         const transaction = activeStorageTransaction();
         const metadata = await this.backend.getMetadata();
-        const bufferedCount = transaction ? this.bufferedRequests(transaction).size : 0;
+        const bufferedCount = transaction ? this.#bufferedRequests(transaction).size : 0;
         if (bufferedCount > 0) {
             return {
                 ...metadata,
@@ -786,12 +786,12 @@ export class RequestQueue {
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    async fetchRequestsFromUrl(source) {
+    async #fetchRequestsFromUrl(source) {
         const { requestsFromUrl, regex, ...sharedOpts } = source;
         // Download remote resource and parse URLs.
         let urlsArr;
         try {
-            urlsArr = await this.downloadListOfUrls({
+            urlsArr = await this.#downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
                 proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,
@@ -810,7 +810,7 @@ export class RequestQueue {
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    async addFetchedRequests(source, fetchedRequests, options) {
+    async #addFetchedRequests(source, fetchedRequests, options) {
         const { requestsFromUrl, regex } = source;
         const { addedRequests } = await this.addRequestsBatched(fetchedRequests, options);
         this.log.info('Fetched and loaded Requests from a remote resource.', {
@@ -826,7 +826,7 @@ export class RequestQueue {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    async downloadListOfUrls(options) {
+    async #downloadListOfUrls(options) {
         return downloadListOfUrls({
             ...options,
             httpClient: this.#httpClient,

package/storages/storage_instance_manager.js

@@ -32,7 +32,7 @@ class StorageCache {
         return undefined;
     }
     /** Write a single entry into a given tier. */
-    setInMap(tier, cls, key, instance, backendCacheKey) {
+    #setInMap(tier, cls, key, instance, backendCacheKey) {
         if (!tier.has(cls))
             tier.set(cls, new Map());
         const keyMap = tier.get(cls);
@@ -45,14 +45,14 @@ class StorageCache {
      */
     set(cls, instance, backendCacheKey, alias) {
         // Always cache by id.
-        this.setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
+        this.#setInMap(this.byId, cls, instance.id, instance, backendCacheKey);
         // Cache by name — only for named storages.
         if (instance.name) {
-            this.setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
+            this.#setInMap(this.byName, cls, instance.name, instance, backendCacheKey);
         }
         // Cache by alias — only for unnamed storages opened via alias.
         if (alias !== undefined) {
-            this.setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
+            this.#setInMap(this.byAlias, cls, alias, instance, backendCacheKey);
         }
     }
     removeFromCache(instance) {

package/storages/transaction.d.ts

@@ -189,7 +189,6 @@ export declare class StorageTransaction implements StorageTransactionView {
      * partway may have applied some of the writes already.
      */
     commit(): Promise<void>;
-    private flush;
     /**
      * Discards the journaled writes. A no-op unless the transaction is `open` — in particular, calling it
      * after a successful `commit()` (which the crawler's error handling can legitimately do) does nothing

package/storages/transaction.js

@@ -95,7 +95,7 @@ export class StorageTransaction {
             // on every operation - and `@apify/timeout` shares one `AbortController` across nested
             // frames, so a request-handler timeout that already fired would abort the commit of a
             // handler that succeeded. Hence a fresh timeout context, which also provides the time bound.
-            await timeoutStorage.exit(async () => addTimeoutToPromise(async () => this.flush(), this.#commitTimeoutMillis, `Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`));
+            await timeoutStorage.exit(async () => addTimeoutToPromise(async () => this.#flush(), this.#commitTimeoutMillis, `Committing the storage transaction timed out after ${this.#commitTimeoutMillis / 1000} seconds.`));
         }
         catch (error) {
             this.#state = 'failed';
@@ -122,7 +122,7 @@ export class StorageTransaction {
             }
         });
     }
-    async flush() {
+    async #flush() {
         // Each participating frontend replays all of its buffered entries in one call. Frontends are
         // ordered by storage type: key-value stores and request queues first (idempotent under retry),
         // datasets last (not idempotent), minimizing the blast radius of a partial commit failure.