@crawlee/core 4.0.0-beta.97 → 4.0.0-beta.98

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.97",
+    "version": "4.0.0-beta.98",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,9 +53,9 @@
         "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.97",
-        "@crawlee/types": "4.0.0-beta.97",
-        "@crawlee/utils": "4.0.0-beta.97",
+        "@crawlee/fs-storage": "4.0.0-beta.98",
+        "@crawlee/types": "4.0.0-beta.98",
+        "@crawlee/utils": "4.0.0-beta.98",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -79,5 +79,5 @@
             }
         }
     },
-    "gitHead": "43008dd43f4832d083353ba1b731c0b4607c9cfa"
+    "gitHead": "3b8cd86b13e253ab5fc71e631e12a68f7465cee5"
 }

package/storages/request_queue.js

@@ -17,11 +17,6 @@ import { RequestDeduplicationCache } from './request_dedup_cache.js';
  * @internal
  */
 const MAX_CACHED_REQUESTS = 2_000_000;
-/**
- * The maximum number of consecutive no-progress retries for unprocessed requests in `addRequestsBatched()`.
- * @internal
- */
-const MAX_UNPROCESSED_REQUESTS_RETRIES = 3;
 /**
  * Represents a queue of URLs to crawl, which is used for deep crawling of websites
  * where you start with several URLs and then recursively
@@ -327,32 +322,27 @@ export class RequestQueue {
         const requestIterator = generateRequests();
         const chunks = peekableAsyncIterable(chunkedAsyncIterable(requestIterator, effectiveChunkSize));
         const chunksIterator = chunks[Symbol.asyncIterator]();
-        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true, unsuccessfulAttempts = 0) => {
-            const resultsToReturn = [];
-            const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
-            resultsToReturn.push(...apiResult.processedRequests);
-            if (apiResult.unprocessedRequests.length) {
-                // Count attempts that make no progress, so permanently rejected requests (e.g. a malformed
-                // `userData` shape causing a 400) don't loop forever. Any progress resets the counter.
-                const attempts = apiResult.processedRequests.length ? 0 : unsuccessfulAttempts + 1;
-                if (attempts >= MAX_UNPROCESSED_REQUESTS_RETRIES) {
-                    this.log.warning(`Some requests were consistently rejected by the request queue and will be skipped after ${MAX_UNPROCESSED_REQUESTS_RETRIES} attempts. This usually means the request data is malformed (e.g. an invalid 'userData' shape).`, { unprocessedRequests: apiResult.unprocessedRequests });
-                    return resultsToReturn;
-                }
-                await sleep(waitBetweenBatchesMillis);
-                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false, attempts)));
-            }
-            return resultsToReturn;
-        };
         /**
          * Process a chunk: send it to the queue, then update the remaining budget if maxNewRequests is active.
+         *
+         * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
+         * `unprocessedRequests` is what remains after the backend's own transient-error handling - a
+         * semantic rejection (e.g. a malformed `userData` shape) that re-sending would only re-poke.
+         * Retrying transient failures is the storage backend's job, not the frontend's.
          */
         const processChunk = async (chunk, cache = true) => {
-            const results = await attemptToAddToQueueAndAddAnyUnprocessed(chunk, cache);
+            const { processedRequests, unprocessedRequests } = await this.addRequests(chunk, {
+                forefront: options.forefront,
+                cache,
+            });
+            if (unprocessedRequests.length > 0) {
+                this.log.warning('Some requests were rejected by the request queue and will be skipped. ' +
+                    "This usually means the request data is malformed (e.g. an invalid 'userData' shape).", { unprocessedRequests });
+            }
             if (maxNewRequests !== undefined) {
-                remainingBudget -= results.filter((r) => !r.wasAlreadyPresent).length;
+                remainingBudget -= processedRequests.filter((r) => !r.wasAlreadyPresent).length;
             }
-            return results;
+            return processedRequests;
         };
         /**
          * Build the final result. When maxNewRequests is set, drains any remaining items