@crawlee/core 3.17.1-beta.21 → 3.17.1-beta.22

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "3.17.1-beta.21",
+    "version": "3.17.1-beta.22",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -59,9 +59,9 @@
         "@apify/pseudo_url": "^2.0.30",
         "@apify/timeout": "^0.3.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/memory-storage": "3.17.1-beta.21",
-        "@crawlee/types": "3.17.1-beta.21",
-        "@crawlee/utils": "3.17.1-beta.21",
+        "@crawlee/memory-storage": "3.17.1-beta.22",
+        "@crawlee/types": "3.17.1-beta.22",
+        "@crawlee/utils": "3.17.1-beta.22",
         "@sapphire/async-queue": "^1.5.1",
         "@vladfrangu/async_event_emitter": "^2.2.2",
         "csv-stringify": "^6.2.0",
@@ -83,5 +83,5 @@
             }
         }
     },
-    "gitHead": "b315b66d9e3dffe8963cad291a85058ee0ef07c9"
+    "gitHead": "913fa8ac27d9a04e8a8b87df05d1c6c9ab7f3a87"
 }

package/storages/request_provider.js

@@ -13,6 +13,11 @@ const request_1 = require("../request");
 const access_checking_1 = require("./access_checking");
 const storage_manager_1 = require("./storage_manager");
 const utils_2 = require("./utils");
+/**
+ * Maximum number of consecutive batch-add attempts that make no progress before the remaining
+ * unprocessed requests are skipped, so permanently rejected requests don't retry forever.
+ */
+const MAX_UNPROCESSED_REQUESTS_RETRIES = 3;
 class RequestProvider {
     constructor(options, config = configuration_1.Configuration.getGlobalConfig()) {
         Object.defineProperty(this, "config", {
@@ -385,13 +390,20 @@ class RequestProvider {
         const requestIterator = generateRequests();
         const chunks = (0, utils_1.peekableAsyncIterable)((0, utils_1.chunkedAsyncIterable)(requestIterator, effectiveChunkSize));
         const chunksIterator = chunks[Symbol.asyncIterator]();
-        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true) => {
+        const attemptToAddToQueueAndAddAnyUnprocessed = async (providedRequests, cache = true, unsuccessfulAttempts = 0) => {
             const resultsToReturn = [];
             const apiResult = await this.addRequests(providedRequests, { forefront: options.forefront, cache });
             resultsToReturn.push(...apiResult.processedRequests);
             if (apiResult.unprocessedRequests.length) {
+                // Count attempts that make no progress, so permanently rejected requests (e.g. a malformed
+                // `userData` shape causing a 400) don't loop forever. Any progress resets the counter.
+                const attempts = apiResult.processedRequests.length ? 0 : unsuccessfulAttempts + 1;
+                if (attempts >= MAX_UNPROCESSED_REQUESTS_RETRIES) {
+                    this.log.warning(`Some requests were consistently rejected by the request queue and will be skipped after ${MAX_UNPROCESSED_REQUESTS_RETRIES} attempts. This usually means the request data is malformed (e.g. an invalid 'userData' shape).`, { unprocessedRequests: apiResult.unprocessedRequests });
+                    return resultsToReturn;
+                }
                 await (0, utils_1.sleep)(waitBetweenBatchesMillis);
-                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false)));
+                resultsToReturn.push(...(await attemptToAddToQueueAndAddAnyUnprocessed(providedRequests.filter((r) => !apiResult.processedRequests.some((pr) => pr.uniqueKey === r.uniqueKey)), false, attempts)));
             }
             return resultsToReturn;
         };