@crawlee/core 4.0.0-beta.265 → 4.0.0-beta.267

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.265",
+    "version": "4.0.0-beta.267",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.1.0",
-        "@crawlee/fs-storage": "4.0.0-beta.265",
-        "@crawlee/http-client": "4.0.0-beta.265",
-        "@crawlee/types": "4.0.0-beta.265",
-        "@crawlee/utils": "4.0.0-beta.265",
+        "@crawlee/fs-storage": "4.0.0-beta.267",
+        "@crawlee/http-client": "4.0.0-beta.267",
+        "@crawlee/types": "4.0.0-beta.267",
+        "@crawlee/utils": "4.0.0-beta.267",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.1.0",
         "@vladfrangu/async_event_emitter": "^2.4.7",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "fa0976bd3977fcc23a2162de18ad0065d66586f1"
+    "gitHead": "a4e49e8a8117acdb808c6cba5b05c3d56bf2b123"
 }

package/storages/request-loader.d.ts

@@ -32,7 +32,7 @@ export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
     status: 'stalled';
 }>;
 /**
- * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
+ * Combines two request sources' statuses, with the precedence `ready` > `waiting` > `stalled` > `finished`.
  *
  * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
  * nothing.

package/storages/request-loader.js

@@ -1,5 +1,5 @@
 /**
- * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
+ * Combines two request sources' statuses, with the precedence `ready` > `waiting` > `stalled` > `finished`.
  *
  * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
  * nothing.
@@ -13,17 +13,9 @@ export function joinRequestSourceStatuses(a, b) {
     if (b.status === 'ready') {
         return b;
     }
-    // `ready` outranking `stalled` masks a stalled source while the other still has work. That is parity with
-    // the crawler before this was a single answer: the stall check was only reached from `isFinishedFunction`,
-    // which the task loop calls only when nothing is in flight and nothing is ready. "Fixing" the masking
-    // turns a crawl that is progressing elsewhere into a `PersistentRateLimitError`. `stalled` outranking
-    // `waiting` is the same parity - that call site fired regardless of other domains' clocks.
-    if (a.status === 'stalled') {
-        return a;
-    }
-    if (b.status === 'stalled') {
-        return b;
-    }
+    // `ready` and `waiting` outranking `stalled` mask a stalled source while the other still has work, now or
+    // later. The crawler turns `stalled` into a `PersistentRateLimitError`, so "fixing" the masking turns a crawl
+    // that can still progress elsewhere into a failure. The stall surfaces once the other source runs out of work.
     if (a.status === 'waiting') {
         // The earlier of the two known wake-up times - unknown only if neither source announced one.
         if (b.status !== 'waiting' || b.readyAt === undefined) {
@@ -31,6 +23,12 @@ export function joinRequestSourceStatuses(a, b) {
         }
         return a.readyAt !== undefined && a.readyAt <= b.readyAt ? a : b;
     }
+    if (b.status === 'waiting') {
+        return b;
+    }
+    if (a.status === 'stalled') {
+        return a;
+    }
     // `a` is finished, so `b` decides.
     return b;
 }