@crawlee/core 4.0.0-beta.266 → 4.0.0-beta.268

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.266",
+    "version": "4.0.0-beta.268",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.1.0",
-        "@crawlee/fs-storage": "4.0.0-beta.266",
-        "@crawlee/http-client": "4.0.0-beta.266",
-        "@crawlee/types": "4.0.0-beta.266",
-        "@crawlee/utils": "4.0.0-beta.266",
+        "@crawlee/fs-storage": "4.0.0-beta.268",
+        "@crawlee/http-client": "4.0.0-beta.268",
+        "@crawlee/types": "4.0.0-beta.268",
+        "@crawlee/utils": "4.0.0-beta.268",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.1.0",
         "@vladfrangu/async_event_emitter": "^2.4.7",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "1b0e67fa736823f6ffd42537c87e41fe373394fd"
+    "gitHead": "9462257a02015e7ef40ebc16d8fdd0665c9554fe"
 }

package/storages/batched-adds.js

@@ -14,8 +14,10 @@ import { activeStorageTransaction, withDirectStorageAccess } from './transaction
 export async function drainRequestBatches(options) {
     const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, log, processChunk, trackBackgroundBatches, } = options;
     const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';
+    // Leftovers are only reported when the caller waits anyway: draining an async producer just to find
+    // them could block indefinitely.
+    const reportsOverLimit = maxNewRequests !== undefined && waitForAllRequestsToBeAdded;
     let remainingBudget = maxNewRequests ?? Infinity;
-    const requestsOverLimit = [];
     // Never hand a chunk more than the budget allows, so an over-large final batch cannot overshoot.
     const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
     const chunks = peekableAsyncIterable(chunkedAsyncIterable(items, effectiveChunkSize));
@@ -28,12 +30,14 @@ export async function drainRequestBatches(options) {
         return processedRequests;
     };
     const buildResult = async (addedRequests, waitForAll) => {
-        if (maxNewRequests !== undefined) {
-            // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
-            // is left is still sitting in `items` rather than in a chunk we have seen.
-            for await (const item of items) {
-                requestsOverLimit.push(item);
-            }
+        if (!reportsOverLimit) {
+            return { addedRequests, waitForAllRequestsToBeAdded: waitForAll };
+        }
+        // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
+        // is left is still sitting in `items` rather than in a chunk we have seen.
+        const requestsOverLimit = [];
+        for await (const item of items) {
+            requestsOverLimit.push(item);
         }
         return { addedRequests, waitForAllRequestsToBeAdded: waitForAll, requestsOverLimit };
     };
@@ -55,9 +59,8 @@ export async function drainRequestBatches(options) {
         }
         return added;
     };
-    // With a budget we must drain everything before we can report what went over it; under `deferred` a
-    // writer that finishes after commit would have nowhere to put its journal entries.
-    const awaitsRemainder = waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
+    // Under `deferred` a writer that finishes after commit would have nowhere to put its journal entries.
+    const awaitsRemainder = waitForAllRequestsToBeAdded || deferred;
     // An un-awaited writer outlives the transaction scope it inherits, so it must not record into a
     // transaction that may already be closed. It writes directly - its write-through additions were never
     // going to be rolled back anyway - which means the requests it adds are not journaled.

package/storages/request-loader.d.ts

@@ -32,7 +32,7 @@ export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
     status: 'stalled';
 }>;
 /**
- * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
+ * Combines two request sources' statuses, with the precedence `ready` > `waiting` > `stalled` > `finished`.
  *
  * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
  * nothing.

package/storages/request-loader.js

@@ -1,5 +1,5 @@
 /**
- * Combines two request sources' statuses, with the precedence `ready` > `stalled` > `waiting` > `finished`.
+ * Combines two request sources' statuses, with the precedence `ready` > `waiting` > `stalled` > `finished`.
  *
  * Binary rather than variadic on purpose: it is on the task loop's probe path and folding a pair allocates
  * nothing.
@@ -13,17 +13,9 @@ export function joinRequestSourceStatuses(a, b) {
     if (b.status === 'ready') {
         return b;
     }
-    // `ready` outranking `stalled` masks a stalled source while the other still has work. That is parity with
-    // the crawler before this was a single answer: the stall check was only reached from `isFinishedFunction`,
-    // which the task loop calls only when nothing is in flight and nothing is ready. "Fixing" the masking
-    // turns a crawl that is progressing elsewhere into a `PersistentRateLimitError`. `stalled` outranking
-    // `waiting` is the same parity - that call site fired regardless of other domains' clocks.
-    if (a.status === 'stalled') {
-        return a;
-    }
-    if (b.status === 'stalled') {
-        return b;
-    }
+    // `ready` and `waiting` outranking `stalled` mask a stalled source while the other still has work, now or
+    // later. The crawler turns `stalled` into a `PersistentRateLimitError`, so "fixing" the masking turns a crawl
+    // that can still progress elsewhere into a failure. The stall surfaces once the other source runs out of work.
     if (a.status === 'waiting') {
         // The earlier of the two known wake-up times - unknown only if neither source announced one.
         if (b.status !== 'waiting' || b.readyAt === undefined) {
@@ -31,6 +23,12 @@ export function joinRequestSourceStatuses(a, b) {
         }
         return a.readyAt !== undefined && a.readyAt <= b.readyAt ? a : b;
     }
+    if (b.status === 'waiting') {
+        return b;
+    }
+    if (a.status === 'stalled') {
+        return a;
+    }
     // `a` is finished, so `b` decides.
     return b;
 }

package/storages/request-queue.d.ts

@@ -303,13 +303,11 @@ export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions
     waitBetweenBatchesMillis?: number;
     /**
      * If set, only this many *actually new* requests (i.e. not already present in the queue) will be added.
-     * Once the budget is reached, remaining requests from the iterable will be collected in
-     * {@link AddRequestsBatchedResult.requestsOverLimit|`requestsOverLimit`} instead.
+     * Once the budget is reached, remaining requests from the iterable are left unconsumed, or collected in
+     * {@link AddRequestsBatchedResult.requestsOverLimit|`requestsOverLimit`} if
+     * {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} is set.
      *
      * This is useful in combination with `maxRequestsPerCrawl` to avoid duplicate URLs consuming the budget.
-     *
-     * **Note:** Setting this option implicitly enables {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`},
-     * since all batches must complete before leftover requests can be accurately reported.
      */
     maxNewRequests?: number;
 }
@@ -335,7 +333,8 @@ export interface AddRequestsBatchedResult {
     /**
      * Requests from the input that were not added to the queue because the
      * {@link AddRequestsBatchedOptions.maxNewRequests|`maxNewRequests`} budget was reached.
-     * Empty when `maxNewRequests` is not set.
+     * `undefined` unless both `maxNewRequests` and
+     * {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} are set.
      */
     requestsOverLimit?: Source[];
 }