@crawlee/core 4.0.0-beta.267 → 4.0.0-beta.268

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.267",
+    "version": "4.0.0-beta.268",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.1.0",
-        "@crawlee/fs-storage": "4.0.0-beta.267",
-        "@crawlee/http-client": "4.0.0-beta.267",
-        "@crawlee/types": "4.0.0-beta.267",
-        "@crawlee/utils": "4.0.0-beta.267",
+        "@crawlee/fs-storage": "4.0.0-beta.268",
+        "@crawlee/http-client": "4.0.0-beta.268",
+        "@crawlee/types": "4.0.0-beta.268",
+        "@crawlee/utils": "4.0.0-beta.268",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.1.0",
         "@vladfrangu/async_event_emitter": "^2.4.7",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "a4e49e8a8117acdb808c6cba5b05c3d56bf2b123"
+    "gitHead": "9462257a02015e7ef40ebc16d8fdd0665c9554fe"
 }

package/storages/batched-adds.js

@@ -14,8 +14,10 @@ import { activeStorageTransaction, withDirectStorageAccess } from './transaction
 export async function drainRequestBatches(options) {
     const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, log, processChunk, trackBackgroundBatches, } = options;
     const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';
+    // Leftovers are only reported when the caller waits anyway: draining an async producer just to find
+    // them could block indefinitely.
+    const reportsOverLimit = maxNewRequests !== undefined && waitForAllRequestsToBeAdded;
     let remainingBudget = maxNewRequests ?? Infinity;
-    const requestsOverLimit = [];
     // Never hand a chunk more than the budget allows, so an over-large final batch cannot overshoot.
     const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
     const chunks = peekableAsyncIterable(chunkedAsyncIterable(items, effectiveChunkSize));
@@ -28,12 +30,14 @@ export async function drainRequestBatches(options) {
         return processedRequests;
     };
     const buildResult = async (addedRequests, waitForAll) => {
-        if (maxNewRequests !== undefined) {
-            // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
-            // is left is still sitting in `items` rather than in a chunk we have seen.
-            for await (const item of items) {
-                requestsOverLimit.push(item);
-            }
+        if (!reportsOverLimit) {
+            return { addedRequests, waitForAllRequestsToBeAdded: waitForAll };
+        }
+        // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
+        // is left is still sitting in `items` rather than in a chunk we have seen.
+        const requestsOverLimit = [];
+        for await (const item of items) {
+            requestsOverLimit.push(item);
         }
         return { addedRequests, waitForAllRequestsToBeAdded: waitForAll, requestsOverLimit };
     };
@@ -55,9 +59,8 @@ export async function drainRequestBatches(options) {
         }
         return added;
     };
-    // With a budget we must drain everything before we can report what went over it; under `deferred` a
-    // writer that finishes after commit would have nowhere to put its journal entries.
-    const awaitsRemainder = waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
+    // Under `deferred` a writer that finishes after commit would have nowhere to put its journal entries.
+    const awaitsRemainder = waitForAllRequestsToBeAdded || deferred;
     // An un-awaited writer outlives the transaction scope it inherits, so it must not record into a
     // transaction that may already be closed. It writes directly - its write-through additions were never
     // going to be rolled back anyway - which means the requests it adds are not journaled.

package/storages/request-queue.d.ts

@@ -303,13 +303,11 @@ export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions
     waitBetweenBatchesMillis?: number;
     /**
      * If set, only this many *actually new* requests (i.e. not already present in the queue) will be added.
-     * Once the budget is reached, remaining requests from the iterable will be collected in
-     * {@link AddRequestsBatchedResult.requestsOverLimit|`requestsOverLimit`} instead.
+     * Once the budget is reached, remaining requests from the iterable are left unconsumed, or collected in
+     * {@link AddRequestsBatchedResult.requestsOverLimit|`requestsOverLimit`} if
+     * {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} is set.
      *
      * This is useful in combination with `maxRequestsPerCrawl` to avoid duplicate URLs consuming the budget.
-     *
-     * **Note:** Setting this option implicitly enables {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`},
-     * since all batches must complete before leftover requests can be accurately reported.
      */
     maxNewRequests?: number;
 }
@@ -335,7 +333,8 @@ export interface AddRequestsBatchedResult {
     /**
      * Requests from the input that were not added to the queue because the
      * {@link AddRequestsBatchedOptions.maxNewRequests|`maxNewRequests`} budget was reached.
-     * Empty when `maxNewRequests` is not set.
+     * `undefined` unless both `maxNewRequests` and
+     * {@link AddRequestsBatchedOptions.waitForAllRequestsToBeAdded|`waitForAllRequestsToBeAdded`} are set.
      */
     requestsOverLimit?: Source[];
 }