@crawlee/core 4.0.0-beta.255 → 4.0.0-beta.256

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.255",
+    "version": "4.0.0-beta.256",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.1.0",
-        "@crawlee/fs-storage": "4.0.0-beta.255",
-        "@crawlee/http-client": "4.0.0-beta.255",
-        "@crawlee/types": "4.0.0-beta.255",
-        "@crawlee/utils": "4.0.0-beta.255",
+        "@crawlee/fs-storage": "4.0.0-beta.256",
+        "@crawlee/http-client": "4.0.0-beta.256",
+        "@crawlee/types": "4.0.0-beta.256",
+        "@crawlee/utils": "4.0.0-beta.256",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.1.0",
         "@vladfrangu/async_event_emitter": "^2.4.7",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "e1b2f6c415f90b3eca1686a39a8af5272b54046e"
+    "gitHead": "06cbfcc77b373a93f1a6a048e8115433db98c862"
 }

package/storages/batched-adds.d.ts

@@ -1,4 +1,5 @@
 import type { ProcessedRequest } from '@crawlee/types';
+import type { CrawleeLogger } from '../log.js';
 import type { Source } from '../request.js';
 import type { AddRequestsBatchedResult } from './request-queue.js';
 export interface DrainRequestBatchesOptions<TItem extends Source> {
@@ -11,6 +12,7 @@ export interface DrainRequestBatchesOptions<TItem extends Source> {
     waitBetweenBatchesMillis: number;
     waitForAllRequestsToBeAdded: boolean;
     maxNewRequests?: number;
+    log: CrawleeLogger;
     /**
      * Adds a single chunk and reports what it processed.
      *

package/storages/batched-adds.js

@@ -12,7 +12,7 @@ import { activeStorageTransaction, withDirectStorageAccess } from './transaction
  * so that is read from the ambient transaction rather than asked of the caller.
  */
 export async function drainRequestBatches(options) {
-    const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, processChunk, trackBackgroundBatches, } = options;
+    const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, log, processChunk, trackBackgroundBatches, } = options;
     const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';
     let remainingBudget = maxNewRequests ?? Infinity;
     const requestsOverLimit = [];
@@ -64,8 +64,13 @@ export async function drainRequestBatches(options) {
     // See `StorageTransactionView.enqueuedUrls`.
     const remainder = awaitsRemainder ? processRemainingChunks() : withDirectStorageAccess(processRemainingChunks);
     // The caller is not obliged to await `remainder`, so give it a handler of its own - an unhandled
-    // rejection here would otherwise take the process down.
-    trackBackgroundBatches?.(remainder.catch(() => { }));
+    // rejection here would otherwise take the process down. Awaited failures reach the caller, so only
+    // background ones get logged.
+    trackBackgroundBatches?.(remainder.catch((error) => {
+        if (!awaitsRemainder) {
+            log.exception(error, 'A batch of requests could not be added to the queue in the background');
+        }
+    }));
     if (awaitsRemainder) {
         addedRequests.push(...(await remainder));
     }

package/storages/request-queue.js

@@ -498,6 +498,7 @@ export class RequestQueue {
             waitBetweenBatchesMillis,
             waitForAllRequestsToBeAdded,
             maxNewRequests,
+            log: this.log,
             /**
              * Requests the backend reports as unprocessed are warned about and skipped rather than retried:
              * `unprocessedRequests` is what remains after the backend's own transient-error handling - a