@crawlee/basic 4.0.0-beta.151 → 4.0.0-beta.153

package/internals/basic-crawler.js

@@ -1118,6 +1118,30 @@ export class BasicCrawler {
         // otherwise the URL alone is enough to build the callback argument and to log with.
         const hasSkippedRequestCallback = this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
         const keepSkippedSource = (source) => (hasSkippedRequestCallback ? source : source.url);
+        const reportSkippedRequests = async () => {
+            const skippedRequests = allSkipped.splice(0);
+            if (skippedRequests.length === 0) {
+                return;
+            }
+            const skippedRobotsUrls = skippedRequests
+                .filter((s) => s.reason === 'robotsTxt')
+                .map(({ source }) => (typeof source === 'string' ? source : source.url));
+            if (skippedRobotsUrls.length > 0) {
+                this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
+            }
+            // Only log the limit message when an explicit `limit` was passed (not the internal
+            // `maxRequestsPerCrawl`-derived one), and only once per call.
+            if (options.limit !== undefined && skippedRequests.some((s) => s.reason === 'limit')) {
+                this.log.info(requestLimit === options.limit
+                    ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
+                    : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
+            }
+            await Promise.all(skippedRequests.map(async ({ source, reason }) => {
+                const args = createSkippedRequestArgs(source, reason);
+                await this.#handleSkippedRequest(args);
+                await options.onSkippedRequest?.(args);
+            }));
+        };
         async function* filteredRequests() {
             for await (const request of requests) {
                 const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
@@ -1164,27 +1188,15 @@ export class BasicCrawler {
         for (const request of result.requestsOverLimit ?? []) {
             allSkipped.push({ source: request, reason: 'limit' });
         }
-        if (allSkipped.length > 0) {
-            const skippedRobotsUrls = allSkipped
-                .filter((s) => s.reason === 'robotsTxt')
-                .map(({ source }) => (typeof source === 'string' ? source : source.url));
-            if (skippedRobotsUrls.length > 0) {
-                this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
-            }
-            // Only log the limit message when an explicit `limit` was passed (not the internal
-            // `maxRequestsPerCrawl`-derived one), and only once per call.
-            if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
-                this.log.info(requestLimit === options.limit
-                    ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
-                    : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
-            }
-            await Promise.all(allSkipped.map(async ({ source, reason }) => {
-                const args = createSkippedRequestArgs(source, reason);
-                await this.#handleSkippedRequest(args);
-                await options.onSkippedRequest?.(args);
-            }));
-        }
-        return result;
+        await reportSkippedRequests();
+        const waitForAllRequestsToBeAdded = result.waitForAllRequestsToBeAdded.then(async (addedRequests) => {
+            await reportSkippedRequests();
+            return addedRequests;
+        });
+        // Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
+        // when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
+        void waitForAllRequestsToBeAdded.catch(() => { });
+        return { ...result, waitForAllRequestsToBeAdded };
     }
     /**
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.151",
+    "version": "4.0.0-beta.153",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.151",
-        "@crawlee/http-client": "4.0.0-beta.151",
-        "@crawlee/types": "4.0.0-beta.151",
-        "@crawlee/utils": "4.0.0-beta.151",
+        "@crawlee/core": "4.0.0-beta.153",
+        "@crawlee/http-client": "4.0.0-beta.153",
+        "@crawlee/types": "4.0.0-beta.153",
+        "@crawlee/utils": "4.0.0-beta.153",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.151"
+        "@crawlee/impit-client": "^4.0.0-beta.153"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "1865bf2b241da81d8ed6f1215c59f9d0b88ec553"
+    "gitHead": "ed8a71673d628a52574d82e30f098cad1fe7a451"
 }