@crawlee/basic 3.17.1-beta.79 → 3.17.1-beta.80

package/internals/basic-crawler.d.ts

@@ -266,6 +266,9 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      * 2. because they don't match enqueueLinks filters,
      * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
      * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
+     *
+     * When `enqueueLinks` is called with its own `onSkippedRequest` callback, both are invoked — this one first,
+     * then the `enqueueLinks` one.
      */
     onSkippedRequest?: SkippedRequestCallback;
     /** @internal */
@@ -636,8 +639,9 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
     /**
      * Wrapper around the crawling context's `enqueueLinks` method:
      * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
+     * - Combines the `enqueueLinks` options with the crawler configuration - the user options take precedence,
+     *   but the crawler limits are always enforced (the `limit` is capped by the remaining `maxRequestsPerCrawl`
+     *   budget and skipped requests are always reported to the crawler too).
      * @internal
      */
     protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestQueue: RequestProvider): Promise<BatchAddRequestsResult>;

package/internals/basic-crawler.js

@@ -1191,8 +1191,9 @@ class BasicCrawler {
     /**
      * Wrapper around the crawling context's `enqueueLinks` method:
      * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
+     * - Combines the `enqueueLinks` options with the crawler configuration - the user options take precedence,
+     *   but the crawler limits are always enforced (the `limit` is capped by the remaining `maxRequestsPerCrawl`
+     *   budget and skipped requests are always reported to the crawler too).
      * @internal
      */
     async enqueueLinksWithCrawlDepth(options, request, requestQueue) {
@@ -1206,29 +1207,34 @@ class BasicCrawler {
             // Its return value is passed through as is, so a falsy one still skips the request.
             return options.transformRequestFunction ? options.transformRequestFunction(newRequest) : newRequest;
         };
+        const limit = this.calculateEnqueuedRequestLimit(options.limit);
         // Create a request-scoped callback that logs enqueueLimit once per request handler call
         // Only log if an explicit limit was passed to enqueueLinks (not the internal maxRequestsPerCrawl-derived limit)
         let loggedEnqueueLimitForThisRequest = false;
         const onSkippedRequest = async (skippedOptions) => {
             if (skippedOptions.reason === 'enqueueLimit') {
                 if (!loggedEnqueueLimitForThisRequest && options.limit !== undefined) {
-                    this.log.info(`Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`);
+                    this.log.info(limit === options.limit
+                        ? `Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`
+                        : `Skipping URLs in the handler for ${request.url} due to the remaining maxRequestsPerCrawl budget of ${limit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
                     loggedEnqueueLimitForThisRequest = true;
                 }
             }
             await this.handleSkippedRequest(skippedOptions);
+            await options.onSkippedRequest?.(skippedOptions);
         };
         // `enqueueLinks` applies `options.label`/`options.userData` to every newly enqueued request, so a single
         // validation against the label's schema covers them all (a no-op unless the router declares a schema).
         await this.validateRequestUserData({ label: options.label, userData: options.userData });
         return (0, core_1.enqueueLinks)({
-            requestQueue,
-            robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
-            respectRobotsTxtFile: this.respectRobotsTxtFile,
-            onSkippedRequest,
-            limit: this.calculateEnqueuedRequestLimit(options.limit),
-            // Allow user options to override defaults set above ⤴
             ...options,
+            // The options below are merged with the user options, so an explicitly `undefined` value
+            // (e.g. `enqueueLinks({ urls, limit: config.limit })`) cannot discard the crawler defaults ⤵
+            requestQueue: options.requestQueue ?? requestQueue,
+            robotsTxtFile: options.robotsTxtFile ?? (await this.getRobotsTxtFileForUrl(request.url)),
+            respectRobotsTxtFile: options.respectRobotsTxtFile ?? this.respectRobotsTxtFile,
+            onSkippedRequest,
+            limit,
             transformRequestFunction: transformRequestFunctionWrapper,
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "3.17.1-beta.79",
+    "version": "3.17.1-beta.80",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -49,9 +49,9 @@
         "@apify/log": "^2.4.0",
         "@apify/timeout": "^0.4.0",
         "@apify/utilities": "^2.7.10",
-        "@crawlee/core": "3.17.1-beta.79",
-        "@crawlee/types": "3.17.1-beta.79",
-        "@crawlee/utils": "3.17.1-beta.79",
+        "@crawlee/core": "3.17.1-beta.80",
+        "@crawlee/types": "3.17.1-beta.80",
+        "@crawlee/utils": "3.17.1-beta.80",
         "csv-stringify": "^6.2.0",
         "fs-extra": "^11.0.0",
         "got-scraping": "^4.2.1",
@@ -67,5 +67,5 @@
             }
         }
     },
-    "gitHead": "267c92b57f2cc05380a2d8e0323fe9f1ee35b142"
+    "gitHead": "5b3c2d52f128643de6f515e87eeb531d55a187f0"
 }