@crawlee/basic 4.0.0-beta.149 → 4.0.0-beta.150

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, PersistentRateLimitError, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createSkippedRequestArgs, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, PersistentRateLimitError, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -608,7 +608,7 @@ export class BasicCrawler {
             request.state = RequestState.SKIPPED;
             request.noRetry = true;
             await this.#handleSkippedRequest({
-                url: request.url,
+                request,
                 reason: 'robotsTxt',
             });
             throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);
@@ -707,7 +707,7 @@ export class BasicCrawler {
                     this.log.debug(message);
                     request.noRetry = true;
                     request.state = RequestState.SKIPPED;
-                    await this.#handleSkippedRequest({ url: request.url, reason: 'redirect' });
+                    await this.#handleSkippedRequest({ request, reason: 'redirect' });
                     throw new ContextPipelineInterruptedError(message);
                 }
                 return context;
@@ -1114,6 +1114,10 @@ export class BasicCrawler {
         const maxCrawlDepth = this.#maxCrawlDepth;
         const validateRequestUserData = this.validateRequestUserData.bind(this);
         const allSkipped = [];
+        // A skipped source (which can carry arbitrary userData) is only retained if something reads it -
+        // otherwise the URL alone is enough to build the callback argument and to log with.
+        const hasSkippedRequestCallback = this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
+        const keepSkippedSource = (source) => (hasSkippedRequestCallback ? source : source.url);
         async function* filteredRequests() {
             for await (const request of requests) {
                 const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
@@ -1121,24 +1125,24 @@ export class BasicCrawler {
                     continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
                 }
                 if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
-                    allSkipped.push({ url: requestOptions.url, reason: 'depth' });
+                    allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });
                     continue;
                 }
                 if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
-                    allSkipped.push({ url: requestOptions.url, reason: 'robotsTxt' });
+                    allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });
                     continue;
                 }
-                const onSkippedFilterUrl = (url) => allSkipped.push({ url, reason: 'filters' });
+                const onSkippedByFilter = (opts) => allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });
                 // Filter by user patterns first (with exclude)...
-                let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedFilterUrl);
+                let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
                 // ...then filter by the enqueue strategy (making this an AND check)
-                filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedFilterUrl);
+                filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedByFilter);
                 if (filtered.length === 0) {
                     continue;
                 }
                 let [finalOptions] = filtered;
                 if (options.transformRequestFunction) {
-                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ url: r.url, reason: r.skippedReason ?? 'transform' }));
+                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: r.skippedReason ?? 'transform' }));
                     if (transformed.length === 0) {
                         continue;
                     }
@@ -1158,10 +1162,12 @@ export class BasicCrawler {
         // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
         // explicit `limit` option)
         for (const request of result.requestsOverLimit ?? []) {
-            allSkipped.push({ url: typeof request === 'string' ? request : request.url, reason: 'limit' });
+            allSkipped.push({ source: request, reason: 'limit' });
         }
         if (allSkipped.length > 0) {
-            const skippedRobotsUrls = allSkipped.filter((s) => s.reason === 'robotsTxt').map((s) => s.url);
+            const skippedRobotsUrls = allSkipped
+                .filter((s) => s.reason === 'robotsTxt')
+                .map(({ source }) => (typeof source === 'string' ? source : source.url));
             if (skippedRobotsUrls.length > 0) {
                 this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
             }
@@ -1172,9 +1178,10 @@ export class BasicCrawler {
                     ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
                     : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
             }
-            await Promise.all(allSkipped.map(async ({ url, reason }) => {
-                await this.#handleSkippedRequest({ url, reason });
-                await options.onSkippedRequest?.({ url, reason });
+            await Promise.all(allSkipped.map(async ({ source, reason }) => {
+                const args = createSkippedRequestArgs(source, reason);
+                await this.#handleSkippedRequest(args);
+                await options.onSkippedRequest?.(args);
             }));
         }
         return result;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.149",
+    "version": "4.0.0-beta.150",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.149",
-        "@crawlee/http-client": "4.0.0-beta.149",
-        "@crawlee/types": "4.0.0-beta.149",
-        "@crawlee/utils": "4.0.0-beta.149",
+        "@crawlee/core": "4.0.0-beta.150",
+        "@crawlee/http-client": "4.0.0-beta.150",
+        "@crawlee/types": "4.0.0-beta.150",
+        "@crawlee/utils": "4.0.0-beta.150",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.149"
+        "@crawlee/impit-client": "^4.0.0-beta.150"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "c88124c43b4c366d4b0ade01da5e90843d59c582"
+    "gitHead": "ef7e9fb2e2fcaa0f74e60a30fa8ae61c60c64626"
 }