@crawlee/basic 4.0.0-beta.122 → 4.0.0-beta.124

package/internals/basic-crawler.d.ts

@@ -1,9 +1,9 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
-import { ConcurrencySystem, Configuration, ContextPipeline, Dataset, EventManager, RequestQueue } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueUrlsOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
+import { ConcurrencySystem, Configuration, ContextPipeline, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
 import { BaseHttpClient } from '@crawlee/http-client';
-import type { Awaitable, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
+import type { Awaitable, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
-import type { ReadonlyDeep, SetRequired } from 'type-fest';
+import type { ReadonlyDeep } from 'type-fest';
 import { z } from 'zod';
 import { TimeoutError } from '@apify/timeout';
 export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
@@ -137,7 +137,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     maxRequestRetries?: number;
     /**
-     * Indicates how much time (in seconds) to wait before crawling another same domain request.
+     * Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
+     * paced together with the site they belong to.
+     *
+     * Wraps the crawler's request manager in a {@link ThrottlingRequestManager}; pass one as `requestManager`
+     * yourself to configure it further.
      * @default 0
      */
     sameDomainDelaySecs?: number;
@@ -733,6 +737,11 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
      * the `waitForAllRequestsToBeAdded` promise you get in the response object.
      *
+     * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
+     * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
+     * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
+     * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
+     *
      * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
      *
      * @param requests The requests to add
@@ -819,22 +828,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Fetches the next request to process from the underlying request provider.
      */
     private fetchNextRequest;
-    /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
-     */
-    private delayRequest;
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
     private handleRequest;
-    /**
-     * Wrapper around the crawling context's `enqueueLinks` method:
-     * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
-     * @internal
-     */
-    protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestManager: IRequestManager): Promise<BatchAddRequestsResult>;
     /**
      * Generator function that yields requests injected with the given crawl depth.
      * @internal
@@ -893,7 +888,7 @@ export interface CreateContextOptions {
     session: ISession;
     proxyInfo?: ProxyInfo;
 }
-export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
+export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
 }
 export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
 }

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, Configuration, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createStorageTransaction, CriticalError, currentStorageTransaction, Dataset, enqueueLinks, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -43,6 +43,33 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;
 const deferredCleanupKey = Symbol('deferredCleanup');
 // The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
 export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
+const urlPatternSchema = z.union([
+    z.string(),
+    z.instanceof(RegExp),
+    schemas.objectWithKeys(['glob']),
+    schemas.objectWithKeys(['regexp']),
+]);
+// `looseObject` (rather than `strictObject`) lets subclasses forward their own extraction-only options
+// (e.g. `selector`) straight through without having to strip them out first.
+const addRequestsOptionsSchema = z.looseObject({
+    forefront: z.boolean().optional(),
+    cache: z.boolean().optional(),
+    waitForAllRequestsToBeAdded: z.boolean().optional(),
+    batchSize: schemas.anyNumber.optional(),
+    waitBetweenBatchesMillis: schemas.anyNumber.optional(),
+    maxNewRequests: schemas.anyNumber.optional(),
+    limit: schemas.anyNumber.optional(),
+    baseUrl: z.string().optional(),
+    userData: schemas.anyObject.optional(),
+    label: z.string().optional(),
+    sessionId: z.string().optional(),
+    skipNavigation: z.boolean().optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').min(1).optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    transformRequestFunction: schemas.anyFunction.optional(),
+    strategy: z.enum(EnqueueStrategy).optional(),
+    onSkippedRequest: schemas.anyFunction.optional(),
+});
 export class BasicCrawler {
     static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
     /**
@@ -171,8 +198,7 @@ export class BasicCrawler {
     internalTimeoutMillis;
     maxRequestRetries;
     maxCrawlDepth;
-    #sameDomainDelayMillis;
-    #domainAccessedTime;
+    #sameDomainDelaySecs;
     maxRequestsPerCrawl;
     get handledRequestsCount() {
         return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
@@ -308,6 +334,12 @@ export class BasicCrawler {
                 if (requestList !== undefined || requestQueue !== undefined) {
                     throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
                 }
+                // Both would pace the same domains, from different keys and with no idea of one another.
+                if (sameDomainDelaySecs > 0 && supportsDomainThrottling(requestManager)) {
+                    throw new Error('The `sameDomainDelaySecs` option cannot be combined with a `requestManager` that throttles ' +
+                        'per domain on its own. Configure the delay on the manager instead, via the ' +
+                        '`minCrawlDelaySecs` option of `ThrottlingRequestManager`.');
+                }
                 this.requestManager = requestManager;
             }
             else if (requestList !== undefined && requestQueue !== undefined) {
@@ -328,7 +360,6 @@ export class BasicCrawler {
             this.proxyConfiguration = proxyConfiguration;
             this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
             this.#statusMessageCallback = statusMessageCallback;
-            this.#domainAccessedTime = new Map();
             this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
             this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
             this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
@@ -355,7 +386,7 @@ export class BasicCrawler {
                     Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
-            this.#sameDomainDelayMillis = sameDomainDelaySecs * 1000;
+            this.#sameDomainDelaySecs = sameDomainDelaySecs;
             this.#statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
@@ -395,7 +426,7 @@ export class BasicCrawler {
                     if (!source)
                         throw new Error('Request provider is not initialized!');
                     const request = await this.resolveRequest();
-                    if (!request || this.delayRequest(request, source)) {
+                    if (!request) {
                         return;
                     }
                     // Started here, rather than in `handleRequest`, so that a failure during context pipeline
@@ -598,17 +629,13 @@ export class BasicCrawler {
         return { session, proxyInfo: session?.proxyInfo };
     }
     async createContextHelpers({ request, session }) {
-        const enqueueLinksWrapper = async (options) => {
-            const requestManager = await this.getRequestManager();
-            return await this.enqueueLinksWithCrawlDepth(options, request, requestManager);
-        };
         const addRequests = async (requests, options = {}) => {
             const newCrawlDepth = request.crawlDepth + 1;
             const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
-            await this.addRequests(requestsGenerator, options);
+            return await this.addRequests(requestsGenerator, options);
         };
         const sendRequest = createSendRequest(this.httpClient, request, session);
-        return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };
+        return { addRequests, sendRequest };
     }
     buildFinalContextPipeline() {
         const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
@@ -744,8 +771,14 @@ export class BasicCrawler {
             // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
             const shouldPurge = purgeRequestQueue !== false;
             const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
-            if (managerToPurge?.purge && shouldPurge) {
-                await managerToPurge.purge();
+            if (shouldPurge) {
+                await managerToPurge?.purge?.();
+                // The per-domain queues a `sameDomainDelaySecs` wrapper created are the crawler's own, whatever
+                // sits underneath them - so they are emptied even when the manager they wrap is spared. Purging
+                // the wrapper itself has already covered them.
+                if (this.requestManager instanceof ThrottlingRequestManager && managerToPurge !== this.requestManager) {
+                    await this.requestManager.purgeDomainQueues();
+                }
             }
             // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
             await this.#statsDep.ifOwned(async (stats) => {
@@ -883,6 +916,19 @@ export class BasicCrawler {
         if (!this.requestManager) {
             this.requestManager = await this.openOwnedRequestQueue();
         }
+        // Wrapped here rather than in the constructor, because the manager being wrapped may only be opened at
+        // this point - and because everything that enqueues goes through here first, so nothing slips past the
+        // wrapper into the queue it hides.
+        if (this.#sameDomainDelaySecs > 0 && !supportsDomainThrottling(this.requestManager)) {
+            this.requestManager = new ThrottlingRequestManager({
+                inner: this.requestManager,
+                domains: 'all',
+                minCrawlDelaySecs: this.#sameDomainDelaySecs,
+                // What `sameDomainDelaySecs` has always meant: one clock for a site, subdomains included.
+                throttleBy: 'registrableDomain',
+                persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.identity.id}`,
+            });
+        }
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
         // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
         // but guard so we do not re-issue it on every call.
@@ -1005,6 +1051,11 @@ export class BasicCrawler {
      * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
      * the `waitForAllRequestsToBeAdded` promise you get in the response object.
      *
+     * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
+     * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
+     * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
+     * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
+     *
      * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
      *
      * @param requests The requests to add
@@ -1012,55 +1063,95 @@ export class BasicCrawler {
      */
     async addRequests(requests, options = {}) {
         await this.getRequestManager();
-        const requestLimit = await this.calculateEnqueuedRequestLimit();
-        const skippedBecauseOfRobots = new Set();
-        const skippedBecauseOfMaxCrawlDepth = new Set();
-        const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
-        const maxCrawlDepth = this.maxCrawlDepth;
-        const validateRequestUserData = this.validateRequestUserData.bind(this);
         if (!isIterable(requests) && !isAsyncIterable(requests)) {
             throw new Error(`Expected an iterable or async iterable, got ${getObjectType(requests)}`);
         }
+        parseArgument(options, addRequestsOptionsSchema, 'EnqueueUrlsOptions');
+        // `label`/`userData` apply to every request this call produces, so a single upfront validation
+        // against the label's schema covers them all and fails the whole call fast, rather than failing
+        // lazily once the generator below is drained. Skipped when neither is set - each item still gets
+        // its own per-item validation below, and validating an absent label/userData here would spuriously
+        // check them against a registered default-route schema.
+        if (options.label !== undefined || options.userData !== undefined) {
+            await this.validateRequestUserData({ label: options.label, userData: options.userData });
+        }
+        const requestLimit = await this.calculateEnqueuedRequestLimit(options.limit);
+        const strategy = options.strategy ?? EnqueueStrategy.All;
+        const urlExcludePatternObjects = options.exclude?.length
+            ? constructUrlPatternObjects(options.exclude)
+            : [];
+        const urlPatternObjects = options.include?.length
+            ? constructUrlPatternObjects(options.include)
+            : [];
+        // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
+        // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
+        const enqueueStrategyPatterns = options.baseUrl
+            ? buildEnqueueStrategyPatterns(options.baseUrl, strategy)
+            : [];
+        const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
+        const maxCrawlDepth = this.maxCrawlDepth;
+        const validateRequestUserData = this.validateRequestUserData.bind(this);
+        const allSkipped = [];
         async function* filteredRequests() {
             for await (const request of requests) {
-                const url = typeof request === 'string' ? request : request.url;
-                if (maxCrawlDepth !== undefined && request.crawlDepth > maxCrawlDepth) {
-                    skippedBecauseOfMaxCrawlDepth.add(url);
+                const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
+                if (!requestOptions) {
+                    continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
+                }
+                if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
+                    allSkipped.push({ url: requestOptions.url, reason: 'depth' });
                     continue;
                 }
-                if (await isAllowedBasedOnRobotsTxtFile(url)) {
-                    await validateRequestUserData(request);
-                    yield request;
+                if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
+                    allSkipped.push({ url: requestOptions.url, reason: 'robotsTxt' });
+                    continue;
                 }
-                else {
-                    skippedBecauseOfRobots.add(url);
+                const onSkippedFilterUrl = (url) => allSkipped.push({ url, reason: 'filters' });
+                // Filter by user patterns first (with exclude)...
+                let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedFilterUrl);
+                // ...then filter by the enqueue strategy (making this an AND check)
+                filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedFilterUrl);
+                if (filtered.length === 0) {
+                    continue;
                 }
+                let [finalOptions] = filtered;
+                if (options.transformRequestFunction) {
+                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ url: r.url, reason: r.skippedReason ?? 'transform' }));
+                    if (transformed.length === 0) {
+                        continue;
+                    }
+                    [finalOptions] = transformed;
+                }
+                await validateRequestUserData(finalOptions);
+                yield new Request(finalOptions);
             }
         }
         const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
-            ...options,
+            forefront: options.forefront,
+            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
+            batchSize: options.batchSize,
+            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
             maxNewRequests: requestLimit,
         });
-        // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit)
-        const skippedBecauseOfLimit = result.requestsOverLimit ?? [];
-        if (skippedBecauseOfRobots.size > 0) {
-            this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
-                skipped: [...skippedBecauseOfRobots],
-            });
-        }
-        if (skippedBecauseOfRobots.size > 0 ||
-            skippedBecauseOfLimit.length > 0 ||
-            skippedBecauseOfMaxCrawlDepth.size > 0) {
-            await Promise.all([...skippedBecauseOfRobots]
-                .map((url) => {
-                return this.handleSkippedRequest({ url, reason: 'robotsTxt' });
-            })
-                .concat(skippedBecauseOfLimit.map((request) => {
-                const url = typeof request === 'string' ? request : request.url;
-                return this.handleSkippedRequest({ url, reason: 'limit' });
-            }), [...skippedBecauseOfMaxCrawlDepth].map((url) => {
-                return this.handleSkippedRequest({ url, reason: 'depth' });
-            })));
+        // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
+        // explicit `limit` option)
+        for (const request of result.requestsOverLimit ?? []) {
+            allSkipped.push({ url: typeof request === 'string' ? request : request.url, reason: 'limit' });
+        }
+        if (allSkipped.length > 0) {
+            const skippedRobotsUrls = allSkipped.filter((s) => s.reason === 'robotsTxt').map((s) => s.url);
+            if (skippedRobotsUrls.length > 0) {
+                this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
+            }
+            // Only log the limit message when an explicit `limit` was passed (not the internal
+            // `maxRequestsPerCrawl`-derived one), and only once per call.
+            if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
+                this.log.info(`Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`);
+            }
+            await Promise.all(allSkipped.map(async ({ url, reason }) => {
+                await this.handleSkippedRequest({ url, reason });
+                await options.onSkippedRequest?.({ url, reason });
+            }));
         }
         return result;
     }
@@ -1363,30 +1454,6 @@ export class BasicCrawler {
         }
         return this.requestManager.fetchNextRequest();
     }
-    /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
-     */
-    delayRequest(request, source) {
-        const domain = getDomain(request.url);
-        if (!domain || !request) {
-            return false;
-        }
-        const now = Date.now();
-        const lastAccessTime = this.#domainAccessedTime.get(domain);
-        if (!lastAccessTime || now - lastAccessTime >= this.#sameDomainDelayMillis) {
-            this.#domainAccessedTime.set(domain, now);
-            return false;
-        }
-        const delay = lastAccessTime + this.#sameDomainDelayMillis - now;
-        this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
-        setTimeout(async () => {
-            this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
-            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
-        }, delay);
-        return true;
-    }
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
     async handleRequest(crawlingContext, requestSource, request) {
         // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
@@ -1459,52 +1526,6 @@ export class BasicCrawler {
             }
         }
     }
-    /**
-     * Wrapper around the crawling context's `enqueueLinks` method:
-     * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
-     * @internal
-     */
-    async enqueueLinksWithCrawlDepth(options, request, requestManager) {
-        const transformRequestFunctionWrapper = (requestOptions) => {
-            requestOptions.crawlDepth = request.crawlDepth + 1;
-            if (this.maxCrawlDepth !== undefined && requestOptions.crawlDepth > this.maxCrawlDepth) {
-                // Setting `skippedReason` before returning `false` ensures that `reportSkippedRequests`
-                // reports `'depth'` as the reason (via `request.skippedReason ?? reason` fallback),
-                // rather than the generic `'transform'` reason.
-                requestOptions.skippedReason = 'depth';
-                return false;
-            }
-            // After injecting the crawlDepth, we call the user-provided transform function, if there is one.
-            return options.transformRequestFunction?.(requestOptions) ?? requestOptions;
-        };
-        // Create a request-scoped callback that logs enqueueLimit once per request handler call
-        // Only log if an explicit limit was passed to enqueueLinks (not the internal maxRequestsPerCrawl-derived limit)
-        let loggedEnqueueLimitForThisRequest = false;
-        const onSkippedRequest = async (skippedOptions) => {
-            if (skippedOptions.reason === 'enqueueLimit') {
-                if (!loggedEnqueueLimitForThisRequest && options.limit !== undefined) {
-                    this.log.info(`Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`);
-                    loggedEnqueueLimitForThisRequest = true;
-                }
-            }
-            await this.handleSkippedRequest(skippedOptions);
-        };
-        // `enqueueLinks` applies `options.label`/`options.userData` to every newly enqueued request, so a single
-        // validation against the label's schema covers them all (a no-op unless the router declares a schema).
-        await this.validateRequestUserData({ label: options.label, userData: options.userData });
-        return await enqueueLinks({
-            requestManager,
-            robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
-            respectRobotsTxtFile: this.#respectRobotsTxtFile,
-            onSkippedRequest,
-            limit: await this.calculateEnqueuedRequestLimit(options.limit),
-            // Allow user options to override defaults set above ⤴
-            ...options,
-            transformRequestFunction: transformRequestFunctionWrapper,
-        });
-    }
     /**
      * Generator function that yields requests injected with the given crawl depth.
      * @internal

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.122",
+    "version": "4.0.0-beta.124",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.122",
-        "@crawlee/http-client": "4.0.0-beta.122",
-        "@crawlee/types": "4.0.0-beta.122",
-        "@crawlee/utils": "4.0.0-beta.122",
+        "@crawlee/core": "4.0.0-beta.124",
+        "@crawlee/http-client": "4.0.0-beta.124",
+        "@crawlee/types": "4.0.0-beta.124",
+        "@crawlee/utils": "4.0.0-beta.124",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.122"
+        "@crawlee/impit-client": "^4.0.0-beta.124"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
+    "gitHead": "0694ee1b94c755b98141671baa93cc363f2bf8e3"
 }