@crawlee/basic 4.0.0-beta.134 → 4.0.0-beta.136

package/index.d.ts

@@ -1,3 +1,2 @@
 export * from '@crawlee/core';
 export * from './internals/basic-crawler.js';
-export type { CheerioRoot, CheerioAPI, Cheerio, Element } from '@crawlee/utils/internal';

package/internals/basic-crawler.d.ts

@@ -367,93 +367,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     additionalHttpErrorStatusCodes?: number[];
 }
-/**
- * Provides a simple framework for parallel crawling of web pages.
- * The URLs to crawl are fed either from a static list of URLs
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
- *
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
- * download and data extraction functionality themselves.
- * If we want a crawler that already facilitates this functionality,
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
- *
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
- * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link IRequestManager|request manager} provided via the
- * {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is
- * itself a request manager). If no `requestManager` is provided, the crawler opens the default {@link RequestQueue}
- * either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called, or if the `requests`
- * parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
- *
- * To read requests from a read-only source such as a {@link RequestList} or {@link SitemapRequestLoader} while
- * still being able to enqueue new ones, combine the loader with a queue into a {@link RequestManagerTandem} using
- * {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`. The tandem
- * first processes URLs from the loader and automatically enqueues them into the queue, ensuring a single URL is not
- * crawled multiple times.
- *
- * > The legacy {@link BasicCrawlerOptions.requestList|`requestList`} and
- * > {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated. They are still accepted and
- * > folded into a single `requestManager` (combined into a tandem when both are given), but new code should use
- * > `requestManager` directly.
- *
- * The crawler finishes if there are no more {@link Request} objects to crawl.
- *
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
- * {@link ConcurrencySystem}.
- * Concurrency is tuned via the {@link BasicCrawlerOptions.minConcurrency|`minConcurrency`},
- * {@link BasicCrawlerOptions.maxConcurrency|`maxConcurrency`} and
- * {@link BasicCrawlerOptions.maxRequestsPerMinute|`maxRequestsPerMinute`} shortcuts, or, for finer control, by
- * injecting a pre-configured {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.
- *
- * **Example usage:**
- *
- * ```javascript
- * import { BasicCrawler, Dataset } from 'crawlee';
- *
- * // Create a crawler instance
- * const crawler = new BasicCrawler({
- *     async requestHandler({ request, sendRequest }) {
- *         // 'request' contains an instance of the Request class
- *         // Here we simply fetch the HTML of the page and store it to a dataset
- *         const { body } = await sendRequest({
- *             url: request.url,
- *             method: request.method,
- *             body: request.payload,
- *             headers: request.headers,
- *         });
- *
- *         await Dataset.pushData({
- *             url: request.url,
- *             html: body,
- *         })
- *     },
- * });
- *
- * // Enqueue the initial requests and run the crawler
- * await crawler.run([
- *     'http://www.example.com/page-1',
- *     'http://www.example.com/page-2',
- * ]);
- * ```
- * @category Crawlers
- */
-/**
- * Identifies a crawler instance for storage aliasing, `useState()` and status-message events.
- */
-interface CrawlerIdentity {
-    /**
-     * 0-based instantiation order across all crawlers in the process.
-     * Note that the value can be subject to race conditions between different script invocations.
-     */
-    readonly instanceIndex: number;
-    /** The user-supplied `id` option, or a fallback derived from `instanceIndex`. */
-    readonly id: string;
-    /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */
-    readonly hasExplicitId: boolean;
-}
 export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     #private;
-    protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
     /**
      * Tracks the number of crawler instances created. The first crawler uses the default
      * request queue; subsequent ones get their own queue via a unique alias so they don't
@@ -515,13 +430,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     hasFinishedBefore: boolean;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
-    protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     private requestHandlerTimeoutMillis;
     protected readonly internalTimeoutMillis: number;
-    protected readonly maxRequestRetries: number;
-    protected readonly maxCrawlDepth?: number;
-    protected readonly maxRequestsPerCrawl?: number;
     private get handledRequestsCount();
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
@@ -534,8 +444,9 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
-    protected readonly onSkippedRequest?: SkippedRequestCallback;
-    protected readonly identity: CrawlerIdentity;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
         contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
@@ -580,50 +491,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
     };
-    protected static optionsSchema: z.ZodObject<{
-        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
-        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
-        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
-        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
-        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
-        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
-        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
-        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
-        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
-        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
-        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
-        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
-            requestQueue: z.ZodOptional<z.ZodEnum<{
-                deferred: "deferred";
-                writeThrough: "writeThrough";
-            }>>;
-        }, z.core.$strict>]>>;
-        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;
-        configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;
-        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;
-        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
-        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
-        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
-        keepAlive: z.ZodOptional<z.ZodBoolean>;
-        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        id: z.ZodOptional<z.ZodString>;
-    }, z.core.$strict>;
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
@@ -745,9 +612,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     private validateRequestUserData;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
-    protected getPendingRequestCountApproximation(): Promise<number>;
-    protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;
-    protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
     private logOncePerRun;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue

package/internals/basic-crawler.js

@@ -1,8 +1,8 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
-import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
+import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
 import { getDomain } from 'tldts';
 import { z } from 'zod';
@@ -71,7 +71,7 @@ const addRequestsOptionsSchema = z.looseObject({
     onSkippedRequest: schemas.anyFunction.optional(),
 });
 export class BasicCrawler {
-    static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
+    static #CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
     /**
      * Tracks the number of crawler instances created. The first crawler uses the default
      * request queue; subsequent ones get their own queue via a unique alias so they don't
@@ -191,15 +191,15 @@ export class BasicCrawler {
         return this.#log;
     }
     requestHandler;
-    errorHandler;
-    failedRequestHandler;
+    #errorHandler;
+    #failedRequestHandler;
     // kept as TS-private: tests read it at runtime
     requestHandlerTimeoutMillis;
     internalTimeoutMillis;
-    maxRequestRetries;
-    maxCrawlDepth;
+    #maxRequestRetries;
+    #maxCrawlDepth;
     #sameDomainDelaySecs;
-    maxRequestsPerCrawl;
+    #maxRequestsPerCrawl;
     get handledRequestsCount() {
         return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;
     }
@@ -223,12 +223,15 @@ export class BasicCrawler {
     #transactionalStorageEnabled;
     /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
     #storageWritePolicy;
-    onSkippedRequest;
+    #onSkippedRequest;
     #closeEvents;
     #loggedPerRun = new Set();
     #robotsTxtFileCache;
-    identity;
+    #identity;
     #contextPipelineOptions;
+    /**
+     * @internal
+     */
     static optionsShape = {
         contextPipelineBuilder: schemas.anyObject.optional(),
         extendContext: schemas.anyFunction.optional(),
@@ -278,12 +281,12 @@ export class BasicCrawler {
         statistics: schemas.anyObject.optional(),
         id: z.string().optional(),
     };
-    static optionsSchema = z.strictObject(BasicCrawler.optionsShape);
+    static #optionsSchema = z.strictObject(BasicCrawler.optionsShape);
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
     constructor(options = {}) {
-        const parsedOptions = parseArgument(options, BasicCrawler.optionsSchema, 'BasicCrawlerOptions');
+        const parsedOptions = parseArgument(options, BasicCrawler.#optionsSchema, 'BasicCrawlerOptions');
         const { 
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
         requestList, 
@@ -329,7 +332,7 @@ export class BasicCrawler {
             // Initialize the Configuration instance to avoid lazy loading in the components
             serviceLocator.getConfiguration();
             const instanceIndex = BasicCrawler.instanceCount++;
-            this.identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
+            this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
             if (requestManager !== undefined) {
                 if (requestList !== undefined || requestQueue !== undefined) {
                     throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
@@ -361,12 +364,11 @@ export class BasicCrawler {
             this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
             this.#statusMessageCallback = statusMessageCallback;
             this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
-            this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
             this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
             this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
             this.requestHandler = requestHandler ?? this.router;
-            this.failedRequestHandler = failedRequestHandler;
-            this.errorHandler = errorHandler;
+            this.#failedRequestHandler = failedRequestHandler;
+            this.#errorHandler = errorHandler;
             if (requestHandlerTimeoutSecs) {
                 this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
             }
@@ -379,13 +381,13 @@ export class BasicCrawler {
             const transactionalStorageOption = transactionalStorage;
             this.#transactionalStorageEnabled = transactionalStorageOption !== false;
             this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
-            this.onSkippedRequest = onSkippedRequest;
+            this.#onSkippedRequest = onSkippedRequest;
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
                 serviceLocator.getConfiguration().internalTimeoutMillis ??
                     Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
-            this.maxRequestRetries = maxRequestRetries;
-            this.maxCrawlDepth = maxCrawlDepth;
+            this.#maxRequestRetries = maxRequestRetries;
+            this.#maxCrawlDepth = maxCrawlDepth;
             this.#sameDomainDelaySecs = sameDomainDelaySecs;
             this.#statisticsDep = OwnedOrInjected.resolve(statistics, 
             // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
@@ -394,7 +396,7 @@ export class BasicCrawler {
             () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
-                id: this.identity.id,
+                id: this.#identity.id,
             }));
             if (sessionPool && proxyConfiguration) {
                 this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
@@ -416,8 +418,8 @@ export class BasicCrawler {
                 this.requestHandlerTimeoutMillis = maxSignedInteger;
             }
             this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
-            this.maxRequestsPerCrawl = maxRequestsPerCrawl;
-            const isMaxPagesExceeded = () => this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;
+            this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
+            const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.handledRequestsCount;
             // eslint-disable-next-line prefer-const
             let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
             // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
@@ -484,7 +486,7 @@ export class BasicCrawler {
                 isTaskReadyFunction: async () => {
                     if (isMaxPagesExceeded()) {
                         this.logOncePerRun('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
-                            `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
+                            `${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
                         return false;
                     }
                     if (this.#unexpectedStop) {
@@ -496,7 +498,7 @@ export class BasicCrawler {
                 },
                 isFinishedFunction: async () => {
                     if (isMaxPagesExceeded()) {
-                        this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` +
+                        this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +
                             'and all requests that were in progress at that time have now finished. ' +
                             `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
                         return true;
@@ -573,7 +575,7 @@ export class BasicCrawler {
             this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
             request.state = RequestState.SKIPPED;
             request.noRetry = true;
-            await this.handleSkippedRequest({
+            await this.#handleSkippedRequest({
                 url: request.url,
                 reason: 'robotsTxt',
             });
@@ -673,7 +675,7 @@ export class BasicCrawler {
                     this.log.debug(message);
                     request.noRetry = true;
                     request.state = RequestState.SKIPPED;
-                    await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
+                    await this.#handleSkippedRequest({ url: request.url, reason: 'redirect' });
                     throw new ContextPipelineInterruptedError(message);
                 }
                 return context;
@@ -707,7 +709,7 @@ export class BasicCrawler {
         // Setting the status message is not a storage concern, so we intentionally don't route it
         // through the storage client anymore.
         serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
-            crawlerId: this.identity.id,
+            crawlerId: this.#identity.id,
             message,
             isStatusMessageTerminal: options.isStatusMessageTerminal,
             level: options.level,
@@ -930,7 +932,7 @@ export class BasicCrawler {
                 minCrawlDelaySecs: this.#sameDomainDelaySecs,
                 // What `sameDomainDelaySecs` has always meant: one clock for a site, subdomains included.
                 throttleBy: 'registrableDomain',
-                persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.identity.id}`,
+                persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,
             });
         }
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
@@ -957,7 +959,7 @@ export class BasicCrawler {
     async openOwnedRequestQueue() {
         // The first crawler instance uses the default queue (null identifier);
         // subsequent instances get their own queue via a unique alias so they don't collide.
-        const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
+        const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };
         const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
         return this.#ownedRequestQueue.set(requestQueue);
     }
@@ -1004,11 +1006,11 @@ export class BasicCrawler {
     }
     async useState(defaultValue = {}) {
         const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
-        if (this.identity.hasExplicitId) {
-            const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
+        if (this.#identity.hasExplicitId) {
+            const stateKey = `${BasicCrawler.#CRAWLEE_STATE_KEY}_${this.#identity.id}`;
             return kvs.getAutoSavedValue(stateKey, defaultValue);
         }
-        BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);
+        BasicCrawler.#useStateAnonymousIndices.add(this.#identity.instanceIndex);
         if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
             serviceLocator
                 .getLogger()
@@ -1017,30 +1019,30 @@ export class BasicCrawler {
                 'To fix this, provide a unique `id` option to each crawler instance. \n' +
                 'Example: new BasicCrawler({ id: "my-crawler-1", ... })');
         }
-        return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
+        return kvs.getAutoSavedValue(BasicCrawler.#CRAWLEE_STATE_KEY, defaultValue);
     }
-    async getPendingRequestCountApproximation() {
+    async #getPendingRequestCountApproximation() {
         return (await this.requestManager?.getPendingCount()) ?? 0;
     }
-    async calculateEnqueuedRequestLimit(explicitLimit) {
-        if (this.maxRequestsPerCrawl === undefined) {
+    async #calculateEnqueuedRequestLimit(explicitLimit) {
+        if (this.#maxRequestsPerCrawl === undefined) {
             return explicitLimit;
         }
-        const limit = Math.max(0, this.maxRequestsPerCrawl - this.handledRequestsCount - (await this.getPendingRequestCountApproximation()));
+        const limit = Math.max(0, this.#maxRequestsPerCrawl - this.handledRequestsCount - (await this.#getPendingRequestCountApproximation()));
         return Math.min(limit, explicitLimit ?? Infinity);
     }
-    async handleSkippedRequest(options) {
+    async #handleSkippedRequest(options) {
         // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
         // transaction scope, which rolls back - so the skip bookkeeping must write directly.
         await withDirectStorageAccess(async () => {
             if (options.reason === 'limit') {
                 this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
-                    `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
+                    `${this.#maxRequestsPerCrawl} requests and no further requests will be added.`);
             }
             if (options.reason === 'depth') {
-                this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
+                this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`);
             }
-            await this.onSkippedRequest?.(options);
+            await this.#onSkippedRequest?.(options);
         });
     }
     logOncePerRun(key, message, level = 'info') {
@@ -1079,7 +1081,7 @@ export class BasicCrawler {
         if (options.label !== undefined || options.userData !== undefined) {
             await this.validateRequestUserData({ label: options.label, userData: options.userData });
         }
-        const requestLimit = await this.calculateEnqueuedRequestLimit(options.limit);
+        const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);
         const strategy = options.strategy ?? EnqueueStrategy.All;
         const urlExcludePatternObjects = options.exclude?.length
             ? constructUrlPatternObjects(options.exclude)
@@ -1093,7 +1095,7 @@ export class BasicCrawler {
             ? buildEnqueueStrategyPatterns(options.baseUrl, strategy)
             : [];
         const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
-        const maxCrawlDepth = this.maxCrawlDepth;
+        const maxCrawlDepth = this.#maxCrawlDepth;
         const validateRequestUserData = this.validateRequestUserData.bind(this);
         const allSkipped = [];
         async function* filteredRequests() {
@@ -1155,7 +1157,7 @@ export class BasicCrawler {
                     : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
             }
             await Promise.all(allSkipped.map(async ({ url, reason }) => {
-                await this.handleSkippedRequest({ url, reason });
+                await this.#handleSkippedRequest({ url, reason });
                 await options.onSkippedRequest?.({ url, reason });
             }));
         }
@@ -1256,7 +1258,7 @@ export class BasicCrawler {
         this.#autoscaledPool = new AutoscaledPool({
             ...this.taskLoopOptions,
             concurrencySystem: this.#concurrencySystemDep.value,
-            consumer: this.identity,
+            consumer: this.#identity,
         });
         await this.getRequestManager();
     }
@@ -1611,7 +1613,7 @@ export class BasicCrawler {
         const shouldRetryRequest = this.canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
             await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
-            await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
+            await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
             error);
             if (error instanceof SessionError) {
                 crawlingContext.session?.retire();
@@ -1656,8 +1658,8 @@ export class BasicCrawler {
         const { id, url, method, uniqueKey } = crawlingContext.request;
         const message = this.getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
-        if (this.failedRequestHandler) {
-            await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
+        if (this.#failedRequestHandler) {
+            await this.#failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
             error);
         }
     }
@@ -1697,7 +1699,7 @@ export class BasicCrawler {
             return true;
         }
         // Ensure there are more retries available for the request
-        const maxRequestRetries = request.maxRetries ?? this.maxRequestRetries;
+        const maxRequestRetries = request.maxRetries ?? this.#maxRequestRetries;
         return request.retryCount < maxRequestRetries;
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.134",
+    "version": "4.0.0-beta.136",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.134",
-        "@crawlee/http-client": "4.0.0-beta.134",
-        "@crawlee/types": "4.0.0-beta.134",
-        "@crawlee/utils": "4.0.0-beta.134",
+        "@crawlee/core": "4.0.0-beta.136",
+        "@crawlee/http-client": "4.0.0-beta.136",
+        "@crawlee/types": "4.0.0-beta.136",
+        "@crawlee/utils": "4.0.0-beta.136",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.134"
+        "@crawlee/impit-client": "^4.0.0-beta.136"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "0f269d5b24ec2829f3d44c19a35973fa20d95999"
+    "gitHead": "d425bb9841885b4b71b84ae6fb0bafbdc4a7e978"
 }