@crawlee/basic 4.0.0-beta.80 → 4.0.0-beta.81

package/internals/basic-crawler.d.ts

@@ -1,4 +1,4 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IRequestLoader, IRequestManager, ProxyConfiguration, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier } from '@crawlee/core';
 import { AutoscaledPool, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
 import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -252,7 +252,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * If set, the crawler will be configured for all connections to use
      * the Proxy URLs provided and rotated according to the configuration.
      */
-    proxyConfiguration?: ProxyConfiguration;
+    proxyConfiguration?: IProxyConfiguration;
     /**
      * Custom configuration to use for this crawler.
      * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
@@ -409,7 +409,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
-    sessionPool: ISessionPool;
+    readonly sessionPool: ISessionPool;
     /**
      * Set when the crawler constructed its own {@link SessionPool} (no `sessionPool` option was provided).
      * Holds the same instance as `sessionPool`, but typed as the concrete class so the crawler can call
@@ -438,10 +438,10 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     autoscaledPool?: AutoscaledPool;
     /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
+     * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
      */
-    proxyConfiguration?: ProxyConfiguration;
+    readonly proxyConfiguration?: IProxyConfiguration;
     /**
      * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
@@ -463,33 +463,29 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
     running: boolean;
     hasFinishedBefore: boolean;
-    protected unexpectedStop: boolean;
+    private unexpectedStop;
     get log(): CrawleeLogger;
-    protected requestHandler: RequestHandler<ExtendedContext>;
-    protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    protected requestHandlerTimeoutMillis: number;
-    protected internalTimeoutMillis: number;
-    protected maxRequestRetries: number;
-    protected maxCrawlDepth?: number;
-    protected sameDomainDelayMillis: number;
-    protected domainAccessedTime: Map<string, number>;
-    protected maxRequestsPerCrawl?: number;
-    protected get handledRequestsCount(): number;
-    /** @deprecated Setting `handledRequestsCount` directly is no longer supported. The count is now derived from `this.stats`. */
-    protected set handledRequestsCount(_value: number);
-    protected statusMessageLoggingInterval: number;
-    protected statusMessageCallback?: StatusMessageCallback;
+    protected readonly requestHandler: RequestHandler<ExtendedContext>;
+    protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
+    protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
+    private requestHandlerTimeoutMillis;
+    protected readonly internalTimeoutMillis: number;
+    protected readonly maxRequestRetries: number;
+    protected readonly maxCrawlDepth?: number;
+    private sameDomainDelayMillis;
+    private domainAccessedTime;
+    protected readonly maxRequestsPerCrawl?: number;
+    private get handledRequestsCount();
+    private statusMessageLoggingInterval;
+    private statusMessageCallback?;
     protected blockedStatusCodes: Set<number>;
-    protected additionalHttpErrorStatusCodes: Set<number>;
-    protected ignoreHttpErrorStatusCodes: Set<number>;
-    protected autoscaledPoolOptions: AutoscaledPoolOptions;
-    protected httpClient: BaseHttpClient;
-    protected retryOnBlocked: boolean;
-    protected respectRobotsTxtFile: boolean | {
-        userAgent?: string;
-    };
-    protected onSkippedRequest?: SkippedRequestCallback;
+    protected readonly additionalHttpErrorStatusCodes: Set<number>;
+    private ignoreHttpErrorStatusCodes;
+    private autoscaledPoolOptions;
+    protected readonly httpClient: BaseHttpClient;
+    protected readonly retryOnBlocked: boolean;
+    private respectRobotsTxtFile;
+    protected readonly onSkippedRequest?: SkippedRequestCallback;
     private _closeEvents?;
     private loggedPerRun;
     private readonly robotsTxtFileCache;
@@ -580,9 +576,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
      * This handles base context creation, session resolution, and context helpers.
      */
-    protected buildBasicContextPipeline(): ContextPipeline<{
-        request: Request;
-    }, CrawlingContext>;
+    private buildBasicContextPipeline;
     private checkRobotsTxt;
     /**
      * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
@@ -662,7 +656,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
      * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
      */
-    protected validateRequestUserData(source: Source | string): Promise<void>;
+    private validateRequestUserData;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
     protected getPendingRequestCountApproximation(): Promise<number>;
     protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;
@@ -708,19 +702,19 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     protected _throwOnBlockedRequest(statusCode: number): void;
     private isAllowedBasedOnRobotsTxtFile;
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
-    protected _pauseOnMigration(): Promise<void>;
+    private pauseOnMigration;
     /**
      * Fetches the next request to process from the underlying request provider.
      */
-    protected _fetchNextRequest(): Promise<Request<Dictionary> | null>;
+    private fetchNextRequest;
     /**
      * Delays processing of the request based on the `sameDomainDelaySecs` option,
      * adding it back to the queue after the timeout passes. Returns `true` if the request
      * should be ignored and will be reclaimed to the queue once ready.
      */
-    protected delayRequest(request: Request, source: IRequestManager): boolean;
+    private delayRequest;
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
-    protected handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request): Promise<void>;
+    private handleRequest;
     /**
      * Wrapper around the crawling context's `enqueueLinks` method:
      * - Injects `crawlDepth` to each request being added based on the crawling context request.
@@ -738,15 +732,15 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Run async callback with given timeout and retry. Returns the result of the callback.
      * @ignore
      */
-    protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
+    private timeoutAndRetry;
     /**
      * Returns true if either RequestList or RequestQueue have a request ready for processing.
      */
-    protected _isTaskReadyFunction(): Promise<boolean>;
+    private isTaskReadyFunction;
     /**
      * Returns true if both RequestList and RequestQueue have all requests finished.
      */
-    protected _defaultIsFinishedFunction(): Promise<boolean>;
+    private defaultIsFinishedFunction;
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
      * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
@@ -757,16 +751,15 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      *
      * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
-    protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, request: Request, source: IRequestManager): Promise<void>;
-    protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
-    protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
+    private requestFunctionErrorHandler;
+    private handleFailedRequestHandler;
     /**
      * Resolves the most verbose error message from a thrown error
      * @param error The error received
      * @returns The message to be logged
      */
     protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
-    protected _canRequestBeRetried(request: Request, error: Error): boolean;
+    private canRequestBeRetried;
     /**
      * Stops the crawler immediately.
      *

package/internals/basic-crawler.js

@@ -93,7 +93,7 @@ export class BasicCrawler {
      */
     autoscaledPool;
     /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
+     * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
      */
     proxyConfiguration;
@@ -144,11 +144,6 @@ export class BasicCrawler {
     get handledRequestsCount() {
         return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
     }
-    /** @deprecated Setting `handledRequestsCount` directly is no longer supported. The count is now derived from `this.stats`. */
-    set handledRequestsCount(_value) {
-        throw new Error('Setting `handledRequestsCount` directly is no longer supported. ' +
-            'The count is now derived from `this.stats.state.requestsFinished` and `this.stats.state.requestsFailed`.');
-    }
     statusMessageLoggingInterval;
     statusMessageCallback;
     blockedStatusCodes = new Set();
@@ -357,7 +352,7 @@ export class BasicCrawler {
                         // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
                         if (error instanceof ContextPipelineInterruptedError) {
                             this.stats.discardJob(request.id || request.uniqueKey);
-                            await this._timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
                             return;
                         }
                         // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
@@ -365,8 +360,8 @@ export class BasicCrawler {
                         const isPipelineError = error instanceof ContextPipelineInitializationError || error instanceof SessionError;
                         if (isPipelineError) {
                             const unwrappedError = this.unwrapError(error);
-                            await this._requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
-                            // SessionError already retired the session in `_requestFunctionErrorHandler`;
+                            await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
+                            // SessionError already retired the session in `requestFunctionErrorHandler`;
                             // skip `markBad` to avoid double-counting usage/error score.
                             if (!(unwrappedError instanceof SessionError)) {
                                 crawlingContext.session?.markBad();
@@ -392,7 +387,7 @@ export class BasicCrawler {
                             'Ongoing requests will be allowed to complete.');
                         return false;
                     }
-                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this._isTaskReadyFunction();
+                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this.isTaskReadyFunction();
                 },
                 isFinishedFunction: async () => {
                     if (isMaxPagesExceeded()) {
@@ -407,7 +402,7 @@ export class BasicCrawler {
                     }
                     const isFinished = isFinishedFunction
                         ? await isFinishedFunction()
-                        : await this._defaultIsFinishedFunction();
+                        : await this.defaultIsFinishedFunction();
                     if (isFinished) {
                         const reason = isFinishedFunction
                             ? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
@@ -481,7 +476,7 @@ export class BasicCrawler {
         };
     }
     async resolveRequest() {
-        const request = await this._timeoutAndRetry(this._fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        const request = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
         // Reset loadedUrl so an old one is not carried over to retries.
         if (request) {
             request.loadedUrl = undefined;
@@ -489,7 +484,7 @@ export class BasicCrawler {
         return request;
     }
     async resolveSession({ request }) {
-        const session = await this._timeoutAndRetry(async () => {
+        const session = await this.timeoutAndRetry(async () => {
             const existingSession = await this.sessionPool.getSession(request.sessionId);
             if (!existingSession) {
                 throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
@@ -654,11 +649,11 @@ export class BasicCrawler {
         this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
         const sigintHandler = async () => {
             this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
-            await this._pauseOnMigration();
+            await this.pauseOnMigration();
             await this.autoscaledPool.abort();
         };
         // Attach a listener to handle migration and aborting events gracefully.
-        const boundPauseOnMigration = this._pauseOnMigration.bind(this);
+        const boundPauseOnMigration = this.pauseOnMigration.bind(this);
         process.once('SIGINT', sigintHandler);
         const eventManager = serviceLocator.getEventManager();
         eventManager.on("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
@@ -1021,7 +1016,7 @@ export class BasicCrawler {
             return undefined;
         }
     }
-    async _pauseOnMigration() {
+    async pauseOnMigration() {
         if (this.autoscaledPool) {
             // if run wasn't called, this is going to crash
             await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
@@ -1058,9 +1053,9 @@ export class BasicCrawler {
     /**
      * Fetches the next request to process from the underlying request provider.
      */
-    async _fetchNextRequest() {
+    async fetchNextRequest() {
         if (this.requestManager === undefined) {
-            throw new Error(`_fetchNextRequest called on an uninitialized crawler`);
+            throw new Error(`fetchNextRequest called on an uninitialized crawler`);
         }
         return this.requestManager.fetchNextRequest();
     }
@@ -1095,7 +1090,7 @@ export class BasicCrawler {
         try {
             request.state = RequestState.REQUEST_HANDLER;
             await this.runRequestHandler(crawlingContext);
-            await this._timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
             isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
             this.stats.finishJob(statisticsId, request.retryCount);
             // reclaim session if request finishes successfully
@@ -1106,9 +1101,9 @@ export class BasicCrawler {
             const err = this.unwrapError(rawError);
             try {
                 request.state = RequestState.ERROR_HANDLER;
-                await addTimeoutToPromise(async () => this._requestFunctionErrorHandler(err, crawlingContext, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, crawlingContext, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
                 if (!(err instanceof CriticalError)) {
-                    isRequestLocked = false; // _requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
+                    isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
                 }
                 request.state = RequestState.DONE;
             }
@@ -1211,7 +1206,7 @@ export class BasicCrawler {
      * Run async callback with given timeout and retry. Returns the result of the callback.
      * @ignore
      */
-    async _timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
+    async timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
         try {
             return await addTimeoutToPromise(handler, timeout, error);
         }
@@ -1219,7 +1214,7 @@ export class BasicCrawler {
             if (retried <= maxRetries) {
                 // we retry on any error, not just timeout
                 this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
-                return this._timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
+                return this.timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
             }
             throw e;
         }
@@ -1227,13 +1222,13 @@ export class BasicCrawler {
     /**
      * Returns true if either RequestList or RequestQueue have a request ready for processing.
      */
-    async _isTaskReadyFunction() {
+    async isTaskReadyFunction() {
         return this.requestManager !== undefined && !(await this.requestManager.isEmpty());
     }
     /**
      * Returns true if both RequestList and RequestQueue have all requests finished.
      */
-    async _defaultIsFinishedFunction() {
+    async defaultIsFinishedFunction() {
         return !this.requestManager || (await this.requestManager.isFinished());
     }
     /**
@@ -1253,12 +1248,12 @@ export class BasicCrawler {
      *
      * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
-    async _requestFunctionErrorHandler(error, crawlingContext, request, source) {
+    async requestFunctionErrorHandler(error, crawlingContext, request, source) {
         request.pushErrorMessage(error);
         if (error instanceof CriticalError) {
             throw error;
         }
-        const shouldRetryRequest = this._canRequestBeRetried(request, error);
+        const shouldRetryRequest = this.canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
             await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
             await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
@@ -1299,18 +1294,9 @@ export class BasicCrawler {
         // Mark the request as failed and do not retry.
         await source.markRequestAsHandled(request);
         this.stats.failJob(request.id || request.uniqueKey, request.retryCount);
-        await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
-    }
-    async _tagUserHandlerError(cb) {
-        try {
-            return (await cb());
-        }
-        catch (e) {
-            Object.defineProperty(e, 'triggeredFromUserHandler', { value: true });
-            throw e;
-        }
+        await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
-    async _handleFailedRequestHandler(crawlingContext, error) {
+    async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
         const message = this._getMessageFromError(error, true);
@@ -1339,7 +1325,7 @@ export class BasicCrawler {
             ? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
             : [error.message || error, userLine].join('\n');
     }
-    _canRequestBeRetried(request, error) {
+    canRequestBeRetried(request, error) {
         // Request should never be retried, or the error encountered makes it not able to be retried.
         if (request.noRetry || error instanceof NonRetryableError) {
             return false;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.80",
+    "version": "4.0.0-beta.81",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.80",
-        "@crawlee/http-client": "4.0.0-beta.80",
-        "@crawlee/types": "4.0.0-beta.80",
-        "@crawlee/utils": "4.0.0-beta.80",
+        "@crawlee/core": "4.0.0-beta.81",
+        "@crawlee/http-client": "4.0.0-beta.81",
+        "@crawlee/types": "4.0.0-beta.81",
+        "@crawlee/utils": "4.0.0-beta.81",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "ow": "^2.0.0",
@@ -54,7 +54,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.80"
+        "@crawlee/impit-client": "^4.0.0-beta.81"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "96c57b4a0c999e4b2bd198792490af28db7aa42d"
+    "gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
 }