@crawlee/basic 4.0.0-beta.211 → 4.0.0-beta.212

package/internals/basic-crawler.d.ts

@@ -441,7 +441,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
     protected readonly internalTimeoutMillis: number;
-    private get handledRequestsCount();
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
     protected readonly httpClient: BaseHttpClient;
@@ -514,22 +513,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @returns `true` if the status code is considered an error, `false` otherwise
      */
     protected isErrorStatusCode(status: number): boolean;
-    /**
-     * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
-     * This handles base context creation, session resolution, and context helpers.
-     */
-    private buildBasicContextPipeline;
-    private checkRobotsTxt;
-    private createBaseContext;
-    /**
-     * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
-     * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
-     * `markRequestAsHandled`, so nothing observes the difference.
-     */
-    private resolveRequest;
-    private resolveSession;
-    private createContextHelpers;
-    private buildFinalContextPipeline;
     /**
      * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
      * Used for retrying requests that failed due to proxy errors.
@@ -547,7 +530,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * their status-reporting backend (e.g. the Apify platform).
      */
     setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
-    private getPeriodicLogger;
     /**
      * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
      * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
@@ -596,26 +578,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
      */
     getRequestQueue(): Promise<IRequestManager>;
-    /**
-     * Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
-     */
-    private openOwnedRequestQueue;
-    /**
-     * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
-     * locking storage backend keeps it reserved for slightly longer than the request handler timeout
-     * (with some padding for overhead), but never for less than a minute. This prevents a long-running
-     * request from being handed out a second time while it is still being processed — and it works
-     * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
-     */
-    private applyRequestManagerTimeouts;
-    /**
-     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
-     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
-     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
-     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
-     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
-     */
-    private validateRequestUserData;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
@@ -661,26 +623,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * browser crawlers override it with their `navigationTimeoutSecs`.
      */
     protected getNavigationTimeoutMillis(): number;
-    /**
-     * Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
-     * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
-     * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
-     * whichever is larger: the configured internal timeout, or this request's combined phase budget.
-     */
-    private withRequestTimeout;
-    /**
-     * The request handler timeout for a request with the given route label. A router route may override the
-     * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
-     *
-     * @param label The request's route label, or `undefined` for the default route / no specific request.
-     * @param fallbackMillis Timeout to use when no route overrides it.
-     */
-    private resolveRequestHandlerTimeoutMillis;
-    /**
-     * The timeout the router route with the given label asked for, or `undefined` when it did not override one
-     * (or the request handler is not a router at all).
-     */
-    private getRouteTimeoutMillis;
     protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     /**
      * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
@@ -692,7 +634,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Handles blocked request
      */
     protected throwOnBlockedRequest(statusCode: number): void;
-    private isAllowedBasedOnRobotsTxtFile;
     /**
      * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
      *
@@ -701,15 +642,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      *  {@link RequestThrottledError} rather than treating the response as a blocked session.
      */
     protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
-    /** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
-    private applyCrawlDelay;
     /** @internal */
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
-    private pauseOnMigration;
-    /**
-     * Fetches the next request to process from the underlying request provider.
-     */
-    private fetchNextRequest;
     /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
     private handleRequest;
     /**
@@ -717,21 +651,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @internal
      */
     protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
-    /**
-     * Run async callback with given timeout and retry. Returns the result of the callback.
-     * @ignore
-     */
-    private timeoutAndRetry;
-    /**
-     * Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
-     * wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
-     */
-    private isTaskReadyFunction;
-    /**
-     * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
-     */
-    private unwrapError;
     /**
      * Handles errors thrown by user provided requestHandler()
      *
@@ -745,12 +664,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @returns The message to be logged
      */
     protected getMessageFromError(error: Error, forceStack?: boolean): string;
-    /**
-     * Whether the session should be spared for this error - either because it was already retired, or because the
-     * failure says nothing about the session (a rate limit is a property of the domain).
-     */
-    private errorAbsolvesSession;
-    private canRequestBeRetried;
     /**
      * Ends the run in progress immediately, without waiting for the requests already in flight.
      *
@@ -769,7 +682,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     destroy(): Promise<void>;
     [Symbol.asyncDispose](): Promise<void>;
     protected getCookieHeaderFromRequest(request: Request): string;
-    private requestMatchesEnqueueStrategy;
 }
 export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
 }

package/internals/basic-crawler.js

@@ -184,14 +184,14 @@ export class BasicCrawler {
      * Context built with this pipeline can be passed into multiple crawler pipelines at once.
      */
     get #basicPipeline() {
-        this.#basicContextPipeline ??= this.buildBasicContextPipeline();
+        this.#basicContextPipeline ??= this.#buildBasicContextPipeline();
         return this.#basicContextPipeline;
     }
     #contextPipeline;
     /** @internal */
     get contextPipeline() {
         if (this.#contextPipeline === undefined) {
-            this.#contextPipeline = this.buildFinalContextPipeline();
+            this.#contextPipeline = this.#buildFinalContextPipeline();
         }
         return this.#contextPipeline;
     }
@@ -212,7 +212,7 @@ export class BasicCrawler {
     #maxRequestRetries;
     #maxCrawlDepth;
     #maxRequestsPerCrawl;
-    get handledRequestsCount() {
+    get #handledRequestsCount() {
         return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;
     }
     #statusMessageLoggingInterval;
@@ -370,12 +370,12 @@ export class BasicCrawler {
                     throttleBy: 'registrableDomain',
                     persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,
                     // A factory, because the default queue is only opened on first use.
-                    inner: suppliedManager ?? (() => this.openOwnedRequestQueue()),
+                    inner: suppliedManager ?? (() => this.#openOwnedRequestQueue()),
                 })
                 : suppliedManager;
             if (requestList !== undefined) {
                 // The list is read first, while new requests still have somewhere writable to go.
-                this.#requestManager = new RequestManagerTandem(requestList, writableManager ?? (() => this.openOwnedRequestQueue()));
+                this.#requestManager = new RequestManagerTandem(requestList, writableManager ?? (() => this.#openOwnedRequestQueue()));
             }
             else if (writableManager !== undefined) {
                 // A RequestQueue is itself a request manager.
@@ -440,7 +440,7 @@ export class BasicCrawler {
             }
             this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
             this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
-            const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.handledRequestsCount;
+            const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.#handledRequestsCount;
             // eslint-disable-next-line prefer-const
             let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
             // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
@@ -452,7 +452,7 @@ export class BasicCrawler {
                     const source = this.requestManager;
                     if (!source)
                         throw new Error('Request provider is not initialized!');
-                    const request = await this.resolveRequest();
+                    const request = await this.#resolveRequest();
                     if (!request) {
                         return;
                     }
@@ -469,20 +469,20 @@ export class BasicCrawler {
                         // Navigation, the navigation hooks and the request handler are timed individually, but the
                         // phases between them are not, so a request could still get stuck indefinitely. This is the
                         // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
-                        await this.withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), (error) => this.#handleRequestFailure(error, crawlingContext, source, request))));
+                        await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), (error) => this.#handleRequestFailure(error, crawlingContext, source, request))));
                     }
                     catch (error) {
                         // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
                         // take the normal error flow. Anything else is fatal.
                         if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
-                            const unwrappedError = this.unwrapError(error);
+                            const unwrappedError = this.#unwrapError(error);
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
-                            if (!this.errorAbsolvesSession(unwrappedError)) {
+                            if (!this.#errorAbsolvesSession(unwrappedError)) {
                                 crawlingContext.session?.markBad();
                             }
                             return;
                         }
-                        throw this.unwrapError(error);
+                        throw this.#unwrapError(error);
                     }
                 },
                 isTaskReadyFunction: async () => {
@@ -496,13 +496,13 @@ export class BasicCrawler {
                             'Ongoing requests will be allowed to complete.');
                         return false;
                     }
-                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this.isTaskReadyFunction();
+                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this.#isTaskReadyFunction();
                 },
                 isFinishedFunction: async () => {
                     if (isMaxPagesExceeded()) {
                         this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +
                             'and all requests that were in progress at that time have now finished. ' +
-                            `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
+                            `In total, the crawler processed ${this.#handledRequestsCount} requests and will shut down.`);
                         return true;
                     }
                     if (this.#run?.stopRequested) {
@@ -571,15 +571,15 @@ export class BasicCrawler {
      * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
      * This handles base context creation, session resolution, and context helpers.
      */
-    buildBasicContextPipeline() {
+    #buildBasicContextPipeline() {
         return ContextPipeline.create()
-            .compose(this.checkRobotsTxt.bind(this))
-            .compose(this.createBaseContext.bind(this))
-            .compose(this.resolveSession.bind(this))
-            .compose(this.createContextHelpers.bind(this));
+            .compose(this.#checkRobotsTxt.bind(this))
+            .compose(this.#createBaseContext.bind(this))
+            .compose(this.#resolveSession.bind(this))
+            .compose(this.#createContextHelpers.bind(this));
     }
-    async checkRobotsTxt({ request }) {
-        if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
+    async #checkRobotsTxt({ request }) {
+        if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {
             this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
             request.state = RequestState.SKIPPED;
             request.noRetry = true;
@@ -591,7 +591,7 @@ export class BasicCrawler {
         }
         return {};
     }
-    createBaseContext(context, onCleanup) {
+    #createBaseContext(context, onCleanup) {
         return {
             id: cryptoRandomObjectId(10),
             log: this.log,
@@ -640,8 +640,8 @@ export class BasicCrawler {
      * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
      * `markRequestAsHandled`, so nothing observes the difference.
      */
-    async resolveRequest() {
-        const fetched = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+    async #resolveRequest() {
+        const fetched = await this.#timeoutAndRetry(this.#fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
         if (!fetched) {
             return null;
         }
@@ -650,8 +650,8 @@ export class BasicCrawler {
         request.loadedUrl = undefined;
         return request;
     }
-    async resolveSession({ request }) {
-        const session = await this.timeoutAndRetry(async () => {
+    async #resolveSession({ request }) {
+        const session = await this.#timeoutAndRetry(async () => {
             const existingSession = await this.sessionPool.getSession(request.sessionId);
             if (!existingSession) {
                 throw new MissingSessionError(request.sessionId);
@@ -660,7 +660,7 @@ export class BasicCrawler {
         }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
         return { session, proxyInfo: session?.proxyInfo };
     }
-    async createContextHelpers({ request, session }) {
+    async #createContextHelpers({ request, session }) {
         const addRequests = async (requests, options = {}) => {
             const newCrawlDepth = request.crawlDepth + 1;
             const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
@@ -669,7 +669,7 @@ export class BasicCrawler {
         const sendRequest = createSendRequest(this.httpClient, request, session);
         return { addRequests, sendRequest };
     }
-    buildFinalContextPipeline() {
+    #buildFinalContextPipeline() {
         const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
             ContextPipeline.create());
         // `extendContext` runs *before* the subclass navigation pipeline (which includes the
@@ -694,7 +694,7 @@ export class BasicCrawler {
         }
         contextPipeline = contextPipeline.compose(async (context) => {
             const { request } = context;
-            if (request && !this.requestMatchesEnqueueStrategy(request)) {
+            if (request && !this.#requestMatchesEnqueueStrategy(request)) {
                 // eslint-disable-next-line dot-notation
                 const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
                 this.log.debug(message);
@@ -741,7 +741,7 @@ export class BasicCrawler {
             level: options.level,
         });
     }
-    getPeriodicLogger() {
+    #getPeriodicLogger() {
         let previousState = { ...this.statistics.state };
         const getOperationMode = () => {
             const { requestsFailed } = this.statistics.state;
@@ -846,15 +846,15 @@ export class BasicCrawler {
                 await this.#concurrencySystem.ifOwned((system) => system.stop());
                 throw error;
             }
-            const periodicLogger = this.getPeriodicLogger();
+            const periodicLogger = this.#getPeriodicLogger();
             this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
             const sigintHandler = async () => {
                 this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
-                await this.pauseOnMigration();
+                await this.#pauseOnMigration();
                 await run.abort();
             };
             // Attach a listener to handle migration and aborting events gracefully.
-            const boundPauseOnMigration = this.pauseOnMigration.bind(this);
+            const boundPauseOnMigration = this.#pauseOnMigration.bind(this);
             process.once('SIGINT', sigintHandler);
             const eventManager = serviceLocator.getEventManager();
             eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
@@ -970,14 +970,14 @@ export class BasicCrawler {
      */
     async getRequestManager() {
         if (!this.#requestManager) {
-            this.#requestManager = await this.openOwnedRequestQueue();
+            this.#requestManager = await this.#openOwnedRequestQueue();
         }
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
         // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
         // but guard so we do not re-issue it on every call.
         if (!this.#requestManagerTimeoutsApplied) {
             this.#requestManagerTimeoutsApplied = true;
-            await this.applyRequestManagerTimeouts(this.#requestManager);
+            await this.#applyRequestManagerTimeouts(this.#requestManager);
         }
         return this.#requestManager;
     }
@@ -991,7 +991,7 @@ export class BasicCrawler {
     /**
      * Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
      */
-    async openOwnedRequestQueue() {
+    async #openOwnedRequestQueue() {
         // The first crawler instance uses the default queue (null identifier);
         // subsequent instances get their own queue via a unique alias so they don't collide.
         const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };
@@ -1004,7 +1004,7 @@ export class BasicCrawler {
      * request from being handed out a second time while it is still being processed — and it works
      * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
      */
-    async applyRequestManagerTimeouts(requestManager) {
+    async #applyRequestManagerTimeouts(requestManager) {
         // A router route may hold a request for longer than the crawler's own timeout, and we cannot know
         // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
         // raise-only, so erring high here is safe.
@@ -1019,7 +1019,7 @@ export class BasicCrawler {
      * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
      * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
      */
-    async validateRequestUserData(source) {
+    async #validateRequestUserData(source) {
         if (typeof source === 'string') {
             return;
         }
@@ -1062,7 +1062,9 @@ export class BasicCrawler {
         if (this.#maxRequestsPerCrawl === undefined) {
             return explicitLimit;
         }
-        const limit = Math.max(0, this.#maxRequestsPerCrawl - this.handledRequestsCount - (await this.#getPendingRequestCountApproximation()));
+        const limit = Math.max(0, this.#maxRequestsPerCrawl -
+            this.#handledRequestsCount -
+            (await this.#getPendingRequestCountApproximation()));
         return Math.min(limit, explicitLimit ?? Infinity);
     }
     async #handleSkippedRequest(options) {
@@ -1109,7 +1111,7 @@ export class BasicCrawler {
         // its own per-item validation below, and validating an absent label/userData here would spuriously
         // check them against a registered default-route schema.
         if (options.label !== undefined || options.userData !== undefined) {
-            await this.validateRequestUserData({ label: options.label, userData: options.userData });
+            await this.#validateRequestUserData({ label: options.label, userData: options.userData });
         }
         const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);
         const strategy = options.strategy ?? EnqueueStrategy.All;
@@ -1118,9 +1120,9 @@ export class BasicCrawler {
         // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
         // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
         const enqueueStrategyPatterns = options.baseUrl ? buildEnqueueStrategyPatterns(options.baseUrl, strategy) : [];
-        const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
+        const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
         const maxCrawlDepth = this.#maxCrawlDepth;
-        const validateRequestUserData = this.validateRequestUserData.bind(this);
+        const validateRequestUserData = this.#validateRequestUserData.bind(this);
         const allSkipped = [];
         // A skipped source (which can carry arbitrary userData) is only retained if something reads it -
         // otherwise the URL alone is enough to build the callback argument and to log with.
@@ -1292,7 +1294,7 @@ export class BasicCrawler {
         // floored per request so it will not actually cut them short, but the configured value is then effectively
         // ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
         // navigation timeout only after `super()`.
-        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(undefined);
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(undefined);
         if (this.internalTimeoutMillis < phasesMillis) {
             this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
                 `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
@@ -1314,9 +1316,9 @@ export class BasicCrawler {
      * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
      * whichever is larger: the configured internal timeout, or this request's combined phase budget.
      */
-    async withRequestTimeout(crawlingContext, work) {
+    async #withRequestTimeout(crawlingContext, work) {
         const { request } = crawlingContext;
-        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(request.label);
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(request.label);
         const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
         await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
     }
@@ -1327,14 +1329,14 @@ export class BasicCrawler {
      * @param label The request's route label, or `undefined` for the default route / no specific request.
      * @param fallbackMillis Timeout to use when no route overrides it.
      */
-    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
-        return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
+    #resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
+        return this.#getRouteTimeoutMillis(label) ?? fallbackMillis;
     }
     /**
      * The timeout the router route with the given label asked for, or `undefined` when it did not override one
      * (or the request handler is not a router at all).
      */
-    getRouteTimeoutMillis(label) {
+    #getRouteTimeoutMillis(label) {
         const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
         if (typeof getTimeoutSecs !== 'function') {
             return undefined;
@@ -1343,7 +1345,7 @@ export class BasicCrawler {
         return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
     }
     async runRequestHandler(crawlingContext) {
-        const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
+        const timeoutMillis = this.#resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
         await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
     }
     /**
@@ -1351,6 +1353,7 @@ export class BasicCrawler {
      * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
      * normal return says nothing about success. `handleRequest` commits on success.
      */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
     async runInStorageTransaction(callback) {
         if (!this.#transactionalStorageEnabled) {
             return callback();
@@ -1390,7 +1393,7 @@ export class BasicCrawler {
             throw new SessionError(`Request blocked - received ${statusCode} status code.`);
         }
     }
-    async isAllowedBasedOnRobotsTxtFile(url) {
+    async #isAllowedBasedOnRobotsTxtFile(url) {
         if (!this.#respectRobotsTxtFile) {
             return true;
         }
@@ -1399,7 +1402,7 @@ export class BasicCrawler {
         if (robotsTxtFile) {
             const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
             if (crawlDelay !== undefined) {
-                this.applyCrawlDelay(url, crawlDelay);
+                this.#applyCrawlDelay(url, crawlDelay);
             }
         }
         return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
@@ -1427,7 +1430,7 @@ export class BasicCrawler {
         return false;
     }
     /** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
-    applyCrawlDelay(url, delaySeconds) {
+    #applyCrawlDelay(url, delaySeconds) {
         // robots.txt is per-origin; `hostname` is the closest pacing scope and errs wide (http and https to one
         // host share a clock).
         if (this.requestManager?.recordPacingSignal({
@@ -1464,7 +1467,7 @@ export class BasicCrawler {
             return undefined;
         }
     }
-    async pauseOnMigration() {
+    async #pauseOnMigration() {
         const run = this.#dispatchingRun;
         if (run) {
             await run.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
@@ -1484,7 +1487,7 @@ export class BasicCrawler {
     /**
      * Fetches the next request to process from the underlying request provider.
      */
-    async fetchNextRequest() {
+    async #fetchNextRequest() {
         if (this.requestManager === undefined) {
             throw new Error(`fetchNextRequest called on an uninitialized crawler`);
         }
@@ -1505,7 +1508,7 @@ export class BasicCrawler {
         // it is retried. This also closes the transaction, so everything below passes through.
         // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
         await currentStorageTransaction()?.commit();
-        await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
         this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
         // reclaim session if request finishes successfully
         request.state = RequestState.DONE;
@@ -1522,7 +1525,7 @@ export class BasicCrawler {
         currentStorageTransaction()?.rollback();
         const skipped = rawError instanceof ContextPipelineInitializationError &&
             rawError.cause instanceof ContextPipelineInterruptedError;
-        const err = this.unwrapError(rawError);
+        const err = this.#unwrapError(rawError);
         const context = crawlingContext;
         const retryCountBefore = request.retryCount;
         const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
@@ -1531,7 +1534,7 @@ export class BasicCrawler {
             // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
             if (skipped) {
                 this.statistics.discardRequestRecord(request.id || request.uniqueKey);
-                await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
                 return;
             }
             request.state = RequestState.ERROR_HANDLER;
@@ -1539,7 +1542,7 @@ export class BasicCrawler {
             request.state = RequestState.DONE;
         }
         catch (secondaryError) {
-            const unwrappedSecondaryError = this.unwrapError(secondaryError);
+            const unwrappedSecondaryError = this.#unwrapError(secondaryError);
             const timedOut = unwrappedSecondaryError instanceof TimeoutError;
             request.state = RequestState.ERROR;
             if (timedOut) {
@@ -1575,7 +1578,7 @@ export class BasicCrawler {
         }
         // decrease the session score if the request fails (but the error handler did not throw);
         // skip when the error is a SessionError, which already retired the session
-        if (!skipped && !this.errorAbsolvesSession(err)) {
+        if (!skipped && !this.#errorAbsolvesSession(err)) {
             context.session?.markBad();
         }
     }
@@ -1603,7 +1606,7 @@ export class BasicCrawler {
      * Run async callback with given timeout and retry. Returns the result of the callback.
      * @ignore
      */
-    async timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
+    async #timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
         try {
             return await addTimeoutToPromise(handler, timeout, error);
         }
@@ -1611,7 +1614,7 @@ export class BasicCrawler {
             if (retried <= maxRetries) {
                 // we retry on any error, not just timeout
                 this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
-                return this.timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
+                return this.#timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
             }
             throw e;
         }
@@ -1620,7 +1623,7 @@ export class BasicCrawler {
      * Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
      * wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
      */
-    async isTaskReadyFunction() {
+    async #isTaskReadyFunction() {
         if (this.requestManager === undefined) {
             return false;
         }
@@ -1634,11 +1637,11 @@ export class BasicCrawler {
      * Unwraps errors thrown by the context pipeline to get the actual user error.
      * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
      */
-    unwrapError(error) {
+    #unwrapError(error) {
         if (error instanceof RequestHandlerError ||
             error instanceof ContextPipelineInitializationError ||
             error instanceof ContextPipelineCleanupError) {
-            return this.unwrapError(error.cause);
+            return this.#unwrapError(error.cause);
         }
         return error;
     }
@@ -1663,7 +1666,7 @@ export class BasicCrawler {
         if (error instanceof CriticalError) {
             throw error;
         }
-        const shouldRetryRequest = this.canRequestBeRetried(request, error);
+        const shouldRetryRequest = this.#canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
             await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
             await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
@@ -1741,10 +1744,10 @@ export class BasicCrawler {
      * Whether the session should be spared for this error - either because it was already retired, or because the
      * failure says nothing about the session (a rate limit is a property of the domain).
      */
-    errorAbsolvesSession(error) {
+    #errorAbsolvesSession(error) {
         return error instanceof SessionError || error instanceof RequestThrottledError;
     }
-    canRequestBeRetried(request, error) {
+    #canRequestBeRetried(request, error) {
         // Request should never be retried, or the error encountered makes it not able to be retried.
         if (request.noRetry || error instanceof NonRetryableError) {
             return false;
@@ -1802,7 +1805,7 @@ export class BasicCrawler {
         }
         return request.headers?.Cookie || request.headers?.cookie || '';
     }
-    requestMatchesEnqueueStrategy(request) {
+    #requestMatchesEnqueueStrategy(request) {
         // If `skipNavigation` was used, just return `true`
         try {
             // eslint-disable-next-line @typescript-eslint/no-unused-expressions