@crawlee/basic 4.0.0-beta.105 → 4.0.0-beta.107

package/internals/basic-crawler.d.ts

@@ -433,13 +433,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * collide.
      */
     private static instanceCount;
-    /**
-     * Tracks crawler instances that accessed shared state without having an explicit id.
-     * Used to detect and warn about multiple crawlers sharing the same state.
-     */
-    private static useStateAnonymousIndices;
-    /** Backs the {@link BasicCrawler.stats|`stats`} getter. */
-    private statsDep;
     /**
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
@@ -451,33 +444,11 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
     protected requestManager?: IRequestManager;
-    /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
-    private sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool(): ISessionPool;
-    /**
-     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
-     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
-     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
-     */
-    private ownedRequestQueue;
-    /**
-     * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
-     * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
-     * so it only needs to be applied once, at the first async access of the manager.
-     */
-    private requestManagerTimeoutsApplied;
-    /**
-     * Resolves the governor for one run: either the injected
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
-     * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
-     */
-    private readonly resolveConcurrencySystem;
-    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    private concurrencySystemDep?;
     /**
      * The concurrency governor this run is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
@@ -491,14 +462,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem(): IConcurrencySystem | undefined;
-    /**
-     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
-     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
-     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
-     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
-     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
-     */
-    private autoscaledPool?;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -509,7 +472,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     readonly router: RouterHandler<Context, Routes>;
-    private _basicContextPipeline?;
     /**
      * The basic part of the context pipeline. Unlike the subclass pipeline, this
      * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
@@ -521,11 +483,9 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     get basicContextPipeline(): ContextPipeline<{
         request: Request;
     }, CrawlingContext>;
-    private _contextPipeline?;
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
     running: boolean;
     hasFinishedBefore: boolean;
-    private unexpectedStop;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
     protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
@@ -534,15 +494,10 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     protected readonly internalTimeoutMillis: number;
     protected readonly maxRequestRetries: number;
     protected readonly maxCrawlDepth?: number;
-    private sameDomainDelayMillis;
-    private domainAccessedTime;
     protected readonly maxRequestsPerCrawl?: number;
     private get handledRequestsCount();
-    private statusMessageLoggingInterval;
-    private statusMessageCallback?;
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
-    private ignoreHttpErrorStatusCodes;
     /**
      * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
      * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
@@ -552,17 +507,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
-    private respectRobotsTxtFile;
-    /** Whether `runInStorageTransaction()` opens a transaction at all. */
-    private readonly transactionalStorageEnabled;
-    /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
-    private readonly storageWritePolicy;
     protected readonly onSkippedRequest?: SkippedRequestCallback;
-    private _closeEvents?;
-    private loggedPerRun;
-    private readonly robotsTxtFileCache;
     protected readonly identity: CrawlerIdentity;
-    private readonly contextPipelineOptions;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         contextPipelineBuilder: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
@@ -793,7 +739,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * Initializes the crawler.
      */
-    protected _init(): Promise<void>;
+    protected init(): Promise<void>;
     /**
      * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
      * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
@@ -830,7 +776,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * Handles blocked request
      */
-    protected _throwOnBlockedRequest(statusCode: number): void;
+    protected throwOnBlockedRequest(statusCode: number): void;
     private isAllowedBasedOnRobotsTxtFile;
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
     private pauseOnMigration;
@@ -889,7 +835,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @param error The error received
      * @returns The message to be logged
      */
-    protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
+    protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
     private canRequestBeRetried;
     /**
      * Stops the crawler immediately.
@@ -899,7 +845,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     teardown(): Promise<void>;
-    protected _getCookieHeaderFromRequest(request: Request): string;
+    protected getCookieHeaderFromRequest(request: Request): string;
     private requestMatchesEnqueueStrategy;
 }
 export interface CreateContextOptions {

package/internals/basic-crawler.js

@@ -12,9 +12,9 @@ import { cryptoRandomObjectId } from '@apify/utilities';
 import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
 class LazyDefaultHttpClient {
-    _delegatePromise;
+    #delegatePromise;
     constructor(options) {
-        this._delegatePromise = import('@crawlee/impit-client')
+        this.#delegatePromise = import('@crawlee/impit-client')
             .then(({ ImpitHttpClient }) => new ImpitHttpClient(options))
             .catch(() => {
             (options?.logger ?? log).warning('Optional dependency @crawlee/impit-client is not installed. ' +
@@ -23,7 +23,7 @@ class LazyDefaultHttpClient {
         });
     }
     async sendRequest(...args) {
-        return (await this._delegatePromise).sendRequest(...args);
+        return (await this.#delegatePromise).sendRequest(...args);
     }
 }
 /**
@@ -46,20 +46,21 @@ export class BasicCrawler {
      * request queue; subsequent ones get their own queue via a unique alias so they don't
      * collide.
      */
+    // kept as TS-private: tests reset the counter at runtime
     static instanceCount = 0;
     /**
      * Tracks crawler instances that accessed shared state without having an explicit id.
      * Used to detect and warn about multiple crawlers sharing the same state.
      */
-    static useStateAnonymousIndices = new Set();
+    static #useStateAnonymousIndices = new Set();
     /** Backs the {@link BasicCrawler.stats|`stats`} getter. */
-    statsDep;
+    #statsDep;
     /**
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
     get stats() {
-        return this.statsDep.value;
+        return this.#statsDep.value;
     }
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
@@ -68,34 +69,34 @@ export class BasicCrawler {
      */
     requestManager;
     /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
-    sessionPoolDep;
+    #sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool() {
-        return this.sessionPoolDep.value;
+        return this.#sessionPoolDep.value;
     }
     /**
      * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
      * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
      * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
      */
-    ownedRequestQueue = OwnedOrInjected.resolve();
+    #ownedRequestQueue = OwnedOrInjected.resolve();
     /**
      * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
      * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
      * so it only needs to be applied once, at the first async access of the manager.
      */
-    requestManagerTimeoutsApplied = false;
+    #requestManagerTimeoutsApplied = false;
     /**
      * Resolves the governor for one run: either the injected
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
      * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
      */
-    resolveConcurrencySystem;
-    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    concurrencySystemDep;
+    #resolveConcurrencySystem;
+    /** As resolved by `init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
+    #concurrencySystemDep;
     /**
      * The concurrency governor this run is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
@@ -109,7 +110,7 @@ export class BasicCrawler {
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem() {
-        return this.concurrencySystemDep?.maybeValue;
+        return this.#concurrencySystemDep?.maybeValue;
     }
     /**
      * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
@@ -118,7 +119,7 @@ export class BasicCrawler {
      * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
      * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
      */
-    autoscaledPool;
+    #autoscaledPool;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -129,7 +130,7 @@ export class BasicCrawler {
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     router = Router.create();
-    _basicContextPipeline;
+    #basicContextPipeline;
     /**
      * The basic part of the context pipeline. Unlike the subclass pipeline, this
      * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
@@ -139,21 +140,21 @@ export class BasicCrawler {
      * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
      */
     get basicContextPipeline() {
-        if (this._basicContextPipeline === undefined) {
-            this._basicContextPipeline = this.buildBasicContextPipeline();
+        if (this.#basicContextPipeline === undefined) {
+            this.#basicContextPipeline = this.buildBasicContextPipeline();
         }
-        return this._basicContextPipeline;
+        return this.#basicContextPipeline;
     }
-    _contextPipeline;
+    #contextPipeline;
     get contextPipeline() {
-        if (this._contextPipeline === undefined) {
-            this._contextPipeline = this.buildFinalContextPipeline();
+        if (this.#contextPipeline === undefined) {
+            this.#contextPipeline = this.buildFinalContextPipeline();
         }
-        return this._contextPipeline;
+        return this.#contextPipeline;
     }
     running = false;
     hasFinishedBefore = false;
-    unexpectedStop = false;
+    #unexpectedStop = false;
     #log;
     get log() {
         return this.#log;
@@ -161,41 +162,43 @@ export class BasicCrawler {
     requestHandler;
     errorHandler;
     failedRequestHandler;
+    // kept as TS-private: tests read it at runtime
     requestHandlerTimeoutMillis;
     internalTimeoutMillis;
     maxRequestRetries;
     maxCrawlDepth;
-    sameDomainDelayMillis;
-    domainAccessedTime;
+    #sameDomainDelayMillis;
+    #domainAccessedTime;
     maxRequestsPerCrawl;
     get handledRequestsCount() {
         return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
     }
-    statusMessageLoggingInterval;
-    statusMessageCallback;
+    #statusMessageLoggingInterval;
+    #statusMessageCallback;
     blockedStatusCodes = new Set();
     additionalHttpErrorStatusCodes;
-    ignoreHttpErrorStatusCodes;
+    #ignoreHttpErrorStatusCodes;
     /**
      * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
      * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
      * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
      * settable here.
      */
+    // kept as TS-private: tests mutate it at runtime
     taskLoopOptions;
     httpClient;
     retryOnBlocked;
-    respectRobotsTxtFile;
+    #respectRobotsTxtFile;
     /** Whether `runInStorageTransaction()` opens a transaction at all. */
-    transactionalStorageEnabled;
+    #transactionalStorageEnabled;
     /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
-    storageWritePolicy;
+    #storageWritePolicy;
     onSkippedRequest;
-    _closeEvents;
-    loggedPerRun = new Set();
-    robotsTxtFileCache;
+    #closeEvents;
+    #loggedPerRun = new Set();
+    #robotsTxtFileCache;
     identity;
-    contextPipelineOptions;
+    #contextPipelineOptions;
     static optionsShape = {
         contextPipelineBuilder: ow.optional.object,
         extendContext: ow.optional.function,
@@ -278,7 +281,7 @@ export class BasicCrawler {
         }
         try {
             serviceLocatorScope.enterScope();
-            this.contextPipelineOptions = {
+            this.#contextPipelineOptions = {
                 contextPipelineBuilder: options.contextPipelineBuilder,
                 extendContext: options.extendContext,
             };
@@ -309,13 +312,13 @@ export class BasicCrawler {
             }
             this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
             this.proxyConfiguration = proxyConfiguration;
-            this.statusMessageLoggingInterval = statusMessageLoggingInterval;
-            this.statusMessageCallback = statusMessageCallback;
-            this.domainAccessedTime = new Map();
-            this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
+            this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
+            this.#statusMessageCallback = statusMessageCallback;
+            this.#domainAccessedTime = new Map();
+            this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
             this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
             this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
-            this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
+            this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
             this.requestHandler = requestHandler ?? this.router;
             this.failedRequestHandler = failedRequestHandler;
             this.errorHandler = errorHandler;
@@ -326,11 +329,11 @@ export class BasicCrawler {
                 this.requestHandlerTimeoutMillis = 60_000;
             }
             this.retryOnBlocked = retryOnBlocked;
-            this.respectRobotsTxtFile = respectRobotsTxtFile;
+            this.#respectRobotsTxtFile = respectRobotsTxtFile;
             // The cast undoes ow's assertion signature, which mangles `boolean | object` unions.
             const transactionalStorageOption = transactionalStorage;
-            this.transactionalStorageEnabled = transactionalStorageOption !== false;
-            this.storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
+            this.#transactionalStorageEnabled = transactionalStorageOption !== false;
+            this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
             this.onSkippedRequest = onSkippedRequest;
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
@@ -338,8 +341,8 @@ export class BasicCrawler {
                     Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
-            this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
-            this.statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
+            this.#sameDomainDelayMillis = sameDomainDelaySecs * 1000;
+            this.#statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
                 id: this.identity.id,
@@ -350,7 +353,7 @@ export class BasicCrawler {
                     '`proxyInfo` they were created with. Configure proxies on the pool instead, ' +
                     'e.g. via `addSession({ proxyInfo })` or a custom `createSessionFunction`.');
             }
-            this.sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
+            this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
                 createSessionFunction: async (opts) => new Session({
                     ...opts?.sessionOptions,
                     proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
@@ -435,7 +438,7 @@ export class BasicCrawler {
                             `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
                         return false;
                     }
-                    if (this.unexpectedStop) {
+                    if (this.#unexpectedStop) {
                         this.logOncePerRun('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
                             'Ongoing requests will be allowed to complete.');
                         return false;
@@ -449,7 +452,7 @@ export class BasicCrawler {
                             `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
                         return true;
                     }
-                    if (this.unexpectedStop) {
+                    if (this.#unexpectedStop) {
                         this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
                         return true;
                     }
@@ -467,7 +470,7 @@ export class BasicCrawler {
                 log: this.log,
             };
             this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
-            this.resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
+            this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
                 minConcurrency,
                 maxConcurrency,
                 maxTasksPerMinute: maxRequestsPerMinute,
@@ -496,7 +499,7 @@ export class BasicCrawler {
      * @returns `true` if the status code is considered an error, `false` otherwise
      */
     isErrorStatusCode(status) {
-        const excludeError = this.ignoreHttpErrorStatusCodes.has(status);
+        const excludeError = this.#ignoreHttpErrorStatusCodes.has(status);
         const includeError = this.additionalHttpErrorStatusCodes.has(status);
         return (status >= 500 && !excludeError) || includeError;
     }
@@ -589,7 +592,7 @@ export class BasicCrawler {
         return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };
     }
     buildFinalContextPipeline() {
-        const subclassPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??
+        const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
             this.buildContextPipeline());
         // `extendContext` runs *before* the subclass navigation pipeline (which includes the
         // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
@@ -601,7 +604,7 @@ export class BasicCrawler {
         // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
         // casts below are sound because `buildFinalContextPipeline` is declared to return the fully
         // resolved `ExtendedContext` (= `Context & ContextExtension`).
-        const { extendContext } = this.contextPipelineOptions;
+        const { extendContext } = this.#contextPipelineOptions;
         let contextPipeline;
         if (extendContext !== undefined) {
             contextPipeline = ContextPipeline.create()
@@ -635,7 +638,7 @@ export class BasicCrawler {
      * @param error The error to check.
      */
     isProxyError(error) {
-        return ROTATE_PROXY_ERRORS.some((x) => this._getMessageFromError(error)?.includes(x));
+        return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error)?.includes(x));
     }
     /**
      * Sets the status message for the current crawler run.
@@ -676,14 +679,14 @@ export class BasicCrawler {
             const { mode: operationMode, failedDelta } = getOperationMode();
             let message;
             if (operationMode === 'ERROR') {
-                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
+                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.#statusMessageLoggingInterval} seconds.`;
             }
             else {
                 const total = await this.requestManager?.getTotalCount();
                 message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
-            if (this.statusMessageCallback) {
-                await this.statusMessageCallback({
+            if (this.#statusMessageCallback) {
+                await this.#statusMessageCallback({
                     crawler: this,
                     state: this.stats.state,
                     previousState,
@@ -693,7 +696,7 @@ export class BasicCrawler {
             }
             this.setStatusMessage(message);
         };
-        const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);
+        const interval = setInterval(log, this.#statusMessageLoggingInterval * 1e3);
         return { log, stop: () => clearInterval(interval) };
     }
     /**
@@ -721,20 +724,20 @@ export class BasicCrawler {
             // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
             // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
             const shouldPurge = purgeRequestQueue !== false;
-            const managerToPurge = this.ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
+            const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
             if (managerToPurge?.purge && shouldPurge) {
                 await managerToPurge.purge();
             }
             // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
-            await this.statsDep.ifOwned(async (stats) => {
+            await this.#statsDep.ifOwned(async (stats) => {
                 stats.reset();
                 await stats.resetStore();
             });
-            await this.sessionPoolDep.ifOwned((pool) => pool.resetStore());
+            await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());
         }
-        this.unexpectedStop = false;
+        this.#unexpectedStop = false;
         this.running = true;
-        this.loggedPerRun.clear();
+        this.#loggedPerRun.clear();
         await purgeDefaultStorages({
             onlyPurgeOnce: true,
             storageBackend: serviceLocator.getStorageBackend(),
@@ -744,7 +747,7 @@ export class BasicCrawler {
             await this.addRequests(requests, addRequestsOptions);
         }
         try {
-            await this._init();
+            await this.init();
             await this.stats.startCapturing();
         }
         catch (error) {
@@ -761,7 +764,7 @@ export class BasicCrawler {
         const sigintHandler = async () => {
             this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
             await this.pauseOnMigration();
-            await this.autoscaledPool.abort();
+            await this.#autoscaledPool.abort();
         };
         // Attach a listener to handle migration and aborting events gracefully.
         const boundPauseOnMigration = this.pauseOnMigration.bind(this);
@@ -771,7 +774,7 @@ export class BasicCrawler {
         eventManager.on("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
         let stats = {};
         try {
-            await this.autoscaledPool.run();
+            await this.#autoscaledPool.run();
         }
         finally {
             await this.teardown();
@@ -821,11 +824,11 @@ export class BasicCrawler {
      * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
     stop(reason = 'The crawler has been gracefully stopped.') {
-        if (this.unexpectedStop) {
+        if (this.#unexpectedStop) {
             return;
         }
         this.log.info(reason);
-        this.unexpectedStop = true;
+        this.#unexpectedStop = true;
     }
     /**
      * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
@@ -836,22 +839,22 @@ export class BasicCrawler {
      * throughout, since a shared one may still be serving other crawlers.
      */
     async pause(timeoutSecs) {
-        if (!this.autoscaledPool) {
+        if (!this.#autoscaledPool) {
             this.log.warning('Cannot pause a crawler that is not running.');
             return;
         }
-        await this.autoscaledPool.pause(timeoutSecs);
+        await this.#autoscaledPool.pause(timeoutSecs);
     }
     /**
      * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
      * again. A no-op on a crawler that is not paused.
      */
     resume() {
-        if (!this.autoscaledPool) {
+        if (!this.#autoscaledPool) {
             this.log.warning('Cannot resume a crawler that is not running.');
             return;
         }
-        this.autoscaledPool.resume();
+        this.#autoscaledPool.resume();
     }
     /**
      * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
@@ -864,8 +867,8 @@ export class BasicCrawler {
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
         // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
         // but guard so we do not re-issue it on every call.
-        if (!this.requestManagerTimeoutsApplied) {
-            this.requestManagerTimeoutsApplied = true;
+        if (!this.#requestManagerTimeoutsApplied) {
+            this.#requestManagerTimeoutsApplied = true;
             await this.applyRequestManagerTimeouts(this.requestManager);
         }
         return this.requestManager;
@@ -887,7 +890,7 @@ export class BasicCrawler {
         // subsequent instances get their own queue via a unique alias so they don't collide.
         const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
         const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
-        return this.ownedRequestQueue.set(requestQueue);
+        return this.#ownedRequestQueue.set(requestQueue);
     }
     /**
      * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
@@ -936,8 +939,8 @@ export class BasicCrawler {
             const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
             return kvs.getAutoSavedValue(stateKey, defaultValue);
         }
-        BasicCrawler.useStateAnonymousIndices.add(this.identity.instanceIndex);
-        if (BasicCrawler.useStateAnonymousIndices.size > 1) {
+        BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);
+        if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
             serviceLocator
                 .getLogger()
                 .warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
@@ -972,9 +975,9 @@ export class BasicCrawler {
         });
     }
     logOncePerRun(key, message) {
-        if (!this.loggedPerRun.has(key)) {
+        if (!this.#loggedPerRun.has(key)) {
             this.log.info(message);
-            this.loggedPerRun.add(key);
+            this.#loggedPerRun.add(key);
         }
     }
     /**
@@ -1112,11 +1115,11 @@ export class BasicCrawler {
     /**
      * Initializes the crawler.
      */
-    async _init() {
+    async init() {
         const eventManager = serviceLocator.getEventManager();
         if (!eventManager.isInitialized()) {
             await eventManager.init();
-            this._closeEvents = true;
+            this.#closeEvents = true;
         }
         // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
         // floored per request so it will not actually cut them short, but the configured value is then effectively
@@ -1131,11 +1134,11 @@ export class BasicCrawler {
         // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
         // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
         // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
-        this.concurrencySystemDep = this.resolveConcurrencySystem();
-        await this.concurrencySystemDep.ifOwned((system) => system.start());
-        this.autoscaledPool = new AutoscaledPool({
+        this.#concurrencySystemDep = this.#resolveConcurrencySystem();
+        await this.#concurrencySystemDep.ifOwned((system) => system.start());
+        this.#autoscaledPool = new AutoscaledPool({
             ...this.taskLoopOptions,
-            concurrencySystem: this.concurrencySystemDep.value,
+            concurrencySystem: this.#concurrencySystemDep.value,
             consumer: this.identity,
         });
         await this.getRequestManager();
@@ -1192,11 +1195,11 @@ export class BasicCrawler {
      * a normal return says nothing about success. `handleRequest` owns the outcome.
      */
     async runInStorageTransaction(callback) {
-        if (!this.transactionalStorageEnabled) {
+        if (!this.#transactionalStorageEnabled) {
             return callback();
         }
         const transaction = createStorageTransaction({
-            policy: this.storageWritePolicy,
+            policy: this.#storageWritePolicy,
             commitTimeoutMillis: this.internalTimeoutMillis,
         });
         let threw = true;
@@ -1223,7 +1226,7 @@ export class BasicCrawler {
     /**
      * Handles blocked request
      */
-    _throwOnBlockedRequest(statusCode) {
+    throwOnBlockedRequest(statusCode) {
         if (this.retryOnBlocked)
             return;
         if (this.blockedStatusCodes.has(statusCode)) {
@@ -1231,25 +1234,25 @@ export class BasicCrawler {
         }
     }
     async isAllowedBasedOnRobotsTxtFile(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return true;
         }
         const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
-        const userAgent = typeof this.respectRobotsTxtFile === 'object' ? this.respectRobotsTxtFile?.userAgent : '*';
+        const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';
         return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
     }
     async getRobotsTxtFileForUrl(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return undefined;
         }
         try {
             const origin = new URL(url).origin;
-            const cachedRobotsTxtFile = this.robotsTxtFileCache.get(origin);
+            const cachedRobotsTxtFile = this.#robotsTxtFileCache.get(origin);
             if (cachedRobotsTxtFile) {
                 return cachedRobotsTxtFile;
             }
             const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
-            this.robotsTxtFileCache.add(origin, robotsTxtFile);
+            this.#robotsTxtFileCache.add(origin, robotsTxtFile);
             return robotsTxtFile;
         }
         catch (e) {
@@ -1258,9 +1261,9 @@ export class BasicCrawler {
         }
     }
     async pauseOnMigration() {
-        if (this.autoscaledPool) {
+        if (this.#autoscaledPool) {
             // if run wasn't called, this is going to crash
-            await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
+            await this.#autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
                 if (err.message.includes('running tasks did not finish')) {
                     this.log.error('The crawler was paused due to migration to another host, ' +
                         "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -1311,12 +1314,12 @@ export class BasicCrawler {
             return false;
         }
         const now = Date.now();
-        const lastAccessTime = this.domainAccessedTime.get(domain);
-        if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {
-            this.domainAccessedTime.set(domain, now);
+        const lastAccessTime = this.#domainAccessedTime.get(domain);
+        if (!lastAccessTime || now - lastAccessTime >= this.#sameDomainDelayMillis) {
+            this.#domainAccessedTime.set(domain, now);
             return false;
         }
-        const delay = lastAccessTime + this.sameDomainDelayMillis - now;
+        const delay = lastAccessTime + this.#sameDomainDelayMillis - now;
         this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
         setTimeout(async () => {
             this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
@@ -1434,7 +1437,7 @@ export class BasicCrawler {
         return await enqueueLinks({
             requestManager,
             robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
-            respectRobotsTxtFile: this.respectRobotsTxtFile,
+            respectRobotsTxtFile: this.#respectRobotsTxtFile,
             onSkippedRequest,
             limit: await this.calculateEnqueuedRequestLimit(options.limit),
             // Allow user options to override defaults set above ⤴
@@ -1521,7 +1524,7 @@ export class BasicCrawler {
                 const { url, retryCount, id } = request;
                 // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
                 // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
-                const message = this._getMessageFromError(error);
+                const message = this.getMessageFromError(error);
                 this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {
                     id,
                     url,
@@ -1554,7 +1557,7 @@ export class BasicCrawler {
     async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
-        const message = this._getMessageFromError(error, true);
+        const message = this.getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
         if (this.failedRequestHandler) {
             await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
@@ -1566,7 +1569,7 @@ export class BasicCrawler {
      * @param error The error received
      * @returns The message to be logged
      */
-    _getMessageFromError(error, forceStack = false) {
+    getMessageFromError(error, forceStack = false) {
         if ([TypeError, SyntaxError, ReferenceError].some((type) => error instanceof type)) {
             forceStack = true;
         }
@@ -1602,14 +1605,14 @@ export class BasicCrawler {
      */
     async teardown() {
         serviceLocator.getEventManager().emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
-        if (this._closeEvents) {
+        if (this.#closeEvents) {
             await serviceLocator.getEventManager().close();
         }
-        await this.sessionPoolDep.ifOwned((pool) => pool.teardown());
-        await this.autoscaledPool?.abort();
-        await this.concurrencySystemDep?.ifOwned((system) => system.stop());
+        await this.#sessionPoolDep.ifOwned((pool) => pool.teardown());
+        await this.#autoscaledPool?.abort();
+        await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
     }
-    _getCookieHeaderFromRequest(request) {
+    getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {
             this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
             return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.105",
+    "version": "4.0.0-beta.107",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.105",
-        "@crawlee/http-client": "4.0.0-beta.105",
-        "@crawlee/types": "4.0.0-beta.105",
-        "@crawlee/utils": "4.0.0-beta.105",
+        "@crawlee/core": "4.0.0-beta.107",
+        "@crawlee/http-client": "4.0.0-beta.107",
+        "@crawlee/types": "4.0.0-beta.107",
+        "@crawlee/utils": "4.0.0-beta.107",
         "csv-stringify": "^6.5.2",
         "ow": "^2.0.0",
         "tldts": "^7.0.6",
@@ -53,7 +53,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.105"
+        "@crawlee/impit-client": "^4.0.0-beta.107"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "26073a822c7699ac487931383a39192bc3daae7a"
+    "gitHead": "5a3cb6242d0ae261df93c5e03cfebbb9b736e6b7"
 }