@crawlee/basic 4.0.0-beta.7 → 4.0.0-beta.70

package/internals/basic-crawler.d.ts

@@ -1,38 +1,13 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, BaseHttpClient, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IRequestList, LoadedContext, ProxyInfo, Request, RequestOptions, RestrictedCrawlingContext, RouterHandler, RouterRoutes, Session, SessionPoolOptions, SkippedRequestCallback, Source, StatisticsOptions, StatisticState } from '@crawlee/core';
-import { AutoscaledPool, Configuration, Dataset, RequestProvider, SessionPool, Statistics } from '@crawlee/core';
-import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IRequestLoader, IRequestManager, ProxyConfiguration, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier } from '@crawlee/core';
+import { AutoscaledPool, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
+import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
-import type { SetRequired } from 'type-fest';
-import type { Log } from '@apify/log';
+import type { ReadonlyDeep, SetRequired } from 'type-fest';
 import { TimeoutError } from '@apify/timeout';
-export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<BasicCrawler, UserData> {
-    /**
-     * This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
-     * currently used by the crawler.
-     *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@link Request} objects.
-     *
-     * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
-     * for more details regarding its usage.
-     *
-     * **Example usage**
-     *
-     * ```ts
-     * async requestHandler({ enqueueLinks }) {
-     *     await enqueueLinks({
-     *       urls: [...],
-     *     });
-     * },
-     * ```
-     *
-     * @param [options] All `enqueueLinks()` parameters are passed via an options object.
-     * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
-     */
-    enqueueLinks(options?: SetRequired<EnqueueLinksOptions, 'urls'>): Promise<BatchAddRequestsResult>;
+export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
 }
-export type RequestHandler<Context extends CrawlingContext = LoadedContext<BasicCrawlingContext & RestrictedCrawlingContext>> = (inputs: LoadedContext<Context>) => Awaitable<void>;
-export type ErrorHandler<Context extends CrawlingContext = LoadedContext<BasicCrawlingContext & RestrictedCrawlingContext>> = (inputs: LoadedContext<Context>, error: Error) => Awaitable<void>;
+export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
+export type ErrorHandler<Context extends CrawlingContext = CrawlingContext, ExtendedContext extends Context = Context> = (inputs: Context & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
 export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> {
     state: StatisticState;
     crawler: Crawler;
@@ -40,7 +15,10 @@ export interface StatusMessageCallbackParams<Context extends CrawlingContext = B
     message: string;
 }
 export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
-export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlingContext> {
+export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
+    contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
+};
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
@@ -58,21 +36,58 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      * The exceptions are logged to the request using the
      * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
      */
-    requestHandler?: RequestHandler<Context>;
+    requestHandler?: RequestHandler<ExtendedContext>;
+    /**
+     * Allows the user to extend the crawling context passed to the request handler with custom functionality.
+     *
+     * **Example usage:**
+     *
+     * ```javascript
+     * import { BasicCrawler } from 'crawlee';
+     *
+     * // Create a crawler instance
+     * const crawler = new BasicCrawler({
+     *     extendContext(context) => ({
+     *         async customHelper() {
+     *             await context.pushData({ url: context.request.url })
+     *         }
+     *     }),
+     *     async requestHandler(context) {
+     *         await context.customHelper();
+     *     },
+     * });
+     * ```
+     */
+    extendContext?: (context: Context) => Awaitable<ContextExtension>;
+    /**
+     * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
+     *
+     * The option is not required if your crawler subclass does not extend the crawling context with custom information or helpers.
+     */
+    contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
     /**
      * Static list of URLs to be processed.
-     * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
-     * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
-     * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
+     *
+     * @deprecated Use the `requestManager` option instead. To combine a read-only loader (such as a `RequestList`)
+     * with a writable queue, build a tandem with {@link IRequestLoader.toTandem|`requestList.toTandem(requestQueue)`}
+     * and pass the result as `requestManager`. When both `requestList` and `requestQueue` are provided, they are
+     * combined into a tandem automatically.
      */
-    requestList?: IRequestList;
+    requestList?: IRequestLoader;
     /**
      * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-     * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
-     * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
-     * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
+     *
+     * @deprecated Use the `requestManager` option instead. A `RequestQueue` is itself a request manager, so you can
+     * pass it directly as `requestManager`.
+     */
+    requestQueue?: RequestQueue;
+    /**
+     * Manager of requests that should be processed by the crawler. Mutually exclusive with the deprecated
+     * `requestQueue` and `requestList` options.
+     *
+     * If not provided, the crawler will open the default {@link RequestQueue} when it is first needed.
      */
-    requestQueue?: RequestProvider;
+    requestManager?: IRequestManager;
     /**
      * Timeout in which the function passed as {@link BasicCrawlerOptions.requestHandler|`requestHandler`} needs to finish, in seconds.
      * @default 60
@@ -87,7 +102,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      */
-    errorHandler?: ErrorHandler<Context>;
+    errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     /**
      * A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
      *
@@ -96,14 +111,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      * Second argument is the `Error` instance that
      * represents the last error thrown during processing of the request.
      */
-    failedRequestHandler?: ErrorHandler<Context>;
+    failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     /**
      * Specifies the maximum number of retries allowed for a request if its processing fails.
-     * This includes retries due to navigation errors or errors thrown from user-supplied functions
-     * (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
-     *
-     * This limit does not apply to retries triggered by session rotation
-     * (see {@link BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).
+     * This includes retries due to navigation errors, session/proxy errors, or errors thrown from user-supplied
+     * functions (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
      * @default 3
      */
     maxRequestRetries?: number;
@@ -112,26 +124,24 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      * @default 0
      */
     sameDomainDelaySecs?: number;
-    /**
-     * Maximum number of session rotations per request.
-     * The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
-     *
-     * The session rotations are not counted towards the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} limit.
-     * @default 10
-     */
-    maxSessionRotations?: number;
     /**
      * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
      * This value should always be set in order to prevent infinite loops in misconfigured crawlers.
      * > *NOTE:* In cases of parallel crawling, the actual number of pages visited might be slightly higher than this value.
      */
     maxRequestsPerCrawl?: number;
+    /**
+     * Maximum depth of the crawl. If not set, the crawl will continue until all requests are processed.
+     * Setting this to `0` will only process the initial requests, skipping all links enqueued by `crawlingContext.enqueueLinks` and `crawlingContext.addRequests`.
+     * Passing `1` will process the initial requests and all links enqueued by `crawlingContext.enqueueLinks` and `crawlingContext.addRequests` in the handler for initial requests.
+     */
+    maxCrawlDepth?: number;
     /**
      * Custom options passed to the underlying {@link AutoscaledPool} constructor.
      * > *NOTE:* The {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
-     * and {@link AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`} options
-     * are provided by the crawler and cannot be overridden.
-     * However, we can provide a custom implementation of {@link AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}.
+     * option is provided by the crawler and cannot be overridden.
+     * However, we can provide custom implementations of {@link AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}
+     * and {@link AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`}.
      */
     autoscaledPoolOptions?: AutoscaledPoolOptions;
     /**
@@ -159,14 +169,14 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      */
     keepAlive?: boolean;
     /**
-     * Basic crawler will initialize the {@link SessionPool} with the corresponding {@link SessionPoolOptions|`sessionPoolOptions`}.
-     * The session instance will be than available in the {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
-     */
-    useSessionPool?: boolean;
-    /**
-     * The configuration options for {@link SessionPool} to use.
+     * An existing session pool instance to use. When provided, the crawler will use this pool directly instead of
+     * creating a new one, enabling session sharing across multiple crawlers. The crawler will not tear down a shared
+     * pool — the caller is responsible for its lifecycle.
+     *
+     * Accepts the built-in {@link SessionPool} or any object implementing the {@link ISessionPool} interface,
+     * so custom session-management strategies can be plugged in.
      */
-    sessionPoolOptions?: SessionPoolOptions;
+    sessionPool?: ISessionPool;
     /**
      * Defines the length of the interval for calling the `setStatusMessage` in seconds.
      */
@@ -188,6 +198,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
      * ```
      */
     statusMessageCallback?: StatusMessageCallback;
+    /**
+     * HTTP status codes that indicate the session should be retired.
+     * @default [401, 403, 429]
+     */
+    blockedStatusCodes?: number[];
     /**
      * If set to `true`, the crawler will automatically try to bypass any detected bot protection.
      *
@@ -199,20 +214,22 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
     /**
      * If set to `true`, the crawler will automatically try to fetch the robots.txt file for each domain,
      * and skip those that are not allowed. This also prevents disallowed URLs to be added via `enqueueLinks`.
+     *
+     * If an object is provided, it may contain a `userAgent` property to specify which user-agent
+     * should be used when checking the robots.txt file. If not provided, the default user-agent `*` will be used.
      */
-    respectRobotsTxtFile?: boolean;
+    respectRobotsTxtFile?: boolean | {
+        userAgent?: string;
+    };
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
-     * This is currently fired only for requests skipped based on robots.txt file.
+     * This is currently fired for requests skipped
+     * 1. based on robots.txt file,
+     * 2. because they don't match enqueueLinks filters,
+     * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
+     * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
      */
     onSkippedRequest?: SkippedRequestCallback;
-    /** @internal */
-    log?: Log;
-    /**
-     * Enables experimental features of Crawlee, which can alter the behavior of the crawler.
-     * WARNING: these options are not guaranteed to be stable and may change or be removed at any time.
-     */
-    experiments?: CrawlerExperiments;
     /**
      * Customize the way statistics collecting works, such as logging interval or
      * whether to output them to the Key-Value store.
@@ -220,24 +237,56 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
     statisticsOptions?: StatisticsOptions;
     /**
      * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
-     * Defaults to a new instance of {@link GotScrapingHttpClient}
+     * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
      */
     httpClient?: BaseHttpClient;
-}
-/**
- * A set of options that you can toggle to enable experimental features in Crawlee.
- *
- * NOTE: These options will not respect semantic versioning and may be removed or changed at any time. Use at your own risk.
- * If you do use these and encounter issues, please report them to us.
- */
-export interface CrawlerExperiments {
     /**
-     * @deprecated This experiment is now enabled by default, and this flag will be removed in a future release.
-     * If you encounter issues due to this change, please:
-     * - report it to us: https://github.com/apify/crawlee
-     * - set `requestLocking` to `false` in the `experiments` option of the crawler
+     * If set, the crawler will be configured for all connections to use
+     * the Proxy URLs provided and rotated according to the configuration.
+     */
+    proxyConfiguration?: ProxyConfiguration;
+    /**
+     * Custom configuration to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
+     */
+    configuration?: Configuration;
+    /**
+     * Custom storage backend to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
      */
-    requestLocking?: boolean;
+    storageBackend?: StorageBackend;
+    /**
+     * Custom event manager to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
+     */
+    eventManager?: EventManager;
+    /**
+     * Custom logger to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
+     */
+    logger?: CrawleeLogger;
+    /**
+     * A unique identifier for the crawler instance. This ID is used to isolate the state returned by
+     * {@link BasicCrawler.useState|`crawler.useState()`} from other crawler instances.
+     *
+     * When multiple crawler instances use `useState()` without an explicit `id`, they will share the same
+     * state object for backward compatibility. A warning will be logged in this case.
+     *
+     * To ensure each crawler has its own isolated state that also persists across script restarts
+     * (e.g., during Apify migrations), provide a stable, unique ID for each crawler instance.
+     *
+     */
+    id?: string;
+    /**
+     * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
+     * By default, status codes >= 500 trigger errors.
+     */
+    ignoreHttpErrorStatusCodes?: number[];
+    /**
+     * An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
+     * By default, status codes >= 500 trigger errors.
+     */
+    additionalHttpErrorStatusCodes?: number[];
 }
 /**
  * Provides a simple framework for parallel crawling of web pages.
@@ -251,15 +300,22 @@ export interface CrawlerExperiments {
  *
  * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
  * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
- * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
- * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
- * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
+ * The {@link Request} objects are fed from the {@link IRequestManager|request manager} provided via the
+ * {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is
+ * itself a request manager). If no `requestManager` is provided, the crawler opens the default {@link RequestQueue}
+ * either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called, or if the `requests`
+ * parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
+ *
+ * To read requests from a read-only source such as a {@link RequestList} or {@link SitemapRequestLoader} while
+ * still being able to enqueue new ones, combine the loader with a queue into a {@link RequestManagerTandem} using
+ * {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`. The tandem
+ * first processes URLs from the loader and automatically enqueues them into the queue, ensuring a single URL is not
+ * crawled multiple times.
  *
- * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The legacy {@link BasicCrawlerOptions.requestList|`requestList`} and
+ * > {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated. They are still accepted and
+ * > folded into a single `requestManager` (combined into a tandem when both are given), but new code should use
+ * > `requestManager` directly.
  *
  * The crawler finishes if there are no more {@link Request} objects to crawl.
  *
@@ -303,29 +359,54 @@ export interface CrawlerExperiments {
  * ```
  * @category Crawlers
  */
-export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext> {
-    readonly config: Configuration;
+export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
+    #private;
     protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
+    /**
+     * Tracks crawler instances that accessed shared state without having an explicit id.
+     * Used to detect and warn about multiple crawlers sharing the same state.
+     */
+    private static useStateCrawlerIds;
+    /**
+     * Tracks the number of crawler instances created. The first crawler uses the default
+     * request queue; subsequent ones get their own queue via a unique alias so they don't
+     * collide.
+     */
+    private static instanceCount;
     /**
      * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
      */
     readonly stats: Statistics;
     /**
-     * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
-     * Only available if used by the crawler.
+     * The main request-handling component of the crawler. It manages the requests that the crawler processes,
+     * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
+     * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
-    requestList?: IRequestList;
+    protected requestManager?: IRequestManager;
     /**
-     * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-     * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
-     * Only available if used by the crawler.
+     * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
+     * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
-    requestQueue?: RequestProvider;
+    sessionPool: ISessionPool;
     /**
-     * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
-     * Only available if used by the crawler.
+     * Set when the crawler constructed its own {@link SessionPool} (no `sessionPool` option was provided).
+     * Holds the same instance as `sessionPool`, but typed as the concrete class so the crawler can call
+     * lifecycle methods (`resetStore`, `teardown`) that aren't part of {@link ISessionPool}. A user-supplied
+     * pool is never owned and never torn down by the crawler.
+     */
+    private ownedSessionPool?;
+    /**
+     * Set when the crawler constructed its own request manager (no `requestManager`, `requestQueue`, or `requestList`
+     * option was provided). The owned manager is purged (not dropped) between repeated `run()` calls.
+     * A user-supplied manager is never purged by the crawler.
+     */
+    private ownedRequestManager?;
+    /**
+     * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
+     * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
+     * so it only needs to be applied once, at the first async access of the manager.
      */
-    sessionPool?: SessionPool;
+    private requestManagerTimeoutsApplied;
     /**
      * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
      * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
@@ -334,40 +415,69 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
      * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
      */
     autoscaledPool?: AutoscaledPool;
+    /**
+     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
+     * Only available if used by the crawler.
+     */
+    proxyConfiguration?: ProxyConfiguration;
     /**
      * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
-    readonly router: RouterHandler<LoadedContext<Context>>;
+    readonly router: RouterHandler<Context>;
+    private _basicContextPipeline?;
+    /**
+     * The basic part of the context pipeline. Unlike the subclass pipeline, this
+     * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
+     * pipelines expect the basic crawler fields to already be present in the context at runtime.
+     *
+     * Context built with this pipeline can be passed into multiple crawler pipelines at once.
+     * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
+     */
+    get basicContextPipeline(): ContextPipeline<{
+        request: Request;
+    }, CrawlingContext>;
+    private _contextPipeline?;
+    get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
     running: boolean;
     hasFinishedBefore: boolean;
-    readonly log: Log;
-    protected requestHandler: RequestHandler<Context>;
-    protected errorHandler?: ErrorHandler<Context>;
-    protected failedRequestHandler?: ErrorHandler<Context>;
+    protected unexpectedStop: boolean;
+    get log(): CrawleeLogger;
+    protected requestHandler: RequestHandler<ExtendedContext>;
+    protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
+    protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     protected requestHandlerTimeoutMillis: number;
     protected internalTimeoutMillis: number;
     protected maxRequestRetries: number;
+    protected maxCrawlDepth?: number;
     protected sameDomainDelayMillis: number;
     protected domainAccessedTime: Map<string, number>;
-    protected maxSessionRotations: number;
+    protected maxRequestsPerCrawl?: number;
     protected handledRequestsCount: number;
     protected statusMessageLoggingInterval: number;
     protected statusMessageCallback?: StatusMessageCallback;
-    protected sessionPoolOptions: SessionPoolOptions;
-    protected useSessionPool: boolean;
-    protected crawlingContexts: Map<string, Context>;
+    protected blockedStatusCodes: Set<number>;
+    protected additionalHttpErrorStatusCodes: Set<number>;
+    protected ignoreHttpErrorStatusCodes: Set<number>;
     protected autoscaledPoolOptions: AutoscaledPoolOptions;
-    protected events: EventManager;
     protected httpClient: BaseHttpClient;
     protected retryOnBlocked: boolean;
-    protected respectRobotsTxtFile: boolean;
+    protected respectRobotsTxtFile: boolean | {
+        userAgent?: string;
+    };
     protected onSkippedRequest?: SkippedRequestCallback;
     private _closeEvents?;
-    private experiments;
+    private loggedPerRun;
     private readonly robotsTxtFileCache;
-    private _experimentWarnings;
+    private readonly crawlerId;
+    private readonly hasExplicitId;
+    private readonly crawlerInstanceIndex;
+    private readonly contextPipelineOptions;
     protected static optionsShape: {
+// @ts-ignore optional peer dependency or compatibility with es2022
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -384,28 +494,42 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        blockedStatusCodes: import("ow").ArrayPredicate<number>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -414,17 +538,40 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
         maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options?: BasicCrawlerOptions<Context>, config?: Configuration);
+    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);
+    /**
+     * Determines if the given HTTP status code is an error status code given
+     * the default behaviour and user-set preferences.
+     * @param status
+     * @returns `true` if the status code is considered an error, `false` otherwise
+     */
+    protected isErrorStatusCode(status: number): boolean;
+    /**
+     * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
+     * This handles base context creation, session resolution, and context helpers.
+     */
+    protected buildBasicContextPipeline(): ContextPipeline<{
+        request: Request;
+    }, CrawlingContext>;
+    private checkRobotsTxt;
+    /**
+     * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
+     * Subclasses should override this to add their own pipeline stages.
+     */
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
+    private createBaseContext;
+    private resolveRequest;
+    private resolveSession;
+    private createContextHelpers;
+    private buildFinalContextPipeline;
     /**
      * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
      * Used for retrying requests that failed due to proxy errors.
@@ -433,15 +580,15 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
      */
     protected isProxyError(error: Error): boolean;
     /**
-     * Checks whether the given crawling context is getting blocked by anti-bot protection using several heuristics.
-     * Returns `false` if the request is not blocked, otherwise returns a string with a description of the block reason.
-     * @param _crawlingContext The crawling context to check.
-     */
-    protected isRequestBlocked(_crawlingContext: Context): Promise<string | false>;
-    /**
+     * Sets the status message for the current crawler run.
+     *
      * This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
+     *
+     * The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
+     * event. Integrations such as the Apify SDK subscribe to that event and forward the message to
+     * their status-reporting backend (e.g. the Apify platform).
      */
-    setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
+    setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
     private getPeriodicLogger;
     /**
      * Runs the crawler. Returns a promise that resolves once all the requests are processed
@@ -453,15 +600,44 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
      * @param [requests] The requests to add.
      * @param [options] Options for the request queue.
      */
-    run(requests?: (string | Request | RequestOptions)[], options?: CrawlerRunOptions): Promise<FinalStatistics>;
+    run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;
     /**
      * Gracefully stops the current run of the crawler.
      *
      * All the tasks active at the time of calling this method will be allowed to finish.
+     *
+     * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
+     */
+    stop(reason?: string): void;
+    /**
+     * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
+     * if none has been configured or opened yet.
+     */
+    getRequestManager(): Promise<IRequestManager>;
+    /**
+     * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
+     * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
+     */
+    getRequestQueue(): Promise<IRequestManager>;
+    /**
+     * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
+     * crawler-owned manager (so it gets purged between repeated `run()` calls).
+     * @private
      */
-    stop(message?: string): void;
-    getRequestQueue(): Promise<RequestProvider>;
+    private openOwnedRequestQueue;
+    /**
+     * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
+     * locking storage backend keeps it reserved for slightly longer than the request handler timeout
+     * (with some padding for overhead), but never for less than a minute. This prevents a long-running
+     * request from being handed out a second time while it is still being processed — and it works
+     * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
+     */
+    private applyRequestManagerTimeouts;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
+    protected getPendingRequestCountApproximation(): Promise<number>;
+    protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;
+    protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
+    private logOncePerRun;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -473,15 +649,15 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
      * @param requests The requests to add
      * @param options Options for the request queue
      */
-    addRequests(requests: (string | Source)[], options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
+    addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
     /**
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
      */
-    pushData(data: Parameters<Dataset['pushData']>[0], datasetIdOrName?: string): Promise<void>;
+    pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
     /**
      * Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
      */
-    getDataset(idOrName?: string): Promise<Dataset>;
+    getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
     /**
      * Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
      */
@@ -491,41 +667,48 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
      * Supported formats are currently 'json' and 'csv', and will be inferred from the `path` automatically.
      */
     exportData<Data>(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise<Data[]>;
+    /**
+     * Initializes the crawler.
+     */
     protected _init(): Promise<void>;
-    protected _runRequestHandler(crawlingContext: Context): Promise<void>;
+    protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     /**
      * Handles blocked request
      */
-    protected _throwOnBlockedRequest(session: Session, statusCode: number): void;
+    protected _throwOnBlockedRequest(statusCode: number): void;
     private isAllowedBasedOnRobotsTxtFile;
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
     protected _pauseOnMigration(): Promise<void>;
     /**
-     * Fetches request from either RequestList or RequestQueue. If request comes from a RequestList
-     * and RequestQueue is present then enqueues it to the queue first.
+     * Fetches the next request to process from the underlying request provider.
      */
-    protected _fetchNextRequest(): Promise<Request<Dictionary> | null | undefined>;
-    /**
-     * Executed when `errorHandler` finishes or the request is successful.
-     * Can be used to clean up orphaned browser pages.
-     */
-    protected _cleanupContext(_crawlingContext: Context): Promise<void>;
+    protected _fetchNextRequest(): Promise<Request<Dictionary> | null>;
     /**
      * Delays processing of the request based on the `sameDomainDelaySecs` option,
      * adding it back to the queue after the timeout passes. Returns `true` if the request
      * should be ignored and will be reclaimed to the queue once ready.
      */
-    protected delayRequest(request: Request, source: IRequestList | RequestProvider): boolean;
+    protected delayRequest(request: Request, source: IRequestManager): boolean;
+    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    protected handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request): Promise<void>;
+    /**
+     * Wrapper around the crawling context's `enqueueLinks` method:
+     * - Injects `crawlDepth` to each request being added based on the crawling context request.
+     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
+     *      - These options can be overridden by the user.
+     * @internal
+     */
+    protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestManager: IRequestManager): Promise<BatchAddRequestsResult>;
     /**
-     * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
-     * then retries them in a case of an error, etc.
+     * Generator function that yields requests injected with the given crawl depth.
+     * @internal
      */
-    protected _runTaskFunction(): Promise<void>;
+    protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
     /**
-     * Run async callback with given timeout and retry.
+     * Run async callback with given timeout and retry. Returns the result of the callback.
      * @ignore
      */
-    protected _timeoutAndRetry(handler: () => Promise<unknown>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<void>;
+    protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
     /**
      * Returns true if either RequestList or RequestQueue have a request ready for processing.
      */
@@ -534,13 +717,19 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
      * Returns true if both RequestList and RequestQueue have all requests finished.
      */
     protected _defaultIsFinishedFunction(): Promise<boolean>;
-    private _rotateSession;
+    /**
+     * Unwraps errors thrown by the context pipeline to get the actual user error.
+     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
+     */
+    private unwrapError;
     /**
      * Handles errors thrown by user provided requestHandler()
+     *
+     * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
-    protected _requestFunctionErrorHandler(error: Error, crawlingContext: Context, source: IRequestList | RequestProvider): Promise<void>;
+    protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, request: Request, source: IRequestManager): Promise<void>;
     protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
-    protected _handleFailedRequestHandler(crawlingContext: Context, error: Error): Promise<void>;
+    protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
     /**
      * Resolves the most verbose error message from a thrown error
      * @param error The error received
@@ -549,27 +738,23 @@ export declare class BasicCrawler<Context extends CrawlingContext = BasicCrawlin
     protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
     protected _canRequestBeRetried(request: Request, error: Error): boolean;
     /**
-     * Updates handledRequestsCount from possibly stored counts,
-     * usually after worker migration. Since one of the stores
-     * needs to have priority when both are present,
-     * it is the request queue, because generally, the request
-     * list will first be dumped into the queue and then left
-     * empty.
+     * Updates handledRequestsCount from possibly stored counts, usually after worker migration.
      */
     protected _loadHandledRequestCount(): Promise<void>;
-    protected _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(hooks: HookLike[], ...args: Parameters<HookLike>): Promise<void>;
     /**
-     * Function for cleaning up after all request are processed.
-     * @ignore
+     * Stops the crawler immediately.
+     *
+     * This method doesn't wait for currently active requests to finish.
+     *
+     * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     teardown(): Promise<void>;
     protected _getCookieHeaderFromRequest(request: Request): string;
-    private _getRequestQueue;
-    protected requestMatchesEnqueueStrategy(request: Request): boolean;
+    private requestMatchesEnqueueStrategy;
 }
 export interface CreateContextOptions {
     request: Request;
-    session?: Session;
+    session: ISession;
     proxyInfo?: ProxyInfo;
 }
 export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
@@ -578,9 +763,14 @@ export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
 }
 export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
     /**
-     * Whether to purge the RequestQueue before running the crawler again. Defaults to true, so it is possible to reprocess failed requests.
-     * When disabled, only new requests will be considered. Note that even a failed request is considered as handled.
-     * @default true
+     * Controls whether the request queue is purged between repeated `run()` calls on the same crawler instance.
+     * Purging clears all requests and resets internal counters, allowing the same URLs to be processed again.
+     *
+     * - **`undefined`** (default) — only the crawler's own (auto-created) queue is purged.
+     *   A user-supplied `requestQueue` is left untouched.
+     * - **`true`** — the queue is always purged, even if it was supplied by the user.
+     * - **`false`** — nothing is purged. Only genuinely new requests will be processed;
+     *   note that even a failed request is considered handled.
      */
     purgeRequestQueue?: boolean;
 }