@crawlee/basic 4.0.0-beta.19 → 4.0.0-beta.191

package/internals/basic-crawler.d.ts

@@ -1,25 +1,45 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, AutoscaledPoolOptions, BaseHttpClient, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IRequestList, IRequestManager, ProxyConfiguration, ProxyInfo, Request, RequestsLike, RouterHandler, RouterRoutes, Session, SessionPoolOptions, SkippedRequestCallback, Source, StatisticsOptions, StatisticState } from '@crawlee/core';
-import { AutoscaledPool, Configuration, ContextPipeline, Dataset, RequestProvider, SessionPool, Statistics } from '@crawlee/core';
-import type { Awaitable, BatchAddRequestsResult, Dictionary, SetStatusMessageOptions } from '@crawlee/types';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, CrawleeLogger, DatasetExportOptions, IProxyConfiguration, IRequestLoader, IRequestManager, RequestsLike, Source, StorageIdentifier, StorageWritePolicy } from '@crawlee/core';
+import { Configuration, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
+import { BaseHttpClient } from '@crawlee/http-client';
+import type { Awaitable, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
-import type { ReadonlyDeep, SetRequired } from 'type-fest';
-import type { Log } from '@apify/log';
+import type { ReadonlyDeep } from 'type-fest';
+import { z } from 'zod';
 import { TimeoutError } from '@apify/timeout';
+import type { TaskLoopOptions } from './autoscaling/autoscaled_pool.js';
+import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency_system.js';
+import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
+import type { FinalStatistics } from './autoscaling/system_status.js';
+import { ContextPipeline } from './crawlers/context_pipeline.js';
+import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler_commons.js';
+import { CrawlingRequest } from './crawling_request.js';
+import type { IStatistics, StatisticState } from './crawlers/statistics.js';
+import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue_links/index.js';
+import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
 export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
 }
+export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
 export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
-export type ErrorHandler<Context extends CrawlingContext = CrawlingContext, ExtendedContext extends Context = Context> = (inputs: Context & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
-export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> {
+/**
+ * An error handler receives the crawling context and the error that was thrown while processing the request.
+ *
+ * Unlike the {@link RequestHandler}, an error handler may run before the context pipeline has finished
+ * building the full context (e.g. when navigation or session setup fails). Therefore only `BaseContext` is
+ * guaranteed to be present, while the extra properties added by the pipeline and `extendContext` (the
+ * difference between `BaseContext` and `ExtendedContext`) are only available as a `Partial`.
+ */
+export type ErrorHandler<BaseContext extends CrawlingContext = CrawlingContext, ExtendedContext extends BaseContext = BaseContext> = (inputs: BaseContext & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
+export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> {
     state: StatisticState;
     crawler: Crawler;
     previousState: StatisticState;
     message: string;
 }
-export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
+export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
 export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
     contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
 };
-export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
@@ -35,11 +55,17 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * To make this work, we should **always**
      * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+     * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
      */
-    requestHandler?: RequestHandler<ExtendedContext>;
+    requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
-     * Allows the user to extend the crawling context passed to the request handler with custom functionality.
+     * Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
+     *
+     * `extendContext` runs *before* navigation, so the returned members are visible to the
+     * `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence,
+     * the `context` passed to `extendContext` is the pre-navigation {@link CrawlingContext} and does
+     * **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
+     * those, use a `postNavigationHook` or the `requestHandler` instead.
      *
      * **Example usage:**
      *
@@ -59,7 +85,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * });
      * ```
      */
-    extendContext?: (context: Context) => Awaitable<ContextExtension>;
+    extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
     /**
      * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
      *
@@ -68,23 +94,25 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
     /**
      * Static list of URLs to be processed.
-     * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
-     * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
-     * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
+     *
+     * @deprecated Use the `requestManager` option instead. To combine a read-only loader (such as a `RequestList`)
+     * with a writable queue, build a tandem with {@link IRequestLoader.toTandem|`requestList.toTandem(requestQueue)`}
+     * and pass the result as `requestManager`. When both `requestList` and `requestQueue` are provided, they are
+     * combined into a tandem automatically.
      */
-    requestList?: IRequestList;
+    requestList?: IRequestLoader;
     /**
      * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-     * If not provided, the crawler will open the default request queue when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called.
-     * > Alternatively, `requests` parameter of {@link BasicCrawler.run|`crawler.run()`} could be used to enqueue the initial requests -
-     * it is a shortcut for running `crawler.addRequests()` before the `crawler.run()`.
+     *
+     * @deprecated Use the `requestManager` option instead. A `RequestQueue` is itself a request manager, so you can
+     * pass it directly as `requestManager`.
      */
-    requestQueue?: RequestProvider;
+    requestQueue?: RequestQueue;
     /**
-     * Allows explicitly configuring a request manager. Mutually exclusive with the `requestQueue` and `requestList` options.
+     * Manager of requests that should be processed by the crawler. Mutually exclusive with the deprecated
+     * `requestQueue` and `requestList` options.
      *
-     * This enables explicitly configuring the crawler to use `RequestManagerTandem`, for instance.
-     * If using this, the type of `BasicCrawler.requestQueue` may not be fully compatible with the `RequestProvider` class.
+     * If not provided, the crawler will open the default {@link RequestQueue} when it is first needed.
      */
     requestManager?: IRequestManager;
     /**
@@ -113,27 +141,21 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     /**
      * Specifies the maximum number of retries allowed for a request if its processing fails.
-     * This includes retries due to navigation errors or errors thrown from user-supplied functions
-     * (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
-     *
-     * This limit does not apply to retries triggered by session rotation
-     * (see {@link BasicCrawlerOptions.maxSessionRotations|`maxSessionRotations`}).
+     * This includes retries due to navigation errors, session/proxy errors, or errors thrown from user-supplied
+     * functions (`requestHandler`, `preNavigationHooks`, `postNavigationHooks`).
      * @default 3
      */
     maxRequestRetries?: number;
     /**
-     * Indicates how much time (in seconds) to wait before crawling another same domain request.
+     * Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
+     * paced together with the site they belong to.
+     *
+     * Offered to the crawler's request manager as a `minIntervalEverywhere` {@link PacingSignal}; a manager that
+     * already paces every domain it dispatches to takes it, so no domain ends up with two clocks. Otherwise the
+     * crawler wraps its request manager in a {@link ThrottlingRequestManager} of its own.
      * @default 0
      */
     sameDomainDelaySecs?: number;
-    /**
-     * Maximum number of session rotations per request.
-     * The crawler will automatically rotate the session in case of a proxy error or if it gets blocked by the website.
-     *
-     * The session rotations are not counted towards the {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} limit.
-     * @default 10
-     */
-    maxSessionRotations?: number;
     /**
      * Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
      * This value should always be set in order to prevent infinite loops in misconfigured crawlers.
@@ -147,29 +169,56 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     maxCrawlDepth?: number;
     /**
-     * Custom options passed to the underlying {@link AutoscaledPool} constructor.
-     * > *NOTE:* The {@link AutoscaledPoolOptions.runTaskFunction|`runTaskFunction`}
-     * option is provided by the crawler and cannot be overridden.
-     * However, we can provide custom implementations of {@link AutoscaledPoolOptions.isFinishedFunction|`isFinishedFunction`}
-     * and {@link AutoscaledPoolOptions.isTaskReadyFunction|`isTaskReadyFunction`}.
+     * Lets you override the predicates that steer the crawler's task loop: `isTaskReadyFunction` (may another request
+     * start?) and `isFinishedFunction` (is the crawl over?). The task itself — fetching a request and running it
+     * through the pipeline — is owned by the crawler and cannot be overridden.
+     *
+     * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
+     * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
      */
-    autoscaledPoolOptions?: AutoscaledPoolOptions;
+    taskLoopOptions?: TaskLoopOptions;
+    /**
+     * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
+     * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
+     * scaling configuration (min/max/desired concurrency, scaling ratios, `maxTasksPerMinute`, snapshotter tuning)
+     * lives on the instance itself.
+     *
+     * Inject the *same* instance into several concurrent crawlers to cap their **combined** concurrency against a
+     * single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
+     * accounting is shared.
+     *
+     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
+     * shortcuts, which configure the default system this one replaces — combining the two throws.
+     *
+     * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
+     * every crawler borrowing it has finished. The crawler does neither on your behalf.
+     */
+    concurrencySystem?: IConcurrencySystem;
     /**
      * Sets the minimum concurrency (parallelism) for the crawl. Shortcut for the
-     * AutoscaledPool {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} option.
+     * {@link ConcurrencySystemOptions.minConcurrency|`minConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}.
      * > *WARNING:* If we set this value too high with respect to the available system memory and CPU, our crawler will run extremely slow or crash.
      * If not sure, it's better to keep the default value and the concurrency will scale up automatically.
      */
     minConcurrency?: number;
     /**
      * Sets the maximum concurrency (parallelism) for the crawl. Shortcut for the
-     * AutoscaledPool {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} option.
+     * {@link ConcurrencySystemOptions.maxConcurrency|`maxConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}.
      */
     maxConcurrency?: number;
+    /**
+     * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
+     * {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}. Defaults to `minConcurrency`.
+     */
+    initialConcurrency?: number;
     /**
      * The maximum number of requests per minute the crawler should run.
      * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
-     * Shortcut for the AutoscaledPool {@link AutoscaledPoolOptions.maxTasksPerMinute|`maxTasksPerMinute`} option.
+     * Shortcut for the {@link ConcurrencySystemOptions.maxTasksPerMinute|`maxTasksPerMinute`} option of the
+     * crawler's default {@link ConcurrencySystem}.
      */
     maxRequestsPerMinute?: number;
     /**
@@ -179,14 +228,14 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     keepAlive?: boolean;
     /**
-     * Basic crawler will initialize the {@link SessionPool} with the corresponding {@link SessionPoolOptions|`sessionPoolOptions`}.
-     * The session instance will be than available in the {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
-     */
-    useSessionPool?: boolean;
-    /**
-     * The configuration options for {@link SessionPool} to use.
+     * An existing session pool instance to use. When provided, the crawler will use this pool directly instead of
+     * creating a new one, enabling session sharing across multiple crawlers. The crawler will not tear down a shared
+     * pool — the caller is responsible for its lifecycle.
+     *
+     * Accepts the built-in {@link SessionPool} or any object implementing the {@link ISessionPool} interface,
+     * so custom session-management strategies can be plugged in.
      */
-    sessionPoolOptions?: SessionPoolOptions;
+    sessionPool?: ISessionPool;
     /**
      * Defines the length of the interval for calling the `setStatusMessage` in seconds.
      */
@@ -208,6 +257,15 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * ```
      */
     statusMessageCallback?: StatusMessageCallback;
+    /**
+     * HTTP status codes that indicate the session should be retired.
+     *
+     * A 429 from a domain covered by a {@link ThrottlingRequestManager} is handled as a rate limit before
+     * this is consulted, so removing 429 here only affects domains that manager does not cover.
+     *
+     * @default [401, 403, 429]
+     */
+    blockedStatusCodes?: number[];
     /**
      * If set to `true`, the crawler will automatically try to bypass any detected bot protection.
      *
@@ -235,256 +293,251 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
      */
     onSkippedRequest?: SkippedRequestCallback;
-    /** @internal */
-    log?: Log;
-    /**
-     * Enables experimental features of Crawlee, which can alter the behavior of the crawler.
-     * WARNING: these options are not guaranteed to be stable and may change or be removed at any time.
-     */
-    experiments?: CrawlerExperiments;
     /**
-     * Customize the way statistics collecting works, such as logging interval or
-     * whether to output them to the Key-Value store.
+     * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
+     * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
+     * implementing {@link IStatistics}.
+     *
+     * Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
+     * {@link BasicCrawler.statistics|`crawler.statistics.state`}:
+     *
+     * ```ts
+     * const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
+     *
+     * const crawler = new BasicCrawler({
+     *     statistics,
+     *     requestHandler: async () => {
+     *         statistics.state.productsFound++;
+     *     },
+     * });
+     *
+     * await crawler.run();
+     * // the custom fields are typed on `crawler.statistics` too
+     * console.log(crawler.statistics.state.productsFound);
+     * ```
      */
-    statisticsOptions?: StatisticsOptions;
+    statistics?: IStatistics<StatisticStateExtension>;
     /**
      * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
-     * Defaults to a new instance of {@link GotScrapingHttpClient}
+     * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
      */
     httpClient?: BaseHttpClient;
     /**
      * If set, the crawler will be configured for all connections to use
      * the Proxy URLs provided and rotated according to the configuration.
      */
-    proxyConfiguration?: ProxyConfiguration;
-}
-/**
- * A set of options that you can toggle to enable experimental features in Crawlee.
- *
- * NOTE: These options will not respect semantic versioning and may be removed or changed at any time. Use at your own risk.
- * If you do use these and encounter issues, please report them to us.
- */
-export interface CrawlerExperiments {
+    proxyConfiguration?: IProxyConfiguration;
     /**
-     * @deprecated This experiment is now enabled by default, and this flag will be removed in a future release.
-     * If you encounter issues due to this change, please:
-     * - report it to us: https://github.com/apify/crawlee
-     * - set `requestLocking` to `false` in the `experiments` option of the crawler
+     * Custom configuration to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
      */
-    requestLocking?: boolean;
-}
-/**
- * Provides a simple framework for parallel crawling of web pages.
- * The URLs to crawl are fed either from a static list of URLs
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
- *
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
- * download and data extraction functionality themselves.
- * If we want a crawler that already facilitates this functionality,
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
- *
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
- * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
- * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
- * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
- * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
- *
- * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
- *
- * The crawler finishes if there are no more {@link Request} objects to crawl.
- *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
- * parameter of the `BasicCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
- *
- * **Example usage:**
- *
- * ```javascript
- * import { BasicCrawler, Dataset } from 'crawlee';
- *
- * // Create a crawler instance
- * const crawler = new BasicCrawler({
- *     async requestHandler({ request, sendRequest }) {
- *         // 'request' contains an instance of the Request class
- *         // Here we simply fetch the HTML of the page and store it to a dataset
- *         const { body } = await sendRequest({
- *             url: request.url,
- *             method: request.method,
- *             body: request.payload,
- *             headers: request.headers,
- *         });
- *
- *         await Dataset.pushData({
- *             url: request.url,
- *             html: body,
- *         })
- *     },
- * });
- *
- * // Enqueue the initial requests and run the crawler
- * await crawler.run([
- *     'http://www.example.com/page-1',
- *     'http://www.example.com/page-2',
- * ]);
- * ```
- * @category Crawlers
- */
-export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
-    readonly config: Configuration;
-    protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
+    configuration?: Configuration;
     /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     * Custom storage backend to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
      */
-    readonly stats: Statistics;
+    storageBackend?: StorageBackend;
     /**
-     * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
-     * Only available if used by the crawler.
+     * Custom event manager to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
      */
-    requestList?: IRequestList;
+    eventManager?: EventManager;
     /**
-     * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-     * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
-     * Only available if used by the crawler.
+     * Custom logger to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
      */
-    requestQueue?: RequestProvider;
+    logger?: CrawleeLogger;
     /**
-     * The main request-handling component of the crawler. It's initialized during the crawler startup.
+     * A unique identifier for the crawler instance. This ID is used to isolate the state returned by
+     * {@link BasicCrawler.useState|`crawler.useState()`} from other crawler instances.
+     *
+     * When multiple crawler instances use `useState()` without an explicit `id`, they will share the same
+     * state object for backward compatibility. A warning will be logged in this case.
+     *
+     * To ensure each crawler has its own isolated state that also persists across script restarts
+     * (e.g., during Apify migrations), provide a stable, unique ID for each crawler instance.
+     *
+     */
+    id?: string;
+    /**
+     * Makes the storage writes performed while handling a request atomic with respect to the request
+     * succeeding: they are recorded in a {@link StorageTransaction} spanning the whole request
+     * lifecycle and only applied when the request handler succeeds, so a thrown handler leaves no partial
+     * writes behind and a retry does not double-write. Reads within the handler see its own writes.
+     *
+     * `false` disables the mechanism entirely; an object overrides the per-storage-type
+     * {@link StorageWritePolicy} (e.g. `{ requestQueue: 'deferred' }` for all-or-nothing enqueues).
+     * {@link withDirectStorageAccess} is the per-call-site escape hatch; `useState()` is deliberately
+     * *not* transactional.
+     *
+     * @default true
+     */
+    transactionalStorage?: boolean | Partial<StorageWritePolicy>;
+    /**
+     * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
+     * By default, status codes >= 500 trigger errors.
+     */
+    ignoreHttpErrorStatusCodes?: number[];
+    /**
+     * An array of additional HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be treated as errors.
+     * By default, status codes >= 500 trigger errors.
+     */
+    additionalHttpErrorStatusCodes?: number[];
+}
+export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
+    #private;
+    /**
+     * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+     * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
+     */
+    get statistics(): IStatistics<StatisticStateExtension>;
+    /**
+     * The main request-handling component of the crawler. It manages the requests that the crawler processes,
+     * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
+     * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
     protected requestManager?: IRequestManager;
     /**
-     * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
-     * Only available if used by the crawler.
+     * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
+     * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
-    sessionPool?: SessionPool;
+    get sessionPool(): ISessionPool;
+    /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
+    get running(): boolean;
     /**
-     * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
-     * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
-     * We can use it to change the concurrency settings on the fly,
-     * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
-     * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+     * The concurrency governor the run in progress is booking its requests against — either the
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
+     * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
+     *
+     * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
+     * during a run rather than caching it across runs.
+     *
+     * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
+     * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
+     * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
-    autoscaledPool?: AutoscaledPool;
+    get concurrencySystem(): IConcurrencySystem | undefined;
     /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
+     * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
      */
-    proxyConfiguration?: ProxyConfiguration;
+    readonly proxyConfiguration?: IProxyConfiguration;
     /**
      * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
-    readonly router: RouterHandler<Context>;
-    private contextPipelineBuilder;
-    private _contextPipeline?;
+    readonly router: RouterHandler<Context, Routes>;
+    /**
+     * The basic part of the context pipeline. Unlike the subclass pipeline, this
+     * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
+     * pipelines expect the basic crawler fields to already be present in the context at runtime.
+     *
+     * Context built with this pipeline can be passed into multiple crawler pipelines at once.
+     * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
+     */
+    get basicContextPipeline(): ContextPipeline<{
+        request: CrawlingRequest;
+    }, CrawlingContext>;
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
-    running: boolean;
-    hasFinishedBefore: boolean;
-    readonly log: Log;
-    protected requestHandler: RequestHandler<ExtendedContext>;
-    protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    protected requestHandlerTimeoutMillis: number;
-    protected internalTimeoutMillis: number;
-    protected maxRequestRetries: number;
-    protected maxCrawlDepth?: number;
-    protected sameDomainDelayMillis: number;
-    protected domainAccessedTime: Map<string, number>;
-    protected maxSessionRotations: number;
-    protected maxRequestsPerCrawl?: number;
-    protected handledRequestsCount: number;
-    protected statusMessageLoggingInterval: number;
-    protected statusMessageCallback?: StatusMessageCallback;
-    protected sessionPoolOptions: SessionPoolOptions;
-    protected useSessionPool: boolean;
-    protected autoscaledPoolOptions: AutoscaledPoolOptions;
-    protected events: EventManager;
-    protected httpClient: BaseHttpClient;
-    protected retryOnBlocked: boolean;
-    protected respectRobotsTxtFile: boolean | {
-        userAgent?: string;
-    };
-    protected onSkippedRequest?: SkippedRequestCallback;
-    private _closeEvents?;
-    private shouldLogMaxProcessedRequestsExceeded;
-    private shouldLogMaxEnqueuedRequestsExceeded;
-    private experiments;
-    private readonly robotsTxtFileCache;
-    private _experimentWarnings;
+    /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
+    get hasFinishedBefore(): boolean;
+    get log(): CrawleeLogger;
+    protected readonly requestHandler: RequestHandler<ExtendedContext>;
+    protected readonly internalTimeoutMillis: number;
+    private get handledRequestsCount();
+    protected blockedStatusCodes: Set<number>;
+    protected readonly additionalHttpErrorStatusCodes: Set<number>;
+    protected readonly httpClient: BaseHttpClient;
+    protected readonly retryOnBlocked: boolean;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;
+        configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
     };
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>, // cast because the constructor logic handles missing `contextPipelineBuilder` - the type is just for DX
-    config?: Configuration);
+    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
+    /**
+     * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
+     *
+     * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
+     * concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
+     */
+    protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
+    /**
+     * Determines if the given HTTP status code is an error status code given
+     * the default behaviour and user-set preferences.
+     * @param status
+     * @returns `true` if the status code is considered an error, `false` otherwise
+     */
+    protected isErrorStatusCode(status: number): boolean;
+    /**
+     * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
+     * This handles base context creation, session resolution, and context helpers.
+     */
+    private buildBasicContextPipeline;
+    private checkRobotsTxt;
+    /**
+     * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
+     * Subclasses should override this to add their own pipeline stages.
+     */
+    protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
+    private createBaseContext;
+    /**
+     * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
+     * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
+     * `markRequestAsHandled`, so nothing observes the difference.
+     */
+    private resolveRequest;
+    private resolveSession;
+    private createContextHelpers;
+    private buildFinalContextPipeline;
     /**
      * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
      * Used for retrying requests that failed due to proxy errors.
@@ -493,21 +546,32 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     protected isProxyError(error: Error): boolean;
     /**
+     * Sets the status message for the current crawler run.
+     *
      * This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
+     *
+     * The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
+     * event. Integrations such as the Apify SDK subscribe to that event and forward the message to
+     * their status-reporting backend (e.g. the Apify platform).
      */
-    setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
+    setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
     private getPeriodicLogger;
     /**
-     * Runs the crawler. Returns a promise that resolves once all the requests are processed
-     * and `autoscaledPool.isFinished` returns `true`.
+     * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
+     * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
+     * default "the request manager is empty") reports that the crawl is over.
      *
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
      *
+     * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
+     * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
+     * if that is what you want.
+     *
      * @param [requests] The requests to add.
-     * @param [options] Options for the request queue.
+     * @param [options] Options for adding the initial requests.
      */
-    run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;
+    run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
     /**
      * Gracefully stops the current run of the crawler.
      *
@@ -515,32 +579,78 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      *
      * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
-    stop(message?: string): void;
-    getRequestQueue(): Promise<RequestProvider>;
+    stop(reason?: string): void;
+    /**
+     * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
+     * after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
+     * run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
+     *
+     * > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
+     * throughout, since a shared one may still be serving other crawlers.
+     */
+    pause(timeoutSecs?: number): Promise<void>;
+    /**
+     * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
+     * again. A no-op on a crawler that is not paused.
+     */
+    resume(): void;
+    /**
+     * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
+     * if none has been configured or opened yet.
+     */
+    getRequestManager(): Promise<IRequestManager>;
+    /**
+     * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
+     * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
+     */
+    getRequestQueue(): Promise<IRequestManager>;
+    /**
+     * Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
+     * @private
+     */
+    private openOwnedRequestQueue;
+    /**
+     * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
+     * locking storage backend keeps it reserved for slightly longer than the request handler timeout
+     * (with some padding for overhead), but never for less than a minute. This prevents a long-running
+     * request from being handed out a second time while it is still being processed — and it works
+     * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
+     */
+    private applyRequestManagerTimeouts;
+    /**
+     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
+     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
+     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
+     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
+     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
+     */
+    private validateRequestUserData;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
-    protected get pendingRequestCountApproximation(): number;
-    protected calculateEnqueuedRequestLimit(explicitLimit?: number): number | undefined;
-    protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
      * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
      * the `waitForAllRequestsToBeAdded` promise you get in the response object.
      *
+     * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
+     * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
+     * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
+     * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
+     *
      * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
      *
      * @param requests The requests to add
      * @param options Options for the request queue
      */
-    addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
+    addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
     /**
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
      */
-    pushData(data: Parameters<Dataset['pushData']>[0], datasetIdOrName?: string): Promise<void>;
+    pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
     /**
      * Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
      */
-    getDataset(idOrName?: string): Promise<Dataset>;
+    getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
     /**
      * Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
      */
@@ -553,42 +663,63 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * Initializes the crawler.
      */
-    protected _init(): Promise<void>;
-    protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
+    protected init(): Promise<void>;
     /**
-     * Handles blocked request
+     * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
+     * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
+     * browser crawlers override it with their `navigationTimeoutSecs`.
      */
-    protected _throwOnBlockedRequest(session: Session, statusCode: number): void;
-    private isAllowedBasedOnRobotsTxtFile;
-    protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
-    protected _pauseOnMigration(): Promise<void>;
+    protected getNavigationTimeoutMillis(): number;
     /**
-     * Initializes the RequestManager based on the configured requestList and requestQueue.
+     * Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
+     * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
+     * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
+     * whichever is larger: the configured internal timeout, or this request's combined phase budget.
      */
-    private initializeRequestManager;
+    private withRequestTimeout;
     /**
-     * Fetches the next request to process from the underlying request provider.
+     * The request handler timeout for a request with the given route label. A router route may override the
+     * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
+     *
+     * @param label The request's route label, or `undefined` for the default route / no specific request.
+     * @param fallbackMillis Timeout to use when no route overrides it.
      */
-    protected _fetchNextRequest(): Promise<Request<Dictionary> | null>;
+    private resolveRequestHandlerTimeoutMillis;
     /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
+     * The timeout the router route with the given label asked for, or `undefined` when it did not override one
+     * (or the request handler is not a router at all).
      */
-    protected delayRequest(request: Request, source: IRequestList | RequestProvider | IRequestManager): boolean;
+    private getRouteTimeoutMillis;
+    protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     /**
-     * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
-     * then retries them in a case of an error, etc.
+     * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
+     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
+     * a normal return says nothing about success. `handleRequest` owns the outcome.
      */
-    protected _runTaskFunction(): Promise<void>;
+    private runInStorageTransaction;
     /**
-     * Wrapper around the crawling context's `enqueueLinks` method:
-     * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
-     * @internal
+     * Handles blocked request
+     */
+    protected throwOnBlockedRequest(statusCode: number): void;
+    private isAllowedBasedOnRobotsTxtFile;
+    /**
+     * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
+     *
+     * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
+     * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
+     *  {@link RequestThrottledError} rather than treating the response as a blocked session.
+     */
+    protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
+    /** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
+    private applyCrawlDelay;
+    protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
+    private pauseOnMigration;
+    /**
+     * Fetches the next request to process from the underlying request provider.
      */
-    protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestQueue: RequestProvider): Promise<BatchAddRequestsResult>;
+    private fetchNextRequest;
+    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    private handleRequest;
     /**
      * Generator function that yields requests injected with the given crawl depth.
      * @internal
@@ -598,67 +729,66 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * Run async callback with given timeout and retry. Returns the result of the callback.
      * @ignore
      */
-    protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
+    private timeoutAndRetry;
     /**
-     * Returns true if either RequestList or RequestQueue have a request ready for processing.
+     * Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
+     * wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
      */
-    protected _isTaskReadyFunction(): Promise<boolean>;
-    /**
-     * Returns true if both RequestList and RequestQueue have all requests finished.
-     */
-    protected _defaultIsFinishedFunction(): Promise<boolean>;
-    private _rotateSession;
+    private isTaskReadyFunction;
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
+     * RequestHandlerError and ContextPipelineCleanupError wrap the actual error.
      */
     private unwrapError;
     /**
      * Handles errors thrown by user provided requestHandler()
+     *
+     * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
-    protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, source: IRequestList | IRequestManager): Promise<void>;
-    protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
-    protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
+    private requestFunctionErrorHandler;
+    private handleFailedRequestHandler;
     /**
      * Resolves the most verbose error message from a thrown error
      * @param error The error received
      * @returns The message to be logged
      */
-    protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
-    protected _canRequestBeRetried(request: Request, error: Error): boolean;
+    protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
     /**
-     * Updates handledRequestsCount from possibly stored counts, usually after worker migration.
+     * Whether the session should be spared for this error - either because it was already retired, or because the
+     * failure says nothing about the session (a rate limit is a property of the domain).
      */
-    protected _loadHandledRequestCount(): Promise<void>;
-    protected _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(hooks: HookLike[], ...args: Parameters<HookLike>): Promise<void>;
+    private errorAbsolvesSession;
+    private canRequestBeRetried;
     /**
-     * Stops the crawler immediately.
+     * Ends the run in progress immediately, without waiting for the requests already in flight.
      *
-     * This method doesn't wait for currently active requests to finish.
+     * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
+     * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
+     * released by {@link BasicCrawler.destroy|`destroy()`} instead.
      *
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     teardown(): Promise<void>;
-    protected _getCookieHeaderFromRequest(request: Request): string;
-    private _getRequestQueue;
+    /**
+     * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
+     * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
+     * could otherwise have run again.
+     */
+    destroy(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+    protected getCookieHeaderFromRequest(request: Request): string;
     private requestMatchesEnqueueStrategy;
 }
 export interface CreateContextOptions {
-    request: Request;
-    session?: Session;
+    request: CrawlingRequest;
+    session: ISession;
     proxyInfo?: ProxyInfo;
 }
-export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
+export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
 }
 export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
 }
 export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
-    /**
-     * Whether to purge the RequestQueue before running the crawler again. Defaults to true, so it is possible to reprocess failed requests.
-     * When disabled, only new requests will be considered. Note that even a failed request is considered as handled.
-     * @default true
-     */
-    purgeRequestQueue?: boolean;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.
@@ -684,5 +814,5 @@ export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
  * await crawler.run();
  * ```
  */
-export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
-//# sourceMappingURL=basic-crawler.d.ts.map
+export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;