@crawlee/basic 4.0.0-beta.12 → 4.0.0-beta.121

package/internals/basic-crawler.js

@@ -1,16 +1,31 @@
-import { writeFile } from 'node:fs/promises';
+import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AutoscaledPool, Configuration, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, GotScrapingHttpClient, KeyValueStore, mergeCookies, NonRetryableError, purgeDefaultStorages, RequestHandlerError, RequestProvider, RequestQueue, RequestQueueV1, RequestState, RetryRequestError, Router, SessionError, SessionPool, Statistics, validators, } from '@crawlee/core';
-import { RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
-import { stringify } from 'csv-stringify/sync';
-import { ensureDir, writeJSON } from 'fs-extra/esm';
-import ow from 'ow';
+import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createStorageTransaction, CriticalError, currentStorageTransaction, Dataset, enqueueLinks, EnqueueStrategy, EventType, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { FetchHttpClient } from '@crawlee/http-client';
+import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
+import { RobotsTxtFile } from '@crawlee/utils';
+import ow, { ArgumentError } from 'ow';
 import { getDomain } from 'tldts';
 import { LruCache } from '@apify/datastructures';
-import defaultLog, { LogLevel } from '@apify/log';
-import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
+import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
+import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
+class LazyDefaultHttpClient {
+    #delegatePromise;
+    constructor(options) {
+        this.#delegatePromise = import('@crawlee/impit-client')
+            .then(({ ImpitHttpClient }) => new ImpitHttpClient(options))
+            .catch(() => {
+            (options?.logger ?? log).warning('Optional dependency @crawlee/impit-client is not installed. ' +
+                'Falling back to native fetch — proxy support and browser fingerprinting are unavailable.');
+            return new FetchHttpClient(options);
+        });
+    }
+    async sendRequest(...args) {
+        return (await this.#delegatePromise).sendRequest(...args);
+    }
+}
 /**
  * Since there's no set number of seconds before the container is terminated after
  * a migration event, we need some reasonable number to use for RequestList persistence.
@@ -21,103 +36,92 @@ import { createSendRequest } from './send-request.js';
  * @ignore
  */
 const SAFE_MIGRATION_WAIT_MILLIS = 20000;
-/**
- * Provides a simple framework for parallel crawling of web pages.
- * The URLs to crawl are fed either from a static list of URLs
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
- *
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
- * download and data extraction functionality themselves.
- * If we want a crawler that already facilitates this functionality,
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
- *
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
- * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link RequestList} or {@link RequestQueue}
- * instances provided by the {@link BasicCrawlerOptions.requestList|`requestList`} or {@link BasicCrawlerOptions.requestQueue|`requestQueue`}
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
- * the crawler will open the default request queue either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called,
- * or if `requests` parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
- *
- * If both {@link BasicCrawlerOptions.requestList|`requestList`} and {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
- *
- * The crawler finishes if there are no more {@link Request} objects to crawl.
- *
- * New requests are only dispatched when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
- * parameter of the `BasicCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BasicCrawler` constructor.
- *
- * **Example usage:**
- *
- * ```javascript
- * import { BasicCrawler, Dataset } from 'crawlee';
- *
- * // Create a crawler instance
- * const crawler = new BasicCrawler({
- *     async requestHandler({ request, sendRequest }) {
- *         // 'request' contains an instance of the Request class
- *         // Here we simply fetch the HTML of the page and store it to a dataset
- *         const { body } = await sendRequest({
- *             url: request.url,
- *             method: request.method,
- *             body: request.payload,
- *             headers: request.headers,
- *         });
- *
- *         await Dataset.pushData({
- *             url: request.url,
- *             html: body,
- *         })
- *     },
- * });
- *
- * // Enqueue the initial requests and run the crawler
- * await crawler.run([
- *     'http://www.example.com/page-1',
- *     'http://www.example.com/page-2',
- * ]);
- * ```
- * @category Crawlers
- */
+const deferredCleanupKey = Symbol('deferredCleanup');
+// The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
+export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
 export class BasicCrawler {
-    config;
     static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
     /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     * Tracks the number of crawler instances created. The first crawler uses the default
+     * request queue; subsequent ones get their own queue via a unique alias so they don't
+     * collide.
      */
-    stats;
+    // kept as TS-private: tests reset the counter at runtime
+    static instanceCount = 0;
     /**
-     * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
-     * Only available if used by the crawler.
+     * Tracks crawler instances that accessed shared state without having an explicit id.
+     * Used to detect and warn about multiple crawlers sharing the same state.
      */
-    requestList;
+    static #useStateAnonymousIndices = new Set();
+    /** Backs the {@link BasicCrawler.stats|`stats`} getter. */
+    #statsDep;
     /**
-     * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-     * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
-     * Only available if used by the crawler.
+     * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+     * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    requestQueue;
+    get stats() {
+        return this.#statsDep.value;
+    }
     /**
-     * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
-     * Only available if used by the crawler.
+     * The main request-handling component of the crawler. It manages the requests that the crawler processes,
+     * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
+     * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
+     */
+    requestManager;
+    /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
+    #sessionPoolDep;
+    /**
+     * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
+     * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
+     */
+    get sessionPool() {
+        return this.#sessionPoolDep.value;
+    }
+    /**
+     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
+     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
+     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
+     */
+    #ownedRequestQueue = OwnedOrInjected.resolve();
+    /**
+     * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
+     * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
+     * so it only needs to be applied once, at the first async access of the manager.
+     */
+    #requestManagerTimeoutsApplied = false;
+    /**
+     * Resolves the governor for one run: either the injected
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
+     * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
+     */
+    #resolveConcurrencySystem;
+    /** As resolved by `init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
+    #concurrencySystemDep;
+    /**
+     * The concurrency governor this run is booking its requests against — either the
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
+     * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
+     *
+     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
+     * is also rebuilt for every run, so the instance is not stable across runs.
+     *
+     * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
+     * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
+     * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
-    sessionPool;
+    get concurrencySystem() {
+        return this.#concurrencySystemDep?.maybeValue;
+    }
     /**
-     * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
-     * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
-     * We can use it to change the concurrency settings on the fly,
-     * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
-     * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
+     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
+     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
+     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
+     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
      */
-    autoscaledPool;
+    #autoscaledPool;
     /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
+     * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
      */
     proxyConfiguration;
@@ -126,46 +130,81 @@ export class BasicCrawler {
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     router = Router.create();
-    contextPipelineBuilder;
-    _contextPipeline;
+    #basicContextPipeline;
+    /**
+     * The basic part of the context pipeline. Unlike the subclass pipeline, this
+     * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
+     * pipelines expect the basic crawler fields to already be present in the context at runtime.
+     *
+     * Context built with this pipeline can be passed into multiple crawler pipelines at once.
+     * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
+     */
+    get basicContextPipeline() {
+        if (this.#basicContextPipeline === undefined) {
+            this.#basicContextPipeline = this.buildBasicContextPipeline();
+        }
+        return this.#basicContextPipeline;
+    }
+    #contextPipeline;
     get contextPipeline() {
-        if (this._contextPipeline === undefined) {
-            this._contextPipeline = this.contextPipelineBuilder();
+        if (this.#contextPipeline === undefined) {
+            this.#contextPipeline = this.buildFinalContextPipeline();
         }
-        return this._contextPipeline;
+        return this.#contextPipeline;
     }
     running = false;
     hasFinishedBefore = false;
-    log;
+    #unexpectedStop = false;
+    #log;
+    get log() {
+        return this.#log;
+    }
     requestHandler;
     errorHandler;
     failedRequestHandler;
+    // kept as TS-private: tests read it at runtime
     requestHandlerTimeoutMillis;
     internalTimeoutMillis;
     maxRequestRetries;
-    sameDomainDelayMillis;
-    domainAccessedTime;
-    maxSessionRotations;
-    handledRequestsCount;
-    statusMessageLoggingInterval;
-    statusMessageCallback;
-    sessionPoolOptions;
-    useSessionPool;
-    autoscaledPoolOptions;
-    events;
+    maxCrawlDepth;
+    #sameDomainDelayMillis;
+    #domainAccessedTime;
+    maxRequestsPerCrawl;
+    get handledRequestsCount() {
+        return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
+    }
+    #statusMessageLoggingInterval;
+    #statusMessageCallback;
+    blockedStatusCodes = new Set();
+    additionalHttpErrorStatusCodes;
+    #ignoreHttpErrorStatusCodes;
+    /**
+     * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
+     * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
+     * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
+     * settable here.
+     */
+    // kept as TS-private: tests mutate it at runtime
+    taskLoopOptions;
     httpClient;
     retryOnBlocked;
-    respectRobotsTxtFile;
+    #respectRobotsTxtFile;
+    /** Whether `runInStorageTransaction()` opens a transaction at all. */
+    #transactionalStorageEnabled;
+    /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
+    #storageWritePolicy;
     onSkippedRequest;
-    _closeEvents;
-    experiments;
-    robotsTxtFileCache;
-    _experimentWarnings = {};
+    #closeEvents;
+    #loggedPerRun = new Set();
+    #robotsTxtFileCache;
+    identity;
+    #contextPipelineOptions;
     static optionsShape = {
         contextPipelineBuilder: ow.optional.object,
         extendContext: ow.optional.function,
         requestList: ow.optional.object.validate(validators.requestList),
         requestQueue: ow.optional.object.validate(validators.requestQueue),
+        requestManager: ow.optional.object,
         // Subclasses override this function instead of passing it
         // in constructor, so this validation needs to apply only
         // if the user creates an instance of BasicCrawler directly.
@@ -175,173 +214,432 @@ export class BasicCrawler {
         failedRequestHandler: ow.optional.function,
         maxRequestRetries: ow.optional.number,
         sameDomainDelaySecs: ow.optional.number,
-        maxSessionRotations: ow.optional.number,
         maxRequestsPerCrawl: ow.optional.number,
-        autoscaledPoolOptions: ow.optional.object,
-        sessionPoolOptions: ow.optional.object,
-        useSessionPool: ow.optional.boolean,
+        maxCrawlDepth: ow.optional.number,
+        taskLoopOptions: ow.optional.object,
+        concurrencySystem: ow.optional.object,
+        sessionPool: ow.optional.object.validate(validators.sessionPool),
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
         statusMessageLoggingInterval: ow.optional.number,
         statusMessageCallback: ow.optional.function,
+        additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
+        ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
+        blockedStatusCodes: ow.optional.array.ofType(ow.number),
         retryOnBlocked: ow.optional.boolean,
-        respectRobotsTxtFile: ow.optional.boolean,
+        respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object),
+        transactionalStorage: ow.optional.any(ow.boolean, ow.object.exactShape({
+            requestQueue: ow.optional.string.oneOf(['deferred', 'writeThrough']),
+        })),
         onSkippedRequest: ow.optional.function,
         httpClient: ow.optional.object,
+        configuration: ow.optional.object,
+        storageBackend: ow.optional.object,
+        eventManager: ow.optional.object,
+        logger: ow.optional.object,
         // AutoscaledPool shorthands
         minConcurrency: ow.optional.number,
         maxConcurrency: ow.optional.number,
         maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
         keepAlive: ow.optional.boolean,
-        // internal
-        log: ow.optional.object,
-        experiments: ow.optional.object,
-        statisticsOptions: ow.optional.object,
+        statistics: ow.optional.object,
+        id: ow.optional.string,
     };
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, // cast because the constructor logic handles missing `contextPipelineBuilder` - the type is just for DX
-    config = Configuration.getGlobalConfig()) {
-        this.config = config;
+    constructor(options = {}) {
         ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
-        const { requestList, requestQueue, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxSessionRotations = 10, maxRequestsPerCrawl, autoscaledPoolOptions = {}, keepAlive, sessionPoolOptions = {}, useSessionPool = true, proxyConfiguration, 
+        const { 
+        // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
+        requestList, 
+        // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
+        requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
+        // Service locator options
+        configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statisticsOptions, httpClient, 
-        // internal
-        log = defaultLog.child({ prefix: this.constructor.name }), experiments = {}, } = options;
-        // Store the builder so that it can be run when the contextPipeline is needed.
-        // Invoking it immediately would cause problems with parent constructor call order.
-        this.contextPipelineBuilder = () => {
-            let contextPipeline = (options.contextPipelineBuilder?.() ??
-                ContextPipeline.create()); // Thanks to the RequireContextPipeline, contextPipeline will only be undefined if InitialContextType is CrawlingContext
-            if (options.extendContext !== undefined) {
-                contextPipeline = contextPipeline.compose({
-                    action: async (context) => await options.extendContext(context),
-                });
+        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statistics, httpClient, id, } = options;
+        // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
+        // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
+        // hammering a site.
+        if (concurrencySystem !== undefined &&
+            (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
+            throw new ArgumentError('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
+                '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
+                'replaces. Pass them to the `ConcurrencySystem` constructor instead.', this.constructor);
+        }
+        // Create per-crawler service locator if custom services were provided.
+        // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
+        // (via AsyncLocalStorage) resolve to this scoped instance instead.
+        // We also enter the scope for the rest of the constructor body, so that any code below
+        // that accesses `serviceLocator` will see the correct (scoped) instance.
+        let serviceLocatorScope = { enterScope: () => { }, exitScope: () => { } };
+        if (storageBackend ||
+            eventManager ||
+            logger ||
+            (configuration !== undefined && configuration !== serviceLocator.getConfiguration())) {
+            // Inherit the ambient locator's already-set services for anything not explicitly
+            // provided - e.g. only passing a `logger` must not detach the crawler from a globally
+            // configured storage backend.
+            const ambientServices = serviceLocator.getServicesIfSet();
+            const scopedServiceLocator = new ServiceLocator(configuration ?? ambientServices.configuration, eventManager ?? ambientServices.eventManager, storageBackend ?? ambientServices.storageBackend, logger ?? ambientServices.logger);
+            serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);
+        }
+        try {
+            serviceLocatorScope.enterScope();
+            this.#contextPipelineOptions = {
+                contextPipelineBuilder: options.contextPipelineBuilder,
+                extendContext: options.extendContext,
+            };
+            this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
+            // Initialize the Configuration instance to avoid lazy loading in the components
+            serviceLocator.getConfiguration();
+            const instanceIndex = BasicCrawler.instanceCount++;
+            this.identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
+            if (requestManager !== undefined) {
+                if (requestList !== undefined || requestQueue !== undefined) {
+                    throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
+                }
+                this.requestManager = requestManager;
+            }
+            else if (requestList !== undefined && requestQueue !== undefined) {
+                // Combine the read-only list with the writable queue into a tandem.
+                this.requestManager = new RequestManagerTandem(requestList, requestQueue);
+            }
+            else if (requestQueue !== undefined) {
+                // A RequestQueue is itself a request manager.
+                this.requestManager = requestQueue;
             }
-            contextPipeline = contextPipeline.compose({
-                action: async (context) => {
-                    const { request } = context;
-                    if (!this.requestMatchesEnqueueStrategy(request)) {
-                        // eslint-disable-next-line dot-notation
-                        const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
-                        this.log.debug(message);
-                        request.noRetry = true;
-                        request.state = RequestState.SKIPPED;
-                        throw new ContextPipelineInterruptedError(message);
+            else if (requestList !== undefined) {
+                // A lone read-only `requestList` (deprecated option) is combined with a lazily-opened default queue
+                // into a tandem, so that its requests are read first and new ones can still be enqueued during the
+                // crawl. The queue is opened on first use; the tandem also forwards `persistState()` to the loader.
+                this.requestManager = new RequestManagerTandem(requestList, () => this.openOwnedRequestQueue());
+            }
+            this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
+            this.proxyConfiguration = proxyConfiguration;
+            this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
+            this.#statusMessageCallback = statusMessageCallback;
+            this.#domainAccessedTime = new Map();
+            this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
+            this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
+            this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
+            this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
+            this.requestHandler = requestHandler ?? this.router;
+            this.failedRequestHandler = failedRequestHandler;
+            this.errorHandler = errorHandler;
+            if (requestHandlerTimeoutSecs) {
+                this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+            }
+            else {
+                this.requestHandlerTimeoutMillis = 60_000;
+            }
+            this.retryOnBlocked = retryOnBlocked;
+            this.#respectRobotsTxtFile = respectRobotsTxtFile;
+            // The cast undoes ow's assertion signature, which mangles `boolean | object` unions.
+            const transactionalStorageOption = transactionalStorage;
+            this.#transactionalStorageEnabled = transactionalStorageOption !== false;
+            this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
+            this.onSkippedRequest = onSkippedRequest;
+            // allow at least 5min for internal timeouts
+            this.internalTimeoutMillis =
+                serviceLocator.getConfiguration().internalTimeoutMillis ??
+                    Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
+            this.maxRequestRetries = maxRequestRetries;
+            this.maxCrawlDepth = maxCrawlDepth;
+            this.#sameDomainDelayMillis = sameDomainDelaySecs * 1000;
+            this.#statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
+                logMessage: `${this.constructor.name} request statistics:`,
+                log: this.log,
+                id: this.identity.id,
+            }));
+            if (sessionPool && proxyConfiguration) {
+                this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
+                    'The `proxyConfiguration` is ignored - sessions from the supplied pool keep whatever ' +
+                    '`proxyInfo` they were created with. Configure proxies on the pool instead, ' +
+                    'e.g. via `addSession({ proxyInfo })` or a custom `createSessionFunction`.');
+            }
+            this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
+                createSessionFunction: async (opts) => new Session({
+                    ...opts?.sessionOptions,
+                    proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
+                }),
+            }));
+            this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
+            const maxSignedInteger = 2 ** 31 - 1;
+            if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
+                this.log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
+                    ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
+                this.requestHandlerTimeoutMillis = maxSignedInteger;
+            }
+            this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
+            this.maxRequestsPerCrawl = maxRequestsPerCrawl;
+            const isMaxPagesExceeded = () => this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;
+            // eslint-disable-next-line prefer-const
+            let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
+            // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
+            if (keepAlive) {
+                isFinishedFunction = async () => false;
+            }
+            const crawlerOwnedTaskLoopConfiguration = {
+                runTaskFunction: async () => {
+                    const source = this.requestManager;
+                    if (!source)
+                        throw new Error('Request provider is not initialized!');
+                    const request = await this.resolveRequest();
+                    if (!request || this.delayRequest(request, source)) {
+                        return;
+                    }
+                    // Started here, rather than in `handleRequest`, so that a failure during context pipeline
+                    // initialization (e.g. a browser page timing out before the request handler ever runs) is
+                    // still accounted for by `failJob` below - which is a no-op without a matching `startJob`.
+                    this.stats.startJob(request.id || request.uniqueKey);
+                    const crawlingContext = { request };
+                    try {
+                        // The transaction spans the whole pipeline call, covering the navigation hooks
+                        // and `extendContext` too; `handleRequest` drives its outcome explicitly.
+                        await this.runInStorageTransaction(async () => 
+                        // Navigation, the navigation hooks and the request handler are timed individually, but the
+                        // phases between them are not, so a request could still get stuck indefinitely. This is the
+                        // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
+                        await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
+                            .chain(this.contextPipeline)
+                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request))));
+                    }
+                    catch (error) {
+                        // ContextPipelineInterruptedError means the request was intentionally skipped
+                        // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
+                        if (error instanceof ContextPipelineInterruptedError) {
+                            this.stats.discardJob(request.id || request.uniqueKey);
+                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                            return;
+                        }
+                        // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
+                        // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`
+                        // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.
+                        const isPipelineError = error instanceof ContextPipelineInitializationError ||
+                            error instanceof SessionError ||
+                            error instanceof TimeoutError;
+                        if (isPipelineError) {
+                            const unwrappedError = this.unwrapError(error);
+                            await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
+                            // SessionError already retired the session in `requestFunctionErrorHandler`;
+                            // skip `markBad` to avoid double-counting usage/error score.
+                            if (!this.errorAbsolvesSession(unwrappedError)) {
+                                crawlingContext.session?.markBad();
+                            }
+                            return;
+                        }
+                        throw this.unwrapError(error);
+                    }
+                    finally {
+                        // Run request-scoped deferred cleanups only after the whole request lifecycle - including the user's error handler - has finished.
+                        const deferredCleanup = crawlingContext[deferredCleanupKey] ?? [];
+                        await Promise.all(deferredCleanup.map((fn) => fn().catch((cleanupError) => this.log.debug('Error in deferred cleanup', { error: cleanupError }))));
                     }
-                    return context;
                 },
+                isTaskReadyFunction: async () => {
+                    if (isMaxPagesExceeded()) {
+                        this.logOncePerRun('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
+                            `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
+                        return false;
+                    }
+                    if (this.#unexpectedStop) {
+                        this.logOncePerRun('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
+                            'Ongoing requests will be allowed to complete.');
+                        return false;
+                    }
+                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this.isTaskReadyFunction();
+                },
+                isFinishedFunction: async () => {
+                    if (isMaxPagesExceeded()) {
+                        this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` +
+                            'and all requests that were in progress at that time have now finished. ' +
+                            `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
+                        return true;
+                    }
+                    if (this.#unexpectedStop) {
+                        this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
+                        return true;
+                    }
+                    // Checked here because this runs only once nothing is in flight, which is exactly when a
+                    // crawl that cannot progress looks indistinguishable from one that is merely waiting.
+                    if (!keepAlive && supportsDomainThrottling(this.requestManager)) {
+                        await this.requestManager.assertNoStalledDomains();
+                    }
+                    const isFinished = isFinishedFunction
+                        ? await isFinishedFunction()
+                        : await this.defaultIsFinishedFunction();
+                    if (isFinished) {
+                        const reason = isFinishedFunction
+                            ? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
+                            : 'All requests from the queue have been processed, the crawler will shut down.';
+                        this.log.info(reason);
+                    }
+                    return isFinished;
+                },
+                log: this.log,
+            };
+            this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
+            this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
+                minConcurrency,
+                maxConcurrency,
+                maxTasksPerMinute: maxRequestsPerMinute,
+                log: this.log,
+            }));
+        }
+        finally {
+            serviceLocatorScope.exitScope();
+        }
+    }
+    /**
+     * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
+     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
+     *
+     * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
+     * concurrency and relaxes the event loop signal) while still honouring the user's shortcuts.
+     */
+    createDefaultConcurrencySystem(options) {
+        return new ConcurrencySystem(options);
+    }
+    /**
+     * Determines if the given HTTP status code is an error status code given
+     * the default behaviour and user-set preferences.
+     * @param status
+     * @returns `true` if the status code is considered an error, `false` otherwise
+     */
+    isErrorStatusCode(status) {
+        const excludeError = this.#ignoreHttpErrorStatusCodes.has(status);
+        const includeError = this.additionalHttpErrorStatusCodes.has(status);
+        return (status >= 500 && !excludeError) || includeError;
+    }
+    /**
+     * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
+     * This handles base context creation, session resolution, and context helpers.
+     */
+    buildBasicContextPipeline() {
+        return ContextPipeline.create()
+            .compose({ action: this.checkRobotsTxt.bind(this) })
+            .compose({ action: (context) => this.createBaseContext(context) })
+            .compose({ action: this.resolveSession.bind(this) })
+            .compose({ action: this.createContextHelpers.bind(this) });
+    }
+    async checkRobotsTxt({ request }) {
+        if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
+            this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
+            request.state = RequestState.SKIPPED;
+            request.noRetry = true;
+            await this.handleSkippedRequest({
+                url: request.url,
+                reason: 'robotsTxt',
             });
-            return contextPipeline;
-        };
-        this.requestList = requestList;
-        this.requestQueue = requestQueue;
-        this.httpClient = httpClient ?? new GotScrapingHttpClient();
-        this.proxyConfiguration = proxyConfiguration;
-        this.log = log;
-        this.statusMessageLoggingInterval = statusMessageLoggingInterval;
-        this.statusMessageCallback = statusMessageCallback;
-        this.events = config.getEventManager();
-        this.domainAccessedTime = new Map();
-        this.experiments = experiments;
-        this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
-        this.requestHandler = requestHandler ?? this.router;
-        this.failedRequestHandler = failedRequestHandler;
-        this.errorHandler = errorHandler;
-        if (requestHandlerTimeoutSecs) {
-            this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+            throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);
         }
-        else {
-            this.requestHandlerTimeoutMillis = 60_000;
-        }
-        this.retryOnBlocked = retryOnBlocked;
-        this.respectRobotsTxtFile = respectRobotsTxtFile;
-        this.onSkippedRequest = onSkippedRequest;
-        const tryEnv = (val) => (val == null ? null : +val);
-        // allow at least 5min for internal timeouts
-        this.internalTimeoutMillis =
-            tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
-        // override the default internal timeout of request queue to respect `requestHandlerTimeoutMillis`
-        if (this.requestQueue) {
-            this.requestQueue.internalTimeoutMillis = this.internalTimeoutMillis;
-            // for request queue v2, we want to lock requests for slightly longer than the request handler timeout so that there is some padding for locking-related overhead,
-            // but never for less than a minute
-            this.requestQueue.requestLockSecs = Math.max(this.requestHandlerTimeoutMillis / 1000 + 5, 60);
-        }
-        this.maxRequestRetries = maxRequestRetries;
-        this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
-        this.maxSessionRotations = maxSessionRotations;
-        this.handledRequestsCount = 0;
-        this.stats = new Statistics({
-            logMessage: `${log.getOptions().prefix} request statistics:`,
-            log,
-            config,
-            ...statisticsOptions,
-        });
-        this.sessionPoolOptions = {
-            ...sessionPoolOptions,
-            log,
+        return {};
+    }
+    /**
+     * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
+     * Subclasses should override this to add their own pipeline stages.
+     */
+    buildContextPipeline() {
+        return ContextPipeline.create();
+    }
+    createBaseContext(context) {
+        const deferredCleanup = [];
+        return {
+            id: cryptoRandomObjectId(10),
+            log: this.log,
+            pushData: this.pushData.bind(this),
+            useState: this.useState.bind(this),
+            getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
+            registerDeferredCleanup: (cleanup) => {
+                deferredCleanup.push(cleanup);
+            },
+            extendTimeout: (secs) => {
+                const extraMillis = secs * 1000;
+                // the current `addTimeoutToPromise` window (the request handler, or a navigation hook)...
+                extendTimeout(extraMillis);
+                // ...the internal timeout around the whole request, which is not an `addTimeoutToPromise` frame...
+                context[extendTimeoutKey]?.(extraMillis);
+                // ...and, when called from within the navigation phase, its shared window, so extending a hook
+                // extends the whole navigation budget rather than just that hook's step.
+                if (context[navigationDeadlineKey] !== undefined) {
+                    context[navigationDeadlineKey] += extraMillis;
+                }
+            },
+            [deferredCleanupKey]: deferredCleanup,
         };
-        if (this.retryOnBlocked) {
-            this.sessionPoolOptions.blockedStatusCodes = sessionPoolOptions.blockedStatusCodes ?? [];
-            if (this.sessionPoolOptions.blockedStatusCodes.length !== 0) {
-                log.warning(`Both 'blockedStatusCodes' and 'retryOnBlocked' are set. Please note that the 'retryOnBlocked' feature might not work as expected.`);
+    }
+    async resolveRequest() {
+        const request = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        // Reset loadedUrl so an old one is not carried over to retries.
+        if (request) {
+            request.loadedUrl = undefined;
+        }
+        return request;
+    }
+    async resolveSession({ request }) {
+        const session = await this.timeoutAndRetry(async () => {
+            const existingSession = await this.sessionPool.getSession(request.sessionId);
+            if (!existingSession) {
+                throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
             }
+            return existingSession;
+        }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        return { session, proxyInfo: session?.proxyInfo };
+    }
+    async createContextHelpers({ request, session }) {
+        const enqueueLinksWrapper = async (options) => {
+            const requestManager = await this.getRequestManager();
+            return await this.enqueueLinksWithCrawlDepth(options, request, requestManager);
+        };
+        const addRequests = async (requests, options = {}) => {
+            const newCrawlDepth = request.crawlDepth + 1;
+            const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
+            await this.addRequests(requestsGenerator, options);
+        };
+        const sendRequest = createSendRequest(this.httpClient, request, session);
+        return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };
+    }
+    buildFinalContextPipeline() {
+        const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
+            this.buildContextPipeline());
+        // `extendContext` runs *before* the subclass navigation pipeline (which includes the
+        // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
+        // request handler alike. The trade-off is that `extendContext` cannot access
+        // navigation-dependent context members (e.g. `page`, `response`, `$`, `body`), as those
+        // don't exist yet at this point in the pipeline.
+        // The `extendContext` output (`ContextExtension`) is carried through the subclass pipeline at
+        // runtime (the pipeline copies each middleware's returned members onto the shared context), but
+        // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
+        // casts below are sound because `buildFinalContextPipeline` is declared to return the fully
+        // resolved `ExtendedContext` (= `Context & ContextExtension`).
+        const { extendContext } = this.#contextPipelineOptions;
+        let contextPipeline;
+        if (extendContext !== undefined) {
+            contextPipeline = ContextPipeline.create()
+                .compose({ action: async (context) => await extendContext(context) })
+                .chain(subclassPipeline);
         }
-        this.useSessionPool = useSessionPool;
-        const maxSignedInteger = 2 ** 31 - 1;
-        if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
-            log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
-                ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
-            this.requestHandlerTimeoutMillis = maxSignedInteger;
-        }
-        this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
-        let shouldLogMaxPagesExceeded = true;
-        const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
-        // eslint-disable-next-line prefer-const
-        let { isFinishedFunction, isTaskReadyFunction } = autoscaledPoolOptions;
-        // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
-        if (keepAlive) {
-            isFinishedFunction = async () => false;
-        }
-        const basicCrawlerAutoscaledPoolConfiguration = {
-            minConcurrency: minConcurrency ?? autoscaledPoolOptions?.minConcurrency,
-            maxConcurrency: maxConcurrency ?? autoscaledPoolOptions?.maxConcurrency,
-            maxTasksPerMinute: maxRequestsPerMinute ?? autoscaledPoolOptions?.maxTasksPerMinute,
-            runTaskFunction: this._runTaskFunction.bind(this),
-            isTaskReadyFunction: async () => {
-                if (isMaxPagesExceeded()) {
-                    if (shouldLogMaxPagesExceeded) {
-                        log.info('Crawler reached the maxRequestsPerCrawl limit of ' +
-                            `${maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
-                        shouldLogMaxPagesExceeded = false;
-                    }
-                    return false;
-                }
-                return isTaskReadyFunction ? await isTaskReadyFunction() : await this._isTaskReadyFunction();
-            },
-            isFinishedFunction: async () => {
-                if (isMaxPagesExceeded()) {
-                    log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${maxRequestsPerCrawl} requests ` +
-                        'and all requests that were in progress at that time have now finished. ' +
-                        `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
-                    return true;
-                }
-                const isFinished = isFinishedFunction
-                    ? await isFinishedFunction()
-                    : await this._defaultIsFinishedFunction();
-                if (isFinished) {
-                    const reason = isFinishedFunction
-                        ? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
-                        : 'All requests from the queue have been processed, the crawler will shut down.';
-                    log.info(reason);
+        else {
+            contextPipeline = subclassPipeline;
+        }
+        contextPipeline = contextPipeline.compose({
+            action: async (context) => {
+                const { request } = context;
+                if (request && !this.requestMatchesEnqueueStrategy(request)) {
+                    // eslint-disable-next-line dot-notation
+                    const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
+                    this.log.debug(message);
+                    request.noRetry = true;
+                    request.state = RequestState.SKIPPED;
+                    await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
+                    throw new ContextPipelineInterruptedError(message);
                 }
-                return isFinished;
+                return context;
             },
-            log,
-        };
-        this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };
+        });
+        return contextPipeline;
     }
     /**
      * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
@@ -350,20 +648,30 @@ export class BasicCrawler {
      * @param error The error to check.
      */
     isProxyError(error) {
-        return ROTATE_PROXY_ERRORS.some((x) => this._getMessageFromError(error)?.includes(x));
+        return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error)?.includes(x));
     }
     /**
+     * Sets the status message for the current crawler run.
+     *
      * This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
+     *
+     * The message is logged and broadcast via the {@link EventType.STATUS_MESSAGE|`statusMessage`}
+     * event. Integrations such as the Apify SDK subscribe to that event and forward the message to
+     * their status-reporting backend (e.g. the Apify platform).
      */
-    async setStatusMessage(message, options = {}) {
+    setStatusMessage(message, options = {}) {
         const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
-        this.log.internal(LogLevel[options.level ?? 'DEBUG'], message, data);
-        const client = this.config.getStorageClient();
-        if (!client.setStatusMessage) {
-            return;
-        }
-        // just to be sure, this should be fast
-        await addTimeoutToPromise(async () => client.setStatusMessage(message, options), 1000, 'Setting status message timed out after 1s').catch((e) => this.log.debug(e.message));
+        this.log.logWithLevel(LogLevel[options.level ?? 'DEBUG'], message, data);
+        // Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can
+        // subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.
+        // Setting the status message is not a storage concern, so we intentionally don't route it
+        // through the storage client anymore.
+        serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
+            crawlerId: this.identity.id,
+            message,
+            isStatusMessageTerminal: options.isStatusMessageTerminal,
+            level: options.level,
+        });
     }
     getPeriodicLogger() {
         let previousState = { ...this.stats.state };
@@ -371,23 +679,24 @@ export class BasicCrawler {
             const { requestsFailed } = this.stats.state;
             const { requestsFailed: previousRequestsFailed } = previousState;
             previousState = { ...this.stats.state };
-            if (requestsFailed - previousRequestsFailed > 0) {
-                return 'ERROR';
+            const failedDelta = requestsFailed - previousRequestsFailed;
+            if (failedDelta > 0) {
+                return { mode: 'ERROR', failedDelta };
             }
-            return 'REGULAR';
+            return { mode: 'REGULAR', failedDelta: 0 };
         };
         const log = async () => {
-            const operationMode = getOperationMode();
+            const { mode: operationMode, failedDelta } = getOperationMode();
             let message;
             if (operationMode === 'ERROR') {
-                message = `Experiencing problems, ${this.stats.state.requestsFailed - previousState.requestsFailed || this.stats.state.requestsFailed} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
+                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.#statusMessageLoggingInterval} seconds.`;
             }
             else {
-                const total = this.requestQueue?.getTotalCount() || this.requestList?.length();
-                message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.autoscaledPool?.desiredConcurrency ?? 0}.`;
+                const total = await this.requestManager?.getTotalCount();
+                message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
-            if (this.statusMessageCallback) {
-                await this.statusMessageCallback({
+            if (this.#statusMessageCallback) {
+                await this.#statusMessageCallback({
                     crawler: this,
                     state: this.stats.state,
                     previousState,
@@ -395,14 +704,15 @@ export class BasicCrawler {
                 });
                 return;
             }
-            await this.setStatusMessage(message);
+            this.setStatusMessage(message);
         };
-        const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);
+        const interval = setInterval(log, this.#statusMessageLoggingInterval * 1e3);
         return { log, stop: () => clearInterval(interval) };
     }
     /**
-     * Runs the crawler. Returns a promise that resolves once all the requests are processed
-     * and `autoscaledPool.isFinished` returns `true`.
+     * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
+     * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
+     * default "the request manager is empty") reports that the crawl is over.
      *
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
@@ -414,49 +724,74 @@ export class BasicCrawler {
         if (this.running) {
             throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
         }
-        const { purgeRequestQueue = true, ...addRequestsOptions } = options ?? {};
+        const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
         if (this.hasFinishedBefore) {
             // When executing the run method for the second time explicitly,
-            // we need to purge the default RQ to allow processing the same requests again - this is important so users can
+            // we need to purge the RQ to allow processing the same requests again — this is important so users can
             // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
-            // ignored - as a failed requests is still handled.
-            if (this.requestQueue?.name === 'default' && purgeRequestQueue) {
-                await this.requestQueue.drop();
-                this.requestQueue = await this._getRequestQueue();
+            // ignored — as a failed request is still handled.
+            // By default (`purgeRequestQueue` unset), only the queue we opened ourselves is purged.
+            // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
+            // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
+            const shouldPurge = purgeRequestQueue !== false;
+            const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
+            if (managerToPurge?.purge && shouldPurge) {
+                await managerToPurge.purge();
             }
-            this.stats.reset();
-            await this.stats.resetStore();
-            await this.sessionPool?.resetStore();
+            // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+            await this.#statsDep.ifOwned(async (stats) => {
+                stats.reset();
+                await stats.resetStore();
+            });
+            await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());
         }
+        this.#unexpectedStop = false;
         this.running = true;
-        await purgeDefaultStorages({ onlyPurgeOnce: true });
+        this.#loggedPerRun.clear();
+        await purgeDefaultStorages({
+            onlyPurgeOnce: true,
+            storageBackend: serviceLocator.getStorageBackend(),
+            configuration: serviceLocator.getConfiguration(),
+        });
         if (requests) {
             await this.addRequests(requests, addRequestsOptions);
         }
-        await this._init();
-        await this.stats.startCapturing();
+        try {
+            await this.init();
+            await this.stats.startCapturing();
+        }
+        catch (error) {
+            // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
+            await this.teardown().catch((teardownError) => {
+                this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
+            });
+            // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
+            this.running = false;
+            throw error;
+        }
         const periodicLogger = this.getPeriodicLogger();
-        await this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
+        this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
         const sigintHandler = async () => {
             this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
-            await this._pauseOnMigration();
-            await this.autoscaledPool.abort();
+            await this.pauseOnMigration();
+            await this.#autoscaledPool.abort();
         };
         // Attach a listener to handle migration and aborting events gracefully.
-        const boundPauseOnMigration = this._pauseOnMigration.bind(this);
+        const boundPauseOnMigration = this.pauseOnMigration.bind(this);
         process.once('SIGINT', sigintHandler);
-        this.events.on("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
-        this.events.on("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
+        const eventManager = serviceLocator.getEventManager();
+        eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
+        eventManager.on(EventType.ABORTING, boundPauseOnMigration);
         let stats = {};
         try {
-            await this.autoscaledPool.run();
+            await this.#autoscaledPool.run();
         }
         finally {
             await this.teardown();
             await this.stats.stopCapturing();
             process.off('SIGINT', sigintHandler);
-            this.events.off("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
-            this.events.off("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
+            eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
+            eventManager.off(EventType.ABORTING, boundPauseOnMigration);
             const finalStats = this.stats.calculate();
             stats = {
                 requestsFinished: this.stats.state.requestsFinished,
@@ -473,7 +808,7 @@ export class BasicCrawler {
                     mostCommonErrors: this.stats.errorTracker.getMostPopularErrors(3).map(prettify),
                 });
             }
-            const client = this.config.getStorageClient();
+            const client = serviceLocator.getStorageBackend();
             if (client.teardown) {
                 let finished = false;
                 setTimeout(() => {
@@ -485,7 +820,7 @@ export class BasicCrawler {
                 finished = true;
             }
             periodicLogger.stop();
-            await this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
+            this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
             this.running = false;
             this.hasFinishedBefore = true;
         }
@@ -495,29 +830,166 @@ export class BasicCrawler {
      * Gracefully stops the current run of the crawler.
      *
      * All the tasks active at the time of calling this method will be allowed to finish.
+     *
+     * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
-    stop(message = 'The crawler has been gracefully stopped.') {
-        // Gracefully starve the this.autoscaledPool, so it doesn't start new tasks. Resolves once the pool is cleared.
-        this.autoscaledPool
-            ?.pause()
-            // Resolves the `autoscaledPool.run()` promise in the `BasicCrawler.run()` method. Since the pool is already paused, it resolves immediately and doesn't kill any tasks.
-            .then(async () => this.autoscaledPool?.abort())
-            .then(() => this.log.info(message))
-            .catch((err) => {
-            this.log.error('An error occurred when stopping the crawler:', err);
-        });
+    stop(reason = 'The crawler has been gracefully stopped.') {
+        if (this.#unexpectedStop) {
+            return;
+        }
+        this.log.info(reason);
+        this.#unexpectedStop = true;
     }
+    /**
+     * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
+     * after `timeoutSecs` if they take too long. Unlike {@link BasicCrawler.stop|`stop()`}, this does not end the
+     * run — {@link BasicCrawler.run|`run()`} stays pending until {@link BasicCrawler.resume|`resume()`}.
+     *
+     * > *NOTE:* The {@link BasicCrawler.concurrencySystem|concurrency system} keeps monitoring and autoscaling
+     * throughout, since a shared one may still be serving other crawlers.
+     */
+    async pause(timeoutSecs) {
+        if (!this.#autoscaledPool) {
+            this.log.warning('Cannot pause a crawler that is not running.');
+            return;
+        }
+        await this.#autoscaledPool.pause(timeoutSecs);
+    }
+    /**
+     * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
+     * again. A no-op on a crawler that is not paused.
+     */
+    resume() {
+        if (!this.#autoscaledPool) {
+            this.log.warning('Cannot resume a crawler that is not running.');
+            return;
+        }
+        this.#autoscaledPool.resume();
+    }
+    /**
+     * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
+     * if none has been configured or opened yet.
+     */
+    async getRequestManager() {
+        if (!this.requestManager) {
+            this.requestManager = await this.openOwnedRequestQueue();
+        }
+        // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
+        // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
+        // but guard so we do not re-issue it on every call.
+        if (!this.#requestManagerTimeoutsApplied) {
+            this.#requestManagerTimeoutsApplied = true;
+            await this.applyRequestManagerTimeouts(this.requestManager);
+        }
+        return this.requestManager;
+    }
+    /**
+     * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
+     * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
+     */
     async getRequestQueue() {
-        if (!this.requestQueue && this.requestList) {
-            this.log.warningOnce('When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.');
+        return this.getRequestManager();
+    }
+    /**
+     * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
+     * crawler-owned queue (so it gets purged between repeated `run()` calls).
+     * @private
+     */
+    async openOwnedRequestQueue() {
+        // The first crawler instance uses the default queue (null identifier);
+        // subsequent instances get their own queue via a unique alias so they don't collide.
+        const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
+        const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
+        return this.#ownedRequestQueue.set(requestQueue);
+    }
+    /**
+     * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
+     * locking storage backend keeps it reserved for slightly longer than the request handler timeout
+     * (with some padding for overhead), but never for less than a minute. This prevents a long-running
+     * request from being handed out a second time while it is still being processed — and it works
+     * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
+     */
+    async applyRequestManagerTimeouts(requestManager) {
+        // A router route may hold a request for longer than the crawler's own timeout, and we cannot know
+        // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
+        // raise-only, so erring high here is safe.
+        const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
+        const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
+        await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
+    }
+    /**
+     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
+     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
+     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
+     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
+     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
+     */
+    async validateRequestUserData(source) {
+        if (typeof source === 'string') {
+            return;
+        }
+        const getSchema = this.requestHandler.getSchema;
+        if (typeof getSchema !== 'function') {
+            return;
+        }
+        // Resolve the label via its public accessors only — the top-level `label` of a `RequestOptions` or the
+        // `Request.label` getter — rather than reaching into `userData`, where the request happens to store it.
+        const target = source;
+        const schema = getSchema(target.label);
+        if (!schema) {
+            return;
         }
-        this.requestQueue ??= await this._getRequestQueue();
-        return this.requestQueue;
+        // Store the parsed value rather than the raw input, so the queue holds the same coerced `userData` the
+        // handler will see. Assigning through a `Request` instance's setter keeps its internal `__crawlee` meta.
+        target.userData = await validateUserData(target.label, schema, target.userData ?? {});
     }
     async useState(defaultValue = {}) {
-        const kvs = await KeyValueStore.open(null, { config: this.config });
+        const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
+        if (this.identity.hasExplicitId) {
+            const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
+            return kvs.getAutoSavedValue(stateKey, defaultValue);
+        }
+        BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);
+        if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
+            serviceLocator
+                .getLogger()
+                .warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
+                'This means they will share the same state object, which is likely unintended. \n' +
+                'To fix this, provide a unique `id` option to each crawler instance. \n' +
+                'Example: new BasicCrawler({ id: "my-crawler-1", ... })');
+        }
         return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
     }
+    async getPendingRequestCountApproximation() {
+        return (await this.requestManager?.getPendingCount()) ?? 0;
+    }
+    async calculateEnqueuedRequestLimit(explicitLimit) {
+        if (this.maxRequestsPerCrawl === undefined) {
+            return explicitLimit;
+        }
+        const limit = Math.max(0, this.maxRequestsPerCrawl - this.handledRequestsCount - (await this.getPendingRequestCountApproximation()));
+        return Math.min(limit, explicitLimit ?? Infinity);
+    }
+    async handleSkippedRequest(options) {
+        // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
+        // transaction scope, which rolls back - so the skip bookkeeping must write directly.
+        await withDirectStorageAccess(async () => {
+            if (options.reason === 'limit') {
+                this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
+                    `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
+            }
+            if (options.reason === 'depth') {
+                this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
+            }
+            await this.onSkippedRequest?.(options);
+        });
+    }
+    logOncePerRun(key, message, level = 'info') {
+        if (!this.#loggedPerRun.has(key)) {
+            this.log[level](message);
+            this.#loggedPerRun.add(key);
+        }
+    }
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -530,46 +1002,73 @@ export class BasicCrawler {
      * @param options Options for the request queue
      */
     async addRequests(requests, options = {}) {
-        const requestQueue = await this.getRequestQueue();
-        if (!this.respectRobotsTxtFile) {
-            return requestQueue.addRequestsBatched(requests, options);
-        }
-        const allowedRequests = [];
-        const skipped = new Set();
-        for (const request of requests) {
-            const url = typeof request === 'string' ? request : request.url;
-            if (await this.isAllowedBasedOnRobotsTxtFile(url)) {
-                allowedRequests.push(request);
-            }
-            else {
-                skipped.add(url);
-                await this.onSkippedRequest?.({ url, reason: 'robotsTxt' });
+        await this.getRequestManager();
+        const requestLimit = await this.calculateEnqueuedRequestLimit();
+        const skippedBecauseOfRobots = new Set();
+        const skippedBecauseOfMaxCrawlDepth = new Set();
+        const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
+        const maxCrawlDepth = this.maxCrawlDepth;
+        const validateRequestUserData = this.validateRequestUserData.bind(this);
+        ow(requests, ow.object
+            .is((value) => isIterable(value) || isAsyncIterable(value))
+            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
+        async function* filteredRequests() {
+            for await (const request of requests) {
+                const url = typeof request === 'string' ? request : request.url;
+                if (maxCrawlDepth !== undefined && request.crawlDepth > maxCrawlDepth) {
+                    skippedBecauseOfMaxCrawlDepth.add(url);
+                    continue;
+                }
+                if (await isAllowedBasedOnRobotsTxtFile(url)) {
+                    await validateRequestUserData(request);
+                    yield request;
+                }
+                else {
+                    skippedBecauseOfRobots.add(url);
+                }
             }
         }
-        if (skipped.size > 0) {
+        const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
+            ...options,
+            maxNewRequests: requestLimit,
+        });
+        // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit)
+        const skippedBecauseOfLimit = result.requestsOverLimit ?? [];
+        if (skippedBecauseOfRobots.size > 0) {
             this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
-                skipped: [...skipped],
+                skipped: [...skippedBecauseOfRobots],
             });
-            if (this.onSkippedRequest) {
-                await Promise.all([...skipped].map((url) => {
-                    return this.onSkippedRequest({ url, reason: 'robotsTxt' });
-                }));
-            }
         }
-        return requestQueue.addRequestsBatched(allowedRequests, options);
+        if (skippedBecauseOfRobots.size > 0 ||
+            skippedBecauseOfLimit.length > 0 ||
+            skippedBecauseOfMaxCrawlDepth.size > 0) {
+            await Promise.all([...skippedBecauseOfRobots]
+                .map((url) => {
+                return this.handleSkippedRequest({ url, reason: 'robotsTxt' });
+            })
+                .concat(skippedBecauseOfLimit.map((request) => {
+                const url = typeof request === 'string' ? request : request.url;
+                return this.handleSkippedRequest({ url, reason: 'limit' });
+            }), [...skippedBecauseOfMaxCrawlDepth].map((url) => {
+                return this.handleSkippedRequest({ url, reason: 'depth' });
+            })));
+        }
+        return result;
     }
     /**
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
      */
-    async pushData(data, datasetIdOrName) {
-        const dataset = await this.getDataset(datasetIdOrName);
+    async pushData(data, datasetIdentifier) {
+        const dataset = await this.getDataset(datasetIdentifier);
         return dataset.pushData(data);
     }
     /**
      * Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
      */
-    async getDataset(idOrName) {
-        return Dataset.open(idOrName, { config: this.config });
+    async getDataset(identifier) {
+        return Dataset.open(identifier, {
+            configuration: serviceLocator.getConfiguration(),
+        });
     }
     /**
      * Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
@@ -584,8 +1083,9 @@ export class BasicCrawler {
      */
     async exportData(path, format, options) {
         const supportedFormats = ['json', 'csv'];
-        if (!format && path.match(/\.(json|csv)$/i)) {
-            format = path.toLowerCase().match(/\.(json|csv)$/)[1];
+        const formatMatch = /\.(json|csv)$/i.exec(path);
+        if (!format && formatMatch) {
+            format = formatMatch[1].toLowerCase();
         }
         if (!format) {
             throw new Error(`Failed to infer format from the path: '${path}'. Supported formats: ${supportedFormats.join(', ')}`);
@@ -596,67 +1096,214 @@ export class BasicCrawler {
         const dataset = await this.getDataset();
         const items = await dataset.export(options);
         if (format === 'csv') {
-            const value = stringify([Object.keys(items[0]), ...items.map((item) => Object.values(item))]);
-            await ensureDir(dirname(path));
+            let value;
+            if (items.length === 0) {
+                value = '';
+            }
+            else {
+                const keys = options?.collectAllKeys
+                    ? Array.from(new Set(items.flatMap(Object.keys)))
+                    : Object.keys(items[0]);
+                const { stringify } = await import('csv-stringify/sync');
+                value = stringify([
+                    keys,
+                    ...items.map((item) => {
+                        return keys.map((k) => item[k]);
+                    }),
+                ]);
+            }
+            await mkdir(dirname(path), { recursive: true });
             await writeFile(path, value);
             this.log.info(`Export to ${path} finished!`);
         }
         if (format === 'json') {
-            await ensureDir(dirname(path));
-            await writeJSON(path, items, { spaces: 4 });
+            await mkdir(dirname(path), { recursive: true });
+            await writeFile(path, `${JSON.stringify(items, null, 4)}\n`);
             this.log.info(`Export to ${path} finished!`);
         }
         return items;
     }
-    async _init() {
-        if (!this.events.isInitialized()) {
-            await this.events.init();
-            this._closeEvents = true;
-        }
-        // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),
-        // so that the caller can get a reference to it before awaiting the promise returned from run()
-        // (otherwise there would be no way)
-        this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions, this.config);
-        if (this.useSessionPool) {
-            this.sessionPool = await SessionPool.open(this.sessionPoolOptions, this.config);
-            // Assuming there are not more than 20 browsers running at once;
-            this.sessionPool.setMaxListeners(20);
+    /**
+     * Initializes the crawler.
+     */
+    async init() {
+        const eventManager = serviceLocator.getEventManager();
+        if (!eventManager.isInitialized()) {
+            await eventManager.init();
+            this.#closeEvents = true;
+        }
+        // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
+        // floored per request so it will not actually cut them short, but the configured value is then effectively
+        // ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
+        // navigation timeout only after `super()`.
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(undefined);
+        if (this.internalTimeoutMillis < phasesMillis) {
+            this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
+                `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
+                `so it does not cut them short.`);
+        }
+        // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
+        // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
+        // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
+        this.#concurrencySystemDep = this.#resolveConcurrencySystem();
+        await this.#concurrencySystemDep.ifOwned((system) => system.start());
+        this.#autoscaledPool = new AutoscaledPool({
+            ...this.taskLoopOptions,
+            concurrencySystem: this.#concurrencySystemDep.value,
+            consumer: this.identity,
+        });
+        await this.getRequestManager();
+    }
+    /**
+     * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
+     * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
+     * browser crawlers override it with their `navigationTimeoutSecs`.
+     */
+    getNavigationTimeoutMillis() {
+        return 0;
+    }
+    /**
+     * Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
+     * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
+     * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
+     * whichever is larger: the configured internal timeout, or this request's combined phase budget.
+     */
+    async withRequestTimeout(crawlingContext, work) {
+        const { request } = crawlingContext;
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(request.label);
+        const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
+        await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
+    }
+    /**
+     * The request handler timeout for a request with the given route label. A router route may override the
+     * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
+     *
+     * @param label The request's route label, or `undefined` for the default route / no specific request.
+     * @param fallbackMillis Timeout to use when no route overrides it.
+     */
+    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.requestHandlerTimeoutMillis) {
+        return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
+    }
+    /**
+     * The timeout the router route with the given label asked for, or `undefined` when it did not override one
+     * (or the request handler is not a router at all).
+     */
+    getRouteTimeoutMillis(label) {
+        const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
+        if (typeof getTimeoutSecs !== 'function') {
+            return undefined;
         }
-        await this._loadHandledRequestCount();
+        const timeoutSecs = getTimeoutSecs(label);
+        return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
     }
     async runRequestHandler(crawlingContext) {
-        await this.contextPipeline.call(crawlingContext, async (finalContext) => {
-            await addTimeoutToPromise(async () => this.requestHandler(finalContext), this.requestHandlerTimeoutMillis, `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${finalContext.request.id}).`);
+        const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
+        await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
+    }
+    /**
+     * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
+     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
+     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     */
+    async runInStorageTransaction(callback) {
+        if (!this.#transactionalStorageEnabled) {
+            return callback();
+        }
+        const transaction = createStorageTransaction({
+            policy: this.#storageWritePolicy,
+            commitTimeoutMillis: this.internalTimeoutMillis,
         });
+        let threw = true;
+        try {
+            const result = await transaction.run(callback);
+            threw = false;
+            return result;
+        }
+        finally {
+            if (transaction.state === 'open') {
+                // `handleRequest` commits or rolls back on every normal path, so an open transaction on
+                // a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
+                // expected. Either way, discard the unvalidated writes; only the former is worth flagging.
+                if (!threw) {
+                    this.log.error('Internal error: a storage transaction was still open after the request pipeline ' +
+                        'returned normally. Its writes are being discarded. Please report this.');
+                }
+                transaction.rollback();
+            }
+            // Unconditional: `failed` is a terminal state that the branch above never reaches.
+            transaction.dispose();
+        }
     }
     /**
      * Handles blocked request
      */
-    _throwOnBlockedRequest(session, statusCode) {
-        const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
-        if (isBlocked) {
-            throw new Error(`Request blocked - received ${statusCode} status code.`);
+    throwOnBlockedRequest(statusCode) {
+        if (this.retryOnBlocked)
+            return;
+        if (this.blockedStatusCodes.has(statusCode)) {
+            throw new SessionError(`Request blocked - received ${statusCode} status code.`);
         }
     }
     async isAllowedBasedOnRobotsTxtFile(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return true;
         }
         const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
-        return !robotsTxtFile || robotsTxtFile.isAllowed(url);
+        const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';
+        if (robotsTxtFile) {
+            const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
+            if (crawlDelay !== undefined) {
+                this.applyCrawlDelay(url, crawlDelay);
+            }
+        }
+        return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
+    }
+    /**
+     * Records an HTTP 429 against the URL's domain so the request manager can pace the retry.
+     *
+     * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
+     * @returns `true` if a manager took responsibility for the delay, in which case the caller should throw
+     *  {@link RequestThrottledError} rather than treating the response as a blocked session.
+     */
+    recordDomainRateLimit(url, retryAfterHeader) {
+        if (supportsDomainThrottling(this.requestManager) &&
+            this.requestManager.recordDomainDelay(url, parseRetryAfterHeader(retryAfterHeader))) {
+            return true;
+        }
+        const domain = hostnameOrUrl(url);
+        this.logOncePerRun(`rateLimitNotThrottled:${domain}`, `"${domain}" responded with HTTP 429 (Too Many Requests), but nothing is set up to back off from it, ` +
+            'so the response is handled like any other, with no per-domain delay. ' +
+            `Pass a \`ThrottlingRequestManager\` as \`requestManager\` and include "${domain}" in its \`domains\` ` +
+            'option to honour `Retry-After` and apply exponential backoff instead.', 'warning');
+        return false;
+    }
+    /**
+     * Hands a robots.txt `Crawl-delay` to the request manager, warning if nothing is able to honour it.
+     *
+     * The warning is driven by whether the delay was actually accepted rather than by the type of the manager,
+     * because a manager that does throttle still drops the delay for a domain missing from its `domains` list.
+     */
+    applyCrawlDelay(url, delaySeconds) {
+        if (supportsDomainThrottling(this.requestManager) && this.requestManager.setCrawlDelay(url, delaySeconds)) {
+            return;
+        }
+        const domain = hostnameOrUrl(url);
+        this.logOncePerRun(`crawlDelayIgnored:${domain}`, `robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but nothing is set up to honour it, ` +
+            'so requests to that domain will not be paced. Pass a `ThrottlingRequestManager` as `requestManager` ' +
+            `and include "${domain}" in its \`domains\` option to enforce the delay.`, 'warning');
     }
     async getRobotsTxtFileForUrl(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return undefined;
         }
         try {
             const origin = new URL(url).origin;
-            const cachedRobotsTxtFile = this.robotsTxtFileCache.get(origin);
+            const cachedRobotsTxtFile = this.#robotsTxtFileCache.get(origin);
             if (cachedRobotsTxtFile) {
                 return cachedRobotsTxtFile;
             }
-            const robotsTxtFile = await RobotsTxtFile.find(url);
-            this.robotsTxtFileCache.add(origin, robotsTxtFile);
+            const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
+            this.#robotsTxtFileCache.add(origin, robotsTxtFile);
             return robotsTxtFile;
         }
         catch (e) {
@@ -664,10 +1311,10 @@ export class BasicCrawler {
             return undefined;
         }
     }
-    async _pauseOnMigration() {
-        if (this.autoscaledPool) {
+    async pauseOnMigration() {
+        if (this.#autoscaledPool) {
             // if run wasn't called, this is going to crash
-            await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
+            await this.#autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
                 if (err.message.includes('running tasks did not finish')) {
                     this.log.error('The crawler was paused due to migration to another host, ' +
                         "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -677,14 +1324,16 @@ export class BasicCrawler {
                 }
             });
         }
-        const requestListPersistPromise = (async () => {
-            if (this.requestList) {
-                if (await this.requestList.isFinished())
+        const requestManagerPersistPromise = (async () => {
+            // The request manager persists its read-only loader's state, if it has one that supports persistence
+            // (e.g. a tandem wrapping a `RequestList`). For a plain `RequestQueue`, this is a no-op.
+            if (this.requestManager?.persistState) {
+                if (await this.requestManager.isFinished())
                     return;
-                await this.requestList.persistState().catch((err) => {
+                await this.requestManager.persistState().catch((err) => {
                     if (err.message.includes('Cannot persist state.')) {
                         this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
-                            'invalid config. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
+                            'invalid configuration. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
                             'constructor to ensure your crawling state is persisted through host migrations and restarts.');
                     }
                     else {
@@ -694,33 +1343,16 @@ export class BasicCrawler {
                 });
             }
         })();
-        await Promise.all([requestListPersistPromise, this.stats.persistState()]);
+        await Promise.all([requestManagerPersistPromise, this.stats.persistState?.()]);
     }
     /**
-     * Fetches request from either RequestList or RequestQueue. If request comes from a RequestList
-     * and RequestQueue is present then enqueues it to the queue first.
+     * Fetches the next request to process from the underlying request provider.
      */
-    async _fetchNextRequest() {
-        if (!this.requestList || (await this.requestList.isFinished())) {
-            return this.requestQueue?.fetchNextRequest();
-        }
-        const request = await this.requestList.fetchNextRequest();
-        if (!this.requestQueue)
-            return request;
-        if (!request)
-            return this.requestQueue.fetchNextRequest();
-        try {
-            await this.requestQueue.addRequest(request, { forefront: true });
+    async fetchNextRequest() {
+        if (this.requestManager === undefined) {
+            throw new Error(`fetchNextRequest called on an uninitialized crawler`);
         }
-        catch (err) {
-            // If requestQueue.addRequest() fails here then we must reclaim it back to
-            // the RequestList because probably it's not yet in the queue!
-            this.log.error('Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.', { request });
-            await this.requestList.reclaimRequest(request);
-            return null;
-        }
-        await this.requestList.markRequestHandled(request);
-        return this.requestQueue.fetchNextRequest();
+        return this.requestManager.fetchNextRequest();
     }
     /**
      * Delays processing of the request based on the `sameDomainDelaySecs` option,
@@ -733,114 +1365,54 @@ export class BasicCrawler {
             return false;
         }
         const now = Date.now();
-        const lastAccessTime = this.domainAccessedTime.get(domain);
-        if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {
-            this.domainAccessedTime.set(domain, now);
+        const lastAccessTime = this.#domainAccessedTime.get(domain);
+        if (!lastAccessTime || now - lastAccessTime >= this.#sameDomainDelayMillis) {
+            this.#domainAccessedTime.set(domain, now);
             return false;
         }
-        if (source instanceof RequestQueueV1) {
-            // eslint-disable-next-line dot-notation
-            source['inProgress']?.delete(request.id);
-        }
-        const delay = lastAccessTime + this.sameDomainDelayMillis - now;
+        const delay = lastAccessTime + this.#sameDomainDelayMillis - now;
         this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
         setTimeout(async () => {
             this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
-            if (source instanceof RequestQueueV1) {
-                // eslint-disable-next-line dot-notation
-                source['inProgress'].add(request.id);
-            }
             await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
         }, delay);
         return true;
     }
-    /**
-     * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
-     * then retries them in a case of an error, etc.
-     */
-    async _runTaskFunction() {
-        const source = this.requestQueue || this.requestList || (await this.getRequestQueue());
-        let request;
-        let session;
-        await this._timeoutAndRetry(async () => {
-            request = await this._fetchNextRequest();
-        }, this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-        tryCancel();
-        if (this.useSessionPool) {
-            await this._timeoutAndRetry(async () => {
-                session = await this.sessionPool.newSession({
-                    proxyInfo: await this.proxyConfiguration?.newProxyInfo({
-                        request: request ?? undefined,
-                    }),
-                    maxUsageCount: 1,
-                });
-            }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-        }
-        tryCancel();
-        if (!request || this.delayRequest(request, source)) {
+    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    async handleRequest(crawlingContext, requestSource, request) {
+        // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
+        // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
+        // execute (and re-report) on top of a request the crawler has already moved past.
+        if (crawlingContext[timeoutExpiredKey]?.()) {
             return;
         }
-        if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
-            this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
-            request.state = RequestState.SKIPPED;
-            request.noRetry = true;
-            await source.markRequestHandled(request);
-            await this.onSkippedRequest?.({
-                url: request.url,
-                reason: 'robotsTxt',
-            });
-            return;
-        }
-        // Reset loadedUrl so an old one is not carried over to retries.
-        request.loadedUrl = undefined;
         const statisticsId = request.id || request.uniqueKey;
-        this.stats.startJob(statisticsId);
-        const deferredCleanup = [];
-        const crawlingContext = {
-            id: cryptoRandomObjectId(10),
-            log: this.log,
-            request,
-            session,
-            proxyInfo: session?.proxyInfo,
-            enqueueLinks: async (options) => {
-                return await enqueueLinks({
-                    // specify the RQ first to allow overriding it
-                    requestQueue: await this.getRequestQueue(),
-                    robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
-                    onSkippedRequest: this.onSkippedRequest,
-                    ...options,
-                });
-            },
-            addRequests: async (requests, options) => {
-                await this.addRequests(requests, options);
-            },
-            pushData: this.pushData.bind(this),
-            useState: this.useState.bind(this),
-            sendRequest: createSendRequest(this.httpClient, request, session),
-            getKeyValueStore: async (idOrName) => KeyValueStore.open(idOrName, { config: this.config }),
-            registerDeferredCleanup: (cleanup) => {
-                deferredCleanup.push(cleanup);
-            },
-        };
+        // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
+        const transaction = currentStorageTransaction();
         let isRequestLocked = true;
         try {
             request.state = RequestState.REQUEST_HANDLER;
             await this.runRequestHandler(crawlingContext);
-            await this._timeoutAndRetry(async () => source.markRequestHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-            isRequestLocked = false; // markRequestHandled succeeded and unlocked the request
+            // Commit *before* marking the request as handled, so a commit failure fails the request and
+            // it is retried. This also closes the transaction, so everything below passes through.
+            await transaction?.commit();
+            await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
             this.stats.finishJob(statisticsId, request.retryCount);
-            this.handledRequestsCount++;
             // reclaim session if request finishes successfully
             request.state = RequestState.DONE;
-            crawlingContext.session?.markGood();
+            crawlingContext.session.markGood();
         }
         catch (rawError) {
+            // Roll back *before* any error handler runs - error handlers write to real storage precisely
+            // because the transaction is already closed. A no-op when the commit above succeeded.
+            transaction?.rollback();
             const err = this.unwrapError(rawError);
             try {
                 request.state = RequestState.ERROR_HANDLER;
-                await addTimeoutToPromise(async () => this._requestFunctionErrorHandler(err, crawlingContext, source), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, crawlingContext, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
                 if (!(err instanceof CriticalError)) {
-                    isRequestLocked = false; // _requestFunctionErrorHandler calls either markRequestHandled or reclaimRequest
+                    isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
                 }
                 request.state = RequestState.DONE;
             }
@@ -858,36 +1430,100 @@ export class BasicCrawler {
                 request.state = RequestState.ERROR;
                 throw unwrappedSecondaryError;
             }
-            // decrease the session score if the request fails (but the error handler did not throw)
-            crawlingContext.session?.markBad();
+            // decrease the session score if the request fails (but the error handler did not throw);
+            // skip when the error is a SessionError, which already retired the session
+            if (!this.errorAbsolvesSession(err)) {
+                crawlingContext.session.markBad();
+            }
         }
         finally {
-            await Promise.all(deferredCleanup.map((cleanup) => cleanup()));
-            // Safety net - release the lock if nobody managed to do it before
-            if (isRequestLocked && source instanceof RequestProvider) {
+            // Safety net - return the request to the queue if nobody managed to mark it as handled
+            // or reclaim it before (e.g. after a CriticalError). Reclaiming a request that is no longer
+            // in progress is a harmless no-op on the storage backend.
+            if (isRequestLocked && requestSource instanceof RequestQueue) {
                 try {
-                    await source.client.deleteRequestLock(request.id);
+                    await requestSource.reclaimRequest(request);
                 }
                 catch {
-                    // We don't have the lock, or the request was never locked. Either way it's fine
+                    // The request was never in progress, or could not be reclaimed. Either way it's fine.
                 }
             }
         }
     }
     /**
-     * Run async callback with given timeout and retry.
+     * Wrapper around the crawling context's `enqueueLinks` method:
+     * - Injects `crawlDepth` to each request being added based on the crawling context request.
+     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
+     *      - These options can be overridden by the user.
+     * @internal
+     */
+    async enqueueLinksWithCrawlDepth(options, request, requestManager) {
+        const transformRequestFunctionWrapper = (requestOptions) => {
+            requestOptions.crawlDepth = request.crawlDepth + 1;
+            if (this.maxCrawlDepth !== undefined && requestOptions.crawlDepth > this.maxCrawlDepth) {
+                // Setting `skippedReason` before returning `false` ensures that `reportSkippedRequests`
+                // reports `'depth'` as the reason (via `request.skippedReason ?? reason` fallback),
+                // rather than the generic `'transform'` reason.
+                requestOptions.skippedReason = 'depth';
+                return false;
+            }
+            // After injecting the crawlDepth, we call the user-provided transform function, if there is one.
+            return options.transformRequestFunction?.(requestOptions) ?? requestOptions;
+        };
+        // Create a request-scoped callback that logs enqueueLimit once per request handler call
+        // Only log if an explicit limit was passed to enqueueLinks (not the internal maxRequestsPerCrawl-derived limit)
+        let loggedEnqueueLimitForThisRequest = false;
+        const onSkippedRequest = async (skippedOptions) => {
+            if (skippedOptions.reason === 'enqueueLimit') {
+                if (!loggedEnqueueLimitForThisRequest && options.limit !== undefined) {
+                    this.log.info(`Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`);
+                    loggedEnqueueLimitForThisRequest = true;
+                }
+            }
+            await this.handleSkippedRequest(skippedOptions);
+        };
+        // `enqueueLinks` applies `options.label`/`options.userData` to every newly enqueued request, so a single
+        // validation against the label's schema covers them all (a no-op unless the router declares a schema).
+        await this.validateRequestUserData({ label: options.label, userData: options.userData });
+        return await enqueueLinks({
+            requestManager,
+            robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
+            respectRobotsTxtFile: this.#respectRobotsTxtFile,
+            onSkippedRequest,
+            limit: await this.calculateEnqueuedRequestLimit(options.limit),
+            // Allow user options to override defaults set above ⤴
+            ...options,
+            transformRequestFunction: transformRequestFunctionWrapper,
+        });
+    }
+    /**
+     * Generator function that yields requests injected with the given crawl depth.
+     * @internal
+     */
+    async *addCrawlDepthRequestGenerator(requests, newRequestDepth) {
+        for await (const request of requests) {
+            if (typeof request === 'string') {
+                yield { url: request, crawlDepth: newRequestDepth };
+            }
+            else {
+                request.crawlDepth ??= newRequestDepth;
+                yield request;
+            }
+        }
+    }
+    /**
+     * Run async callback with given timeout and retry. Returns the result of the callback.
      * @ignore
      */
-    async _timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
+    async timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
         try {
-            await addTimeoutToPromise(handler, timeout, error);
+            return await addTimeoutToPromise(handler, timeout, error);
         }
         catch (e) {
             if (retried <= maxRetries) {
                 // we retry on any error, not just timeout
                 this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
-                void this._timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
-                return;
+                return this.timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
             }
             throw e;
         }
@@ -895,31 +1531,14 @@ export class BasicCrawler {
     /**
      * Returns true if either RequestList or RequestQueue have a request ready for processing.
      */
-    async _isTaskReadyFunction() {
-        // First check RequestList, since it's only in memory.
-        const isRequestListEmpty = this.requestList ? await this.requestList.isEmpty() : true;
-        // If RequestList is not empty, task is ready, no reason to check RequestQueue.
-        if (!isRequestListEmpty)
-            return true;
-        // If RequestQueue is not empty, task is ready, return true, otherwise false.
-        return this.requestQueue ? !(await this.requestQueue.isEmpty()) : false;
+    async isTaskReadyFunction() {
+        return this.requestManager !== undefined && !(await this.requestManager.isEmpty());
     }
     /**
      * Returns true if both RequestList and RequestQueue have all requests finished.
      */
-    async _defaultIsFinishedFunction() {
-        const [isRequestListFinished, isRequestQueueFinished] = await Promise.all([
-            this.requestList ? this.requestList.isFinished() : true,
-            this.requestQueue ? this.requestQueue.isFinished() : true,
-        ]);
-        // If both are finished, return true, otherwise return false.
-        return isRequestListFinished && isRequestQueueFinished;
-    }
-    async _rotateSession(crawlingContext) {
-        const { request } = crawlingContext;
-        request.sessionRotationCount ??= 0;
-        request.sessionRotationCount++;
-        crawlingContext.session?.retire();
+    async defaultIsFinishedFunction() {
+        return !this.requestManager || (await this.requestManager.isFinished());
     }
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
@@ -935,27 +1554,38 @@ export class BasicCrawler {
     }
     /**
      * Handles errors thrown by user provided requestHandler()
+     *
+     * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
-    async _requestFunctionErrorHandler(error, crawlingContext, source) {
-        const { request } = crawlingContext;
+    async requestFunctionErrorHandler(error, crawlingContext, request, source) {
+        if (error instanceof RequestThrottledError) {
+            // The domain told us to come back later, so the request was never really attempted. Put it back
+            // without recording a failure - it costs neither a retry nor session reputation.
+            this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
+                id: request.id,
+                url: request.url,
+            });
+            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+            return;
+        }
         request.pushErrorMessage(error);
         if (error instanceof CriticalError) {
             throw error;
         }
-        const shouldRetryRequest = this._canRequestBeRetried(request, error);
+        const shouldRetryRequest = this.canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
             await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
             await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
             error);
             if (error instanceof SessionError) {
-                await this._rotateSession(crawlingContext);
+                crawlingContext.session?.retire();
             }
             if (!request.noRetry) {
                 request.retryCount++;
                 const { url, retryCount, id } = request;
                 // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
                 // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
-                const message = this._getMessageFromError(error);
+                const message = this.getMessageFromError(error);
                 this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {
                     id,
                     url,
@@ -965,6 +1595,9 @@ export class BasicCrawler {
                 return;
             }
         }
+        if (error instanceof SessionError) {
+            crawlingContext.session?.retire();
+        }
         // If the request is non-retryable, the error and snapshot aren't saved in the errorTrackerRetry object.
         // Therefore, we pass the crawlingContext to the errorTracker.add method, enabling snapshot capture.
         // This is to make sure the error snapshot is not duplicated in the errorTrackerRetry and errorTracker objects.
@@ -978,24 +1611,14 @@ export class BasicCrawler {
         // If we get here, the request is either not retryable
         // or failed more than retryCount times and will not be retried anymore.
         // Mark the request as failed and do not retry.
-        this.handledRequestsCount++;
-        await source.markRequestHandled(request);
+        await source.markRequestAsHandled(request);
         this.stats.failJob(request.id || request.uniqueKey, request.retryCount);
-        await this._handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
+        await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
-    async _tagUserHandlerError(cb) {
-        try {
-            return (await cb());
-        }
-        catch (e) {
-            Object.defineProperty(e, 'triggeredFromUserHandler', { value: true });
-            throw e;
-        }
-    }
-    async _handleFailedRequestHandler(crawlingContext, error) {
+    async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
-        const message = this._getMessageFromError(error, true);
+        const message = this.getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
         if (this.failedRequestHandler) {
             await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
@@ -1007,7 +1630,7 @@ export class BasicCrawler {
      * @param error The error received
      * @returns The message to be logged
      */
-    _getMessageFromError(error, forceStack = false) {
+    getMessageFromError(error, forceStack = false) {
         if ([TypeError, SyntaxError, ReferenceError].some((type) => error instanceof type)) {
             forceStack = true;
         }
@@ -1021,11 +1644,16 @@ export class BasicCrawler {
             ? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
             : [error.message || error, userLine].join('\n');
     }
-    _canRequestBeRetried(request, error) {
-        // Request should never be retried, or the error encountered makes it not able to be retried, or the session rotation limit has been reached
-        if (request.noRetry ||
-            error instanceof NonRetryableError ||
-            (error instanceof SessionError && this.maxSessionRotations <= (request.sessionRotationCount ?? 0))) {
+    /**
+     * Whether the session should be spared for this error - either because it was already retired, or because the
+     * failure says nothing about the session (a rate limit is a property of the domain).
+     */
+    errorAbsolvesSession(error) {
+        return error instanceof SessionError || error instanceof RequestThrottledError;
+    }
+    canRequestBeRetried(request, error) {
+        // Request should never be retried, or the error encountered makes it not able to be retried.
+        if (request.noRetry || error instanceof NonRetryableError) {
             return false;
         }
         // User requested retry (we ignore retry count here as its explicitly told by the user to retry)
@@ -1037,59 +1665,40 @@ export class BasicCrawler {
         return request.retryCount < maxRequestRetries;
     }
     /**
-     * Updates handledRequestsCount from possibly stored counts,
-     * usually after worker migration. Since one of the stores
-     * needs to have priority when both are present,
-     * it is the request queue, because generally, the request
-     * list will first be dumped into the queue and then left
-     * empty.
-     */
-    async _loadHandledRequestCount() {
-        if (this.requestQueue) {
-            this.handledRequestsCount = await this.requestQueue.handledCount();
-        }
-        else if (this.requestList) {
-            this.handledRequestsCount = this.requestList.handledCount();
-        }
-    }
-    async _executeHooks(hooks, ...args) {
-        if (Array.isArray(hooks) && hooks.length) {
-            for (const hook of hooks) {
-                await hook(...args);
-            }
-        }
-    }
-    /**
-     * Function for cleaning up after all request are processed.
-     * @ignore
+     * Stops the crawler immediately.
+     *
+     * This method doesn't wait for currently active requests to finish.
+     *
+     * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     async teardown() {
-        this.events.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
-        await this.sessionPool?.teardown();
-        if (this._closeEvents) {
-            await this.events.close();
+        serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
+        if (this.#closeEvents) {
+            await serviceLocator.getEventManager().close();
         }
-        await this.autoscaledPool?.abort();
+        await this.#sessionPoolDep.ifOwned((pool) => pool.teardown());
+        await this.#autoscaledPool?.abort();
+        await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
     }
-    _getCookieHeaderFromRequest(request) {
+    getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {
             this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
             return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);
         }
         return request.headers?.Cookie || request.headers?.cookie || '';
     }
-    async _getRequestQueue() {
-        // Check if it's explicitly disabled
-        if (this.experiments.requestLocking === false) {
-            if (!this._experimentWarnings.requestLocking) {
-                this.log.info('Using the old RequestQueue implementation without request locking.');
-                this._experimentWarnings.requestLocking = true;
+    requestMatchesEnqueueStrategy(request) {
+        // If `skipNavigation` was used, just return `true`
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+            request.loadedUrl;
+        }
+        catch (err) {
+            if (err instanceof NavigationSkippedError) {
+                return true;
             }
-            return RequestQueueV1.open(null, { config: this.config });
+            throw err;
         }
-        return RequestQueue.open(null, { config: this.config });
-    }
-    requestMatchesEnqueueStrategy(request) {
         const { url, loadedUrl } = request;
         // eslint-disable-next-line dot-notation -- private access
         const strategy = request['enqueueStrategy'];
@@ -1127,31 +1736,10 @@ export class BasicCrawler {
         }
     }
 }
-/**
- * Creates new {@link Router} instance that works based on request labels.
- * This instance can then serve as a {@link BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@link BasicCrawler}.
- * Defaults to the {@link BasicCrawlingContext}.
- *
- * > Serves as a shortcut for using `Router.create<BasicCrawlingContext>()`.
- *
- * ```ts
- * import { BasicCrawler, createBasicRouter } from 'crawlee';
- *
- * const router = createBasicRouter();
- * router.addHandler('label-a', async (ctx) => {
- *    ctx.log.info('...');
- * });
- * router.addDefaultHandler(async (ctx) => {
- *    ctx.log.info('...');
- * });
- *
- * const crawler = new BasicCrawler({
- *     requestHandler: router,
- * });
- * await crawler.run();
- * ```
- */
+/** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
+function hostnameOrUrl(url) {
+    return URL.canParse(url) ? new URL(url).hostname : url;
+}
 export function createBasicRouter(routes) {
     return Router.create(routes);
 }
-//# sourceMappingURL=basic-crawler.js.map