@crawlee/basic 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/basic-crawler.js

@@ -1,21 +1,35 @@
-import { writeFile } from 'node:fs/promises';
+import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, } from '@crawlee/core';
-import { FetchHttpClient } from '@crawlee/http-client';
-import { isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
-import { stringify } from 'csv-stringify/sync';
-import { ensureDir, writeJSON } from 'fs-extra/esm';
-import ow, { ArgumentError } from 'ow';
+import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
+import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
+import { RobotsTxtFile } from '@crawlee/utils';
 import { getDomain } from 'tldts';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
-import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
+import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
-import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
+import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
+import { mergeCookies } from './cookie_utils.js';
+import { ContextPipeline } from './crawlers/context_pipeline.js';
+import { CrawlingRequest, RequestState } from './crawling_request.js';
+import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
+import { Statistics } from './crawlers/statistics.js';
+import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue_links/index.js';
+import { parseRetryAfterHeader } from './http.js';
+import { CrawlerRun } from './crawler-run.js';
+import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
-class LazyDefaultHttpClient {
-    _delegatePromise;
+import { Router, validateUserData } from './router.js';
+import { BLOCKED_STATUS_CODES } from './session_pool/consts.js';
+import { Session } from './session_pool/session.js';
+import { SessionPool } from './session_pool/session_pool.js';
+import { ThrottlingRequestManager } from './throttling_request_manager.js';
+class LazyDefaultHttpClient extends BaseHttpClient {
+    #delegatePromise;
     constructor(options) {
-        this._delegatePromise = import('@crawlee/impit-client')
+        super(options);
+        this.#delegatePromise = import('@crawlee/impit-client')
             .then(({ ImpitHttpClient }) => new ImpitHttpClient(options))
             .catch(() => {
             (options?.logger ?? log).warning('Optional dependency @crawlee/impit-client is not installed. ' +
@@ -23,8 +37,11 @@ class LazyDefaultHttpClient {
             return new FetchHttpClient(options);
         });
     }
+    fetch() {
+        throw new Error('LazyDefaultHttpClient delegates `sendRequest` entirely; `fetch` is never called.');
+    }
     async sendRequest(...args) {
-        return (await this._delegatePromise).sendRequest(...args);
+        return (await this.#delegatePromise).sendRequest(...args);
     }
 }
 /**
@@ -37,84 +54,117 @@ class LazyDefaultHttpClient {
  * @ignore
  */
 const SAFE_MIGRATION_WAIT_MILLIS = 20000;
-const deferredCleanupKey = Symbol('deferredCleanup');
 // The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
 export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
+const urlPatternSchema = z.union([
+    z.string(),
+    z.instanceof(RegExp),
+    schemas.objectWithKeys(['glob']),
+    schemas.objectWithKeys(['regexp']),
+]);
+// `looseObject` (rather than `strictObject`) lets subclasses forward their own extraction-only options
+// (e.g. `selector`) straight through without having to strip them out first.
+const addRequestsOptionsSchema = z.looseObject({
+    forefront: z.boolean().optional(),
+    cache: z.boolean().optional(),
+    waitForAllRequestsToBeAdded: z.boolean().optional(),
+    batchSize: schemas.anyNumber.optional(),
+    waitBetweenBatchesMillis: schemas.anyNumber.optional(),
+    maxNewRequests: schemas.anyNumber.optional(),
+    limit: schemas.anyNumber.optional(),
+    baseUrl: z.string().optional(),
+    userData: schemas.anyObject.optional(),
+    label: z.string().optional(),
+    sessionId: z.string().optional(),
+    skipNavigation: z.boolean().optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').min(1).optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    transformRequestFunction: schemas.anyFunction.optional(),
+    strategy: z.enum(EnqueueStrategy).optional(),
+    onSkippedRequest: schemas.anyFunction.optional(),
+});
 export class BasicCrawler {
-    static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
+    static #CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
     /**
      * Tracks the number of crawler instances created. The first crawler uses the default
      * request queue; subsequent ones get their own queue via a unique alias so they don't
      * collide.
      */
-    static instanceCount = 0;
+    static #instanceCount = 0;
     /**
      * Tracks crawler instances that accessed shared state without having an explicit id.
      * Used to detect and warn about multiple crawlers sharing the same state.
      */
-    static useStateAnonymousIndices = new Set();
+    static #useStateAnonymousIndices = new Set();
+    /** Backs the {@link BasicCrawler.statistics|`statistics`} getter. */
+    #statisticsDep;
     /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+     * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    stats;
+    get statistics() {
+        return this.#statisticsDep.value;
+    }
+    #requestManager;
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
-    requestManager;
+    get requestManager() {
+        return this.#requestManager;
+    }
     /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
-    sessionPoolDep;
+    #sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool() {
-        return this.sessionPoolDep.value;
+        return this.#sessionPoolDep.value;
     }
-    /**
-     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
-     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
-     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
-     */
-    ownedRequestQueue = OwnedOrInjected.resolve();
     /**
      * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
      * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
      * so it only needs to be applied once, at the first async access of the manager.
      */
-    requestManagerTimeoutsApplied = false;
+    #requestManagerTimeoutsApplied = false;
     /**
      * Resolves the governor for one run: either the injected
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
      * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
      */
-    resolveConcurrencySystem;
-    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    concurrencySystemDep;
+    #resolveConcurrencySystem;
+    /** The last run started on this instance. It outlives the run, so anything that drives one goes through `#liveRun`. */
+    #run;
+    #concurrencySystem;
+    /** The run in progress, if any — a finished run is history and cannot be driven. */
+    get #liveRun() {
+        return this.#run?.isLive === true ? this.#run : undefined;
+    }
+    /** The run whose task loop is dispatching, if any — only that one can be paused and resumed. */
+    get #dispatchingRun() {
+        return this.#liveRun?.isDispatching === true ? this.#liveRun : undefined;
+    }
+    /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
+    get running() {
+        return this.#liveRun !== undefined;
+    }
     /**
-     * The concurrency governor this run is booking its requests against — either the
+     * The concurrency governor the run in progress is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
      * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
      *
-     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
-     * is also rebuilt for every run, so the instance is not stable across runs.
+     * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
+     * during a run rather than caching it across runs.
      *
      * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
      * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem() {
-        return this.concurrencySystemDep?.maybeValue;
+        return this.#concurrencySystem?.value;
     }
-    /**
-     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
-     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
-     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
-     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
-     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
-     */
-    autoscaledPool;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -125,132 +175,148 @@ export class BasicCrawler {
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     router = Router.create();
-    _basicContextPipeline;
+    #basicContextPipeline;
     /**
      * The basic part of the context pipeline. Unlike the subclass pipeline, this
      * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
      * pipelines expect the basic crawler fields to already be present in the context at runtime.
      *
      * Context built with this pipeline can be passed into multiple crawler pipelines at once.
-     * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
      */
-    get basicContextPipeline() {
-        if (this._basicContextPipeline === undefined) {
-            this._basicContextPipeline = this.buildBasicContextPipeline();
-        }
-        return this._basicContextPipeline;
+    get #basicPipeline() {
+        this.#basicContextPipeline ??= this.#buildBasicContextPipeline();
+        return this.#basicContextPipeline;
     }
-    _contextPipeline;
+    #contextPipeline;
+    /** @internal */
     get contextPipeline() {
-        if (this._contextPipeline === undefined) {
-            this._contextPipeline = this.buildFinalContextPipeline();
+        if (this.#contextPipeline === undefined) {
+            this.#contextPipeline = this.#buildFinalContextPipeline();
         }
-        return this._contextPipeline;
+        return this.#contextPipeline;
+    }
+    #hasFinishedBefore = false;
+    /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
+    get hasFinishedBefore() {
+        return this.#hasFinishedBefore;
     }
-    running = false;
-    hasFinishedBefore = false;
-    unexpectedStop = false;
     #log;
     get log() {
         return this.#log;
     }
     requestHandler;
-    errorHandler;
-    failedRequestHandler;
-    requestHandlerTimeoutMillis;
+    #errorHandler;
+    #failedRequestHandler;
+    #requestHandlerTimeoutMillis;
     internalTimeoutMillis;
-    maxRequestRetries;
-    maxCrawlDepth;
-    sameDomainDelayMillis;
-    domainAccessedTime;
-    maxRequestsPerCrawl;
-    get handledRequestsCount() {
-        return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
-    }
-    statusMessageLoggingInterval;
-    statusMessageCallback;
+    #maxRequestRetries;
+    #maxCrawlDepth;
+    #maxRequestsPerCrawl;
+    get #handledRequestsCount() {
+        return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;
+    }
+    #statusMessageLoggingInterval;
+    #statusMessageCallback;
     blockedStatusCodes = new Set();
     additionalHttpErrorStatusCodes;
-    ignoreHttpErrorStatusCodes;
+    #ignoreHttpErrorStatusCodes;
     /**
      * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
      * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
      * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
      * settable here.
      */
-    taskLoopOptions;
+    #taskLoopOptions;
     httpClient;
     retryOnBlocked;
-    respectRobotsTxtFile;
-    onSkippedRequest;
-    _closeEvents;
-    loggedPerRun = new Set();
-    robotsTxtFileCache;
-    identity;
-    contextPipelineOptions;
+    #respectRobotsTxtFile;
+    /** Whether `runInStorageTransaction()` opens a transaction at all. */
+    #transactionalStorageEnabled;
+    /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
+    #storageWritePolicy;
+    #onSkippedRequest;
+    #robotsTxtFileCache;
+    #identity;
+    #contextPipelineOptions;
+    /**
+     * @internal
+     */
     static optionsShape = {
-        contextPipelineBuilder: ow.optional.object,
-        extendContext: ow.optional.function,
-        requestList: ow.optional.object.validate(validators.requestList),
-        requestQueue: ow.optional.object.validate(validators.requestQueue),
+        contextPipelineBuilder: schemas.anyObject.optional(),
+        extendContext: schemas.anyFunction.optional(),
+        requestList: validators.requestList.optional(),
+        requestQueue: validators.requestQueue.optional(),
+        requestManager: validators.requestManager.optional(),
         // Subclasses override this function instead of passing it
         // in constructor, so this validation needs to apply only
         // if the user creates an instance of BasicCrawler directly.
-        requestHandler: ow.optional.function,
-        requestHandlerTimeoutSecs: ow.optional.number,
-        errorHandler: ow.optional.function,
-        failedRequestHandler: ow.optional.function,
-        maxRequestRetries: ow.optional.number,
-        sameDomainDelaySecs: ow.optional.number,
-        maxRequestsPerCrawl: ow.optional.number,
-        maxCrawlDepth: ow.optional.number,
-        taskLoopOptions: ow.optional.object,
-        concurrencySystem: ow.optional.object,
-        sessionPool: ow.optional.object.validate(validators.sessionPool),
-        proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
-        statusMessageLoggingInterval: ow.optional.number,
-        statusMessageCallback: ow.optional.function,
-        additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
-        ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
-        blockedStatusCodes: ow.optional.array.ofType(ow.number),
-        retryOnBlocked: ow.optional.boolean,
-        respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object),
-        onSkippedRequest: ow.optional.function,
-        httpClient: ow.optional.object,
-        configuration: ow.optional.object,
-        storageBackend: ow.optional.object,
-        eventManager: ow.optional.object,
-        logger: ow.optional.object,
+        requestHandler: schemas.anyFunction.optional(),
+        requestHandlerTimeoutSecs: schemas.anyNumber.optional(),
+        errorHandler: schemas.anyFunction.optional(),
+        failedRequestHandler: schemas.anyFunction.optional(),
+        maxRequestRetries: schemas.anyNumber.default(3),
+        sameDomainDelaySecs: schemas.anyNumber.default(0),
+        maxRequestsPerCrawl: schemas.anyNumber.optional(),
+        maxCrawlDepth: schemas.anyNumber.optional(),
+        // No zod default — subclasses provide their own fallback (e.g. HTTP-optimized pool options).
+        taskLoopOptions: schemas.anyObject.optional(),
+        concurrencySystem: schemas.anyObject.optional(),
+        sessionPool: validators.sessionPool.optional(),
+        proxyConfiguration: validators.proxyConfiguration.optional(),
+        statusMessageLoggingInterval: schemas.anyNumber.default(10),
+        statusMessageCallback: schemas.anyFunction.optional(),
+        additionalHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
+        ignoreHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
+        blockedStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').optional(),
+        retryOnBlocked: z.boolean().default(false),
+        respectRobotsTxtFile: z.union([z.boolean(), schemas.anyObject]).default(false),
+        transactionalStorage: z
+            .union([z.boolean(), z.strictObject({ requestQueue: z.enum(['deferred', 'writeThrough']).optional() })])
+            .optional(),
+        onSkippedRequest: schemas.anyFunction.optional(),
+        httpClient: schemas.httpClient.optional(),
+        configuration: z.instanceof(Configuration).optional(),
+        storageBackend: validators.storageBackend.optional(),
+        eventManager: z.instanceof(EventManager).optional(),
+        logger: validators.logger.optional(),
         // AutoscaledPool shorthands
-        minConcurrency: ow.optional.number,
-        maxConcurrency: ow.optional.number,
-        maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
-        keepAlive: ow.optional.boolean,
-        statisticsOptions: ow.optional.object,
-        id: ow.optional.string,
+        minConcurrency: schemas.anyNumber.optional(),
+        maxConcurrency: schemas.anyNumber.optional(),
+        initialConcurrency: schemas.anyNumber.optional(),
+        maxRequestsPerMinute: schemas.anyNumber
+            .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
+            .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
+            .optional(),
+        keepAlive: z.boolean().optional(),
+        statistics: schemas.anyObject.optional(),
+        id: z.string().optional(),
     };
+    static #optionsSchema = z.strictObject(BasicCrawler.optionsShape);
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
     constructor(options = {}) {
-        ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
+        const parsedOptions = parseArgument(options, BasicCrawler.#optionsSchema, 'BasicCrawlerOptions');
         const { 
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
         requestList, 
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
-        requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
+        requestQueue, requestManager, maxRequestRetries, sameDomainDelaySecs, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes, ignoreHttpErrorStatusCodes, 
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statisticsOptions, httpClient, id, } = options;
+        minConcurrency, maxConcurrency, initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
         // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
         // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
         // hammering a site.
         if (concurrencySystem !== undefined &&
-            (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
-            throw new ArgumentError('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
-                '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
-                'replaces. Pass them to the `ConcurrencySystem` constructor instead.', this.constructor);
+            (minConcurrency !== undefined ||
+                maxConcurrency !== undefined ||
+                initialConcurrency !== undefined ||
+                maxRequestsPerMinute !== undefined)) {
+            throw new Error('The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
+                'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
+                'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.');
         }
         // Create per-crawler service locator if custom services were provided.
         // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
@@ -262,96 +328,121 @@ export class BasicCrawler {
             eventManager ||
             logger ||
             (configuration !== undefined && configuration !== serviceLocator.getConfiguration())) {
-            const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageBackend, logger);
+            // Inherit the ambient locator's already-set services for anything not explicitly
+            // provided - e.g. only passing a `logger` must not detach the crawler from a globally
+            // configured storage backend.
+            const ambientServices = serviceLocator.getServicesIfSet();
+            const scopedServiceLocator = new ServiceLocator(configuration ?? ambientServices.configuration, eventManager ?? ambientServices.eventManager, storageBackend ?? ambientServices.storageBackend, logger ?? ambientServices.logger);
             serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);
         }
         try {
             serviceLocatorScope.enterScope();
-            this.contextPipelineOptions = {
-                contextPipelineBuilder: options.contextPipelineBuilder,
-                extendContext: options.extendContext,
+            this.#contextPipelineOptions = {
+                contextPipelineBuilder: parsedOptions.contextPipelineBuilder,
+                extendContext: parsedOptions.extendContext,
             };
             this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
             // Initialize the Configuration instance to avoid lazy loading in the components
             serviceLocator.getConfiguration();
-            const instanceIndex = BasicCrawler.instanceCount++;
-            this.identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
-            if (requestManager !== undefined) {
-                if (requestList !== undefined || requestQueue !== undefined) {
-                    throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
-                }
-                this.requestManager = requestManager;
+            const instanceIndex = BasicCrawler.#instanceCount++;
+            this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
+            if (requestManager !== undefined && (requestList !== undefined || requestQueue !== undefined)) {
+                throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
             }
-            else if (requestList !== undefined && requestQueue !== undefined) {
-                // Combine the read-only list with the writable queue into a tandem.
-                this.requestManager = new RequestManagerTandem(requestList, requestQueue);
+            const suppliedManager = requestManager ?? requestQueue;
+            // Offered before building a pacer of our own: anything that paces takes the floor - through any
+            // number of wrappers, since they all forward - so no domain ends up with two clocks.
+            const floorTaken = sameDomainDelaySecs > 0 &&
+                (suppliedManager?.recordPacingSignal({
+                    reason: 'minIntervalEverywhere',
+                    intervalMs: sameDomainDelaySecs * 1000,
+                    // What `sameDomainDelaySecs` has always meant: one clock per site, subdomains included.
+                    scope: 'registrableDomain',
+                }) ??
+                    false);
+            const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken;
+            // Built here rather than at first use so it can sit *inside* the tandem below, which is where a
+            // loader's transferred requests pass through it.
+            const writableManager = pacerNeeded
+                ? new ThrottlingRequestManager({
+                    domains: 'all',
+                    minCrawlDelaySecs: sameDomainDelaySecs,
+                    throttleBy: 'registrableDomain',
+                    persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,
+                    // A factory, because the default queue is only opened on first use.
+                    inner: suppliedManager ?? (() => this.#openOwnedRequestQueue()),
+                })
+                : suppliedManager;
+            if (requestList !== undefined) {
+                // The list is read first, while new requests still have somewhere writable to go.
+                this.#requestManager = new RequestManagerTandem(requestList, writableManager ?? (() => this.#openOwnedRequestQueue()));
             }
-            else if (requestQueue !== undefined) {
+            else if (writableManager !== undefined) {
                 // A RequestQueue is itself a request manager.
-                this.requestManager = requestQueue;
-            }
-            else if (requestList !== undefined) {
-                // A lone read-only `requestList` (deprecated option) is combined with a lazily-opened default queue
-                // into a tandem, so that its requests are read first and new ones can still be enqueued during the
-                // crawl. The queue is opened on first use; the tandem also forwards `persistState()` to the loader.
-                this.requestManager = new RequestManagerTandem(requestList, () => this.openOwnedRequestQueue());
+                this.#requestManager = writableManager;
             }
             this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
             this.proxyConfiguration = proxyConfiguration;
-            this.statusMessageLoggingInterval = statusMessageLoggingInterval;
-            this.statusMessageCallback = statusMessageCallback;
-            this.domainAccessedTime = new Map();
-            this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
-            this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
+            this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
+            this.#statusMessageCallback = statusMessageCallback;
+            this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
             this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
-            this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
+            this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
             this.requestHandler = requestHandler ?? this.router;
-            this.failedRequestHandler = failedRequestHandler;
-            this.errorHandler = errorHandler;
+            this.#failedRequestHandler = failedRequestHandler;
+            this.#errorHandler = errorHandler;
             if (requestHandlerTimeoutSecs) {
-                this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+                this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
             }
             else {
-                this.requestHandlerTimeoutMillis = 60_000;
+                this.#requestHandlerTimeoutMillis = 60_000;
             }
             this.retryOnBlocked = retryOnBlocked;
-            this.respectRobotsTxtFile = respectRobotsTxtFile;
-            this.onSkippedRequest = onSkippedRequest;
+            this.#respectRobotsTxtFile = respectRobotsTxtFile;
+            // The cast undoes ow's assertion signature, which mangles `boolean | object` unions.
+            const transactionalStorageOption = transactionalStorage;
+            this.#transactionalStorageEnabled = transactionalStorageOption !== false;
+            this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
+            this.#onSkippedRequest = onSkippedRequest;
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
                 serviceLocator.getConfiguration().internalTimeoutMillis ??
-                    Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
-            this.maxRequestRetries = maxRequestRetries;
-            this.maxCrawlDepth = maxCrawlDepth;
-            this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
-            this.stats = new Statistics({
+                    Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
+            this.#maxRequestRetries = maxRequestRetries;
+            this.#maxCrawlDepth = maxCrawlDepth;
+            this.#statisticsDep = OwnedOrInjected.resolve(statistics, 
+            // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
+            // only be satisfied by an injected instance carrying the matching `state`, so this branch does
+            // not run in that case - hence the cast.
+            () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
-                id: this.identity.id,
-                ...statisticsOptions,
-            });
+                id: this.#identity.id,
+            }));
             if (sessionPool && proxyConfiguration) {
                 this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
                     'The `proxyConfiguration` is ignored - sessions from the supplied pool keep whatever ' +
                     '`proxyInfo` they were created with. Configure proxies on the pool instead, ' +
                     'e.g. via `addSession({ proxyInfo })` or a custom `createSessionFunction`.');
             }
-            this.sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
+            this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
                 createSessionFunction: async (opts) => new Session({
                     ...opts?.sessionOptions,
-                    proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
+                    proxyInfo: this.proxyConfiguration
+                        ? await this.proxyConfiguration.newProxyInfo(opts?.sessionOptions?.proxyInfo)
+                        : opts?.sessionOptions?.proxyInfo,
                 }),
             }));
             this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
             const maxSignedInteger = 2 ** 31 - 1;
-            if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
-                this.log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
+            if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
+                this.log.warning(`requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
                     ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
-                this.requestHandlerTimeoutMillis = maxSignedInteger;
+                this.#requestHandlerTimeoutMillis = maxSignedInteger;
             }
             this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
-            this.maxRequestsPerCrawl = maxRequestsPerCrawl;
-            const isMaxPagesExceeded = () => this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;
+            this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
+            const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.#handledRequestsCount;
             // eslint-disable-next-line prefer-const
             let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
             // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
@@ -363,82 +454,74 @@ export class BasicCrawler {
                     const source = this.requestManager;
                     if (!source)
                         throw new Error('Request provider is not initialized!');
-                    const request = await this.resolveRequest();
-                    if (!request || this.delayRequest(request, source)) {
+                    const request = await this.#resolveRequest();
+                    if (!request) {
                         return;
                     }
                     // Started here, rather than in `handleRequest`, so that a failure during context pipeline
                     // initialization (e.g. a browser page timing out before the request handler ever runs) is
-                    // still accounted for by `failJob` below - which is a no-op without a matching `startJob`.
-                    this.stats.startJob(request.id || request.uniqueKey);
+                    // still accounted for by `recordRequestFailure` below - which is a no-op without a
+                    // matching `recordRequestStart`.
+                    this.statistics.recordRequestStart(request.id || request.uniqueKey);
                     const crawlingContext = { request };
                     try {
+                        // The transaction spans the whole pipeline call, covering the navigation hooks
+                        // and `extendContext` too; `handleRequest` commits it and `handleRequestFailure` rolls it back.
+                        await this.runInStorageTransaction(async () => 
                         // Navigation, the navigation hooks and the request handler are timed individually, but the
                         // phases between them are not, so a request could still get stuck indefinitely. This is the
                         // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
-                        await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
-                            .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request)));
+                        await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), (error) => this.#handleRequestFailure(error, crawlingContext, source, request))));
                     }
                     catch (error) {
-                        // ContextPipelineInterruptedError means the request was intentionally skipped
-                        // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
-                        if (error instanceof ContextPipelineInterruptedError) {
-                            this.stats.discardJob(request.id || request.uniqueKey);
-                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                            return;
-                        }
-                        // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
-                        // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`
-                        // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.
-                        const isPipelineError = error instanceof ContextPipelineInitializationError ||
-                            error instanceof SessionError ||
-                            error instanceof TimeoutError;
-                        if (isPipelineError) {
-                            const unwrappedError = this.unwrapError(error);
+                        // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
+                        // take the normal error flow. Anything else is fatal.
+                        if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
+                            const unwrappedError = this.#unwrapError(error);
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
-                            // SessionError already retired the session in `requestFunctionErrorHandler`;
-                            // skip `markBad` to avoid double-counting usage/error score.
-                            if (!(unwrappedError instanceof SessionError)) {
+                            if (!this.#errorAbsolvesSession(unwrappedError)) {
                                 crawlingContext.session?.markBad();
                             }
                             return;
                         }
-                        throw this.unwrapError(error);
-                    }
-                    finally {
-                        // Run request-scoped deferred cleanups only after the whole request lifecycle - including the user's error handler - has finished.
-                        const deferredCleanup = crawlingContext[deferredCleanupKey] ?? [];
-                        await Promise.all(deferredCleanup.map((fn) => fn().catch((cleanupError) => this.log.debug('Error in deferred cleanup', { error: cleanupError }))));
+                        throw this.#unwrapError(error);
                     }
                 },
                 isTaskReadyFunction: async () => {
                     if (isMaxPagesExceeded()) {
-                        this.logOncePerRun('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
-                            `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
+                        this.#liveRun?.logOnce('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
+                            `${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
                         return false;
                     }
-                    if (this.unexpectedStop) {
-                        this.logOncePerRun('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
+                    if (this.#run?.stopRequested) {
+                        this.#liveRun?.logOnce('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
                             'Ongoing requests will be allowed to complete.');
                         return false;
                     }
-                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this.isTaskReadyFunction();
+                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this.#isTaskReadyFunction();
                 },
                 isFinishedFunction: async () => {
                     if (isMaxPagesExceeded()) {
-                        this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` +
+                        this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +
                             'and all requests that were in progress at that time have now finished. ' +
-                            `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
+                            `In total, the crawler processed ${this.#handledRequestsCount} requests and will shut down.`);
                         return true;
                     }
-                    if (this.unexpectedStop) {
+                    if (this.#run?.stopRequested) {
                         this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
                         return true;
                     }
+                    // `maybeFinish()` calls this only once nothing is in flight (`autoscaled_pool.ts`) - the point
+                    // where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
+                    // and the only place where throwing does not abandon requests mid-processing.
+                    const state = await this.requestManager?.checkReadiness();
+                    // Under `keepAlive`, outliving a domain that will not let us through is the whole point.
+                    if (state?.status === 'stalled' && !keepAlive) {
+                        throw new PersistentRateLimitError(`Giving up: ${state.reason}`);
+                    }
                     const isFinished = isFinishedFunction
                         ? await isFinishedFunction()
-                        : await this.defaultIsFinishedFunction();
+                        : state === undefined || state.status === 'finished';
                     if (isFinished) {
                         const reason = isFinishedFunction
                             ? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
@@ -449,11 +532,14 @@ export class BasicCrawler {
                 },
                 log: this.log,
             };
-            this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
-            this.resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
+            this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
+            this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
                 minConcurrency,
                 maxConcurrency,
                 maxTasksPerMinute: maxRequestsPerMinute,
+                // Spread conditionally - an explicit `undefined` would clobber a subclass default, see
+                // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
+                ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
                 log: this.log,
             }));
         }
@@ -463,7 +549,7 @@ export class BasicCrawler {
     }
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
@@ -479,7 +565,7 @@ export class BasicCrawler {
      * @returns `true` if the status code is considered an error, `false` otherwise
      */
     isErrorStatusCode(status) {
-        const excludeError = this.ignoreHttpErrorStatusCodes.has(status);
+        const excludeError = this.#ignoreHttpErrorStatusCodes.has(status);
         const includeError = this.additionalHttpErrorStatusCodes.has(status);
         return (status >= 500 && !excludeError) || includeError;
     }
@@ -487,35 +573,21 @@ export class BasicCrawler {
      * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
      * This handles base context creation, session resolution, and context helpers.
      */
-    buildBasicContextPipeline() {
+    #buildBasicContextPipeline() {
         return ContextPipeline.create()
-            .compose({ action: this.checkRobotsTxt.bind(this) })
-            .compose({ action: (context) => this.createBaseContext(context) })
-            .compose({ action: this.resolveSession.bind(this) })
-            .compose({ action: this.createContextHelpers.bind(this) });
+            .compose(this.#checkRobotsTxt.bind(this))
+            .compose(this.#createBaseContext.bind(this))
+            .compose(this.#resolveSession.bind(this))
+            .compose(this.#createContextHelpers.bind(this));
     }
-    async checkRobotsTxt({ request }) {
-        if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
+    async #checkRobotsTxt({ request }) {
+        if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {
             this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
-            request.state = RequestState.SKIPPED;
-            request.noRetry = true;
-            await this.handleSkippedRequest({
-                url: request.url,
-                reason: 'robotsTxt',
-            });
-            throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);
+            throw new ContextPipelineInterruptedError('robotsTxt');
         }
         return {};
     }
-    /**
-     * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
-     * Subclasses should override this to add their own pipeline stages.
-     */
-    buildContextPipeline() {
-        return ContextPipeline.create();
-    }
-    createBaseContext(context) {
-        const deferredCleanup = [];
+    #createBaseContext(context, onCleanup) {
         return {
             id: cryptoRandomObjectId(10),
             log: this.log,
@@ -523,7 +595,23 @@ export class BasicCrawler {
             useState: this.useState.bind(this),
             getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
             registerDeferredCleanup: (cleanup) => {
-                deferredCleanup.push(cleanup);
+                onCleanup(async () => {
+                    try {
+                        await cleanup();
+                    }
+                    catch (error) {
+                        this.log.debug('Error in deferred cleanup', { error });
+                    }
+                });
+            },
+            afterStorageCommit: (callback) => {
+                const transaction = currentStorageTransaction();
+                if (transaction === undefined) {
+                    throw new Error('afterStorageCommit() needs an active storage transaction, and there is none. ' +
+                        "With `transactionalStorage: false` the request's writes are applied as they are " +
+                        'made, so they throw at the call site instead of at commit time.');
+                }
+                transaction.afterCommit(callback);
             },
             extendTimeout: (secs) => {
                 const extraMillis = secs * 1000;
@@ -536,44 +624,54 @@ export class BasicCrawler {
                 if (context[navigationDeadlineKey] !== undefined) {
                     context[navigationDeadlineKey] += extraMillis;
                 }
+                // Extension failure must not fail the request that asked for more time.
+                this.requestManager?.extendRequestProcessingTimeSecs?.(context.request, secs)?.catch((error) => {
+                    this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
+                });
+            },
+            skipRequest: (message) => {
+                this.log.debug(`Skipping request ${context.request.url} (${context.request.id})${message ? `: ${message}` : ''}`);
+                throw new ContextPipelineInterruptedError('manual', message);
             },
-            [deferredCleanupKey]: deferredCleanup,
         };
     }
-    async resolveRequest() {
-        const request = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-        // Reset loadedUrl so an old one is not carried over to retries.
-        if (request) {
-            request.loadedUrl = undefined;
+    /**
+     * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
+     * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
+     * `markRequestAsHandled`, so nothing observes the difference.
+     */
+    async #resolveRequest() {
+        const fetched = await this.#timeoutAndRetry(this.#fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        if (!fetched) {
+            return null;
         }
+        const request = CrawlingRequest.fromSchema(fetched);
+        // Reset loadedUrl so an old one is not carried over to retries.
+        request.loadedUrl = undefined;
         return request;
     }
-    async resolveSession({ request }) {
-        const session = await this.timeoutAndRetry(async () => {
+    async #resolveSession({ request }) {
+        const session = await this.#timeoutAndRetry(async () => {
             const existingSession = await this.sessionPool.getSession(request.sessionId);
             if (!existingSession) {
-                throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
+                throw new MissingSessionError(request.sessionId);
             }
             return existingSession;
         }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
         return { session, proxyInfo: session?.proxyInfo };
     }
-    async createContextHelpers({ request, session }) {
-        const enqueueLinksWrapper = async (options) => {
-            const requestManager = await this.getRequestManager();
-            return await this.enqueueLinksWithCrawlDepth(options, request, requestManager);
-        };
+    async #createContextHelpers({ request, session }) {
         const addRequests = async (requests, options = {}) => {
             const newCrawlDepth = request.crawlDepth + 1;
             const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
-            await this.addRequests(requestsGenerator, options);
+            return await this.addRequests(requestsGenerator, options);
         };
         const sendRequest = createSendRequest(this.httpClient, request, session);
-        return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };
+        return { addRequests, sendRequest };
     }
-    buildFinalContextPipeline() {
-        const subclassPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??
-            this.buildContextPipeline());
+    #buildFinalContextPipeline() {
+        const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
+            ContextPipeline.create());
         // `extendContext` runs *before* the subclass navigation pipeline (which includes the
         // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
         // request handler alike. The trade-off is that `extendContext` cannot access
@@ -584,30 +682,25 @@ export class BasicCrawler {
         // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
         // casts below are sound because `buildFinalContextPipeline` is declared to return the fully
         // resolved `ExtendedContext` (= `Context & ContextExtension`).
-        const { extendContext } = this.contextPipelineOptions;
+        const { extendContext } = this.#contextPipelineOptions;
         let contextPipeline;
         if (extendContext !== undefined) {
             contextPipeline = ContextPipeline.create()
-                .compose({ action: async (context) => await extendContext(context) })
+                .compose(async (context) => await extendContext(context))
                 .chain(subclassPipeline);
         }
         else {
             contextPipeline = subclassPipeline;
         }
-        contextPipeline = contextPipeline.compose({
-            action: async (context) => {
-                const { request } = context;
-                if (request && !this.requestMatchesEnqueueStrategy(request)) {
-                    // eslint-disable-next-line dot-notation
-                    const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
-                    this.log.debug(message);
-                    request.noRetry = true;
-                    request.state = RequestState.SKIPPED;
-                    await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
-                    throw new ContextPipelineInterruptedError(message);
-                }
-                return context;
-            },
+        contextPipeline = contextPipeline.compose(async (context) => {
+            const { request } = context;
+            if (request && !this.#requestMatchesEnqueueStrategy(request)) {
+                // eslint-disable-next-line dot-notation
+                const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
+                this.log.debug(message);
+                throw new ContextPipelineInterruptedError('redirect');
+            }
+            return context;
         });
         return contextPipeline;
     }
@@ -618,7 +711,7 @@ export class BasicCrawler {
      * @param error The error to check.
      */
     isProxyError(error) {
-        return ROTATE_PROXY_ERRORS.some((x) => this._getMessageFromError(error)?.includes(x));
+        return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error).includes(x));
     }
     /**
      * Sets the status message for the current crawler run.
@@ -631,24 +724,26 @@ export class BasicCrawler {
      */
     setStatusMessage(message, options = {}) {
         const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
-        this.log.logWithLevel(LogLevel[options.level ?? 'DEBUG'], message, data);
+        // Each allowed level has its own method on the logger, so this goes through them rather than through
+        // `logWithLevel`, which is abstract and therefore cannot be instrumented.
+        this.log[{ DEBUG: 'debug', INFO: 'info', WARNING: 'warning', ERROR: 'error' }[options.level ?? 'DEBUG']](message, data);
         // Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can
         // subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.
         // Setting the status message is not a storage concern, so we intentionally don't route it
         // through the storage client anymore.
-        serviceLocator.getEventManager().emit("statusMessage" /* EventType.STATUS_MESSAGE */, {
-            crawlerId: this.identity.id,
+        serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
+            crawlerId: this.#identity.id,
             message,
             isStatusMessageTerminal: options.isStatusMessageTerminal,
             level: options.level,
         });
     }
-    getPeriodicLogger() {
-        let previousState = { ...this.stats.state };
+    #getPeriodicLogger() {
+        let previousState = { ...this.statistics.state };
         const getOperationMode = () => {
-            const { requestsFailed } = this.stats.state;
+            const { requestsFailed } = this.statistics.state;
             const { requestsFailed: previousRequestsFailed } = previousState;
-            previousState = { ...this.stats.state };
+            previousState = { ...this.statistics.state };
             const failedDelta = requestsFailed - previousRequestsFailed;
             if (failedDelta > 0) {
                 return { mode: 'ERROR', failedDelta };
@@ -659,16 +754,16 @@ export class BasicCrawler {
             const { mode: operationMode, failedDelta } = getOperationMode();
             let message;
             if (operationMode === 'ERROR') {
-                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
+                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.#statusMessageLoggingInterval} seconds.`;
             }
             else {
                 const total = await this.requestManager?.getTotalCount();
-                message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
+                message = `Crawled ${this.statistics.state.requestsSucceeded}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
-            if (this.statusMessageCallback) {
-                await this.statusMessageCallback({
+            if (this.#statusMessageCallback) {
+                await this.#statusMessageCallback({
                     crawler: this,
-                    state: this.stats.state,
+                    state: this.statistics.state,
                     previousState,
                     message,
                 });
@@ -676,7 +771,7 @@ export class BasicCrawler {
             }
             this.setStatusMessage(message);
         };
-        const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);
+        const interval = setInterval(log, this.#statusMessageLoggingInterval * 1e3);
         return { log, stop: () => clearInterval(interval) };
     }
     /**
@@ -687,111 +782,141 @@ export class BasicCrawler {
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
      *
+     * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
+     * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
+     * if that is what you want.
+     *
      * @param [requests] The requests to add.
-     * @param [options] Options for the request queue.
+     * @param [options] Options for adding the initial requests.
      */
     async run(requests, options) {
-        if (this.running) {
-            throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
-        }
-        const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
-        if (this.hasFinishedBefore) {
-            // When executing the run method for the second time explicitly,
-            // we need to purge the RQ to allow processing the same requests again — this is important so users can
-            // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
-            // ignored — as a failed request is still handled.
-            // By default (`purgeRequestQueue` unset), only the queue we opened ourselves is purged.
-            // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
-            // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
-            const shouldPurge = purgeRequestQueue !== false;
-            const managerToPurge = this.ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
-            if (managerToPurge?.purge && shouldPurge) {
-                await managerToPurge.purge();
+        // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
+        return withDirectStorageAccess(async () => timeoutStorage.exit(async () => {
+            if (this.running) {
+                throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
             }
-            this.stats.reset();
-            await this.stats.resetStore();
-            await this.sessionPoolDep.ifOwned((pool) => pool.resetStore());
-        }
-        this.unexpectedStop = false;
-        this.running = true;
-        this.loggedPerRun.clear();
-        await purgeDefaultStorages({
-            onlyPurgeOnce: true,
-            storageBackend: serviceLocator.getStorageBackend(),
-            configuration: serviceLocator.getConfiguration(),
-        });
-        if (requests) {
-            await this.addRequests(requests, addRequestsOptions);
-        }
-        try {
-            await this._init();
-            await this.stats.startCapturing();
-        }
-        catch (error) {
-            // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
-            await this.teardown().catch((teardownError) => {
-                this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
+            if (this.#hasFinishedBefore) {
+                // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+                await this.#statisticsDep.ifOwned(async (stats) => {
+                    stats.reset();
+                    await stats.resetStore();
+                });
+                // A session pool the crawler built follows the same blank-slate rule as its statistics.
+                await this.#sessionPoolDep.ifOwned(async (pool) => {
+                    pool.reset();
+                    await pool.resetStore();
+                });
+            }
+            // An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
+            // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
+            // run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
+            this.#concurrencySystem = this.#resolveConcurrencySystem();
+            const run = new CrawlerRun({
+                log: this.log,
+                concurrencySystem: this.#concurrencySystem.value,
+                taskLoopOptions: this.#taskLoopOptions,
+                consumer: this.#identity,
             });
-            // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
-            this.running = false;
-            throw error;
-        }
-        const periodicLogger = this.getPeriodicLogger();
-        this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
-        const sigintHandler = async () => {
-            this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
-            await this.pauseOnMigration();
-            await this.autoscaledPool.abort();
-        };
-        // Attach a listener to handle migration and aborting events gracefully.
-        const boundPauseOnMigration = this.pauseOnMigration.bind(this);
-        process.once('SIGINT', sigintHandler);
-        const eventManager = serviceLocator.getEventManager();
-        eventManager.on("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
-        eventManager.on("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
-        let stats = {};
-        try {
-            await this.autoscaledPool.run();
-        }
-        finally {
-            await this.teardown();
-            await this.stats.stopCapturing();
-            process.off('SIGINT', sigintHandler);
-            eventManager.off("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
-            eventManager.off("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
-            const finalStats = this.stats.calculate();
-            stats = {
-                requestsFinished: this.stats.state.requestsFinished,
-                requestsFailed: this.stats.state.requestsFailed,
-                retryHistogram: this.stats.requestRetryHistogram,
-                ...finalStats,
-            };
-            this.log.info('Final request statistics:', stats);
-            if (this.stats.errorTracker.total !== 0) {
-                const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
-                this.log.info(`Error analysis:`, {
-                    totalErrors: this.stats.errorTracker.total,
-                    uniqueErrors: this.stats.errorTracker.getUniqueErrorCount(),
-                    mostCommonErrors: this.stats.errorTracker.getMostPopularErrors(3).map(prettify),
+            this.#run = run;
+            try {
+                await purgeDefaultStorages({
+                    onlyPurgeOnce: true,
+                    storageBackend: serviceLocator.getStorageBackend(),
+                    configuration: serviceLocator.getConfiguration(),
                 });
+                if (requests) {
+                    await this.addRequests(requests, options);
+                }
+                // An injected governor is the caller's to run; an owned one is started here and stopped again
+                // below if the startup fails. Either way the task loop rejects one that is not running.
+                await this.#concurrencySystem.ifOwned((system) => system.start());
+                await this.init();
+                await this.statistics.startCapturing();
+            }
+            catch (error) {
+                // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
+                await this.teardown().catch((teardownError) => {
+                    this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
+                });
+                // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
+                await run.finish();
+                await this.#concurrencySystem.ifOwned((system) => system.stop());
+                throw error;
+            }
+            const periodicLogger = this.#getPeriodicLogger();
+            this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
+            const sigintHandler = async () => {
+                this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
+                await this.#pauseOnMigration();
+                await run.abort();
+            };
+            // Attach a listener to handle migration and aborting events gracefully.
+            const boundPauseOnMigration = this.#pauseOnMigration.bind(this);
+            process.once('SIGINT', sigintHandler);
+            const eventManager = serviceLocator.getEventManager();
+            eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
+            eventManager.on(EventType.ABORTING, boundPauseOnMigration);
+            let stats = {};
+            try {
+                await run.dispatchRequests();
             }
-            const client = serviceLocator.getStorageBackend();
-            if (client.teardown) {
-                let finished = false;
-                setTimeout(() => {
-                    if (!finished) {
-                        this.log.info('Waiting for the storage to write its state to file system.');
+            finally {
+                await this.statistics.stopCapturing();
+                await this.teardown();
+                process.off('SIGINT', sigintHandler);
+                eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
+                eventManager.off(EventType.ABORTING, boundPauseOnMigration);
+                const finalStats = this.statistics.calculate();
+                stats = {
+                    requestsSucceeded: this.statistics.state.requestsSucceeded,
+                    requestsFailed: this.statistics.state.requestsFailed,
+                    retryHistogram: this.statistics.requestRetryHistogram,
+                    ...finalStats,
+                };
+                this.log.info('Final request statistics:', stats);
+                // A crawl that did nothing while the manager holds only handled requests is a mistake whoever
+                // handled them - this run, another crawler on the same queue, or a previous process. Starting
+                // against handled requests is not: that is what resuming a crawl looks like.
+                if (stats.requestsSucceeded + stats.requestsFailed === 0) {
+                    // Never let the diagnostic itself break the run.
+                    const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
+                    if (alreadyHandled > 0) {
+                        this.log.warningOnce('This crawl processed no requests - the request manager holds ' +
+                            `${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` +
+                            'already handled, and a failed request counts as handled too. Nothing ' +
+                            'empties a queue between runs, so to crawl them again, purge it ' +
+                            '(`await queue.purge()`) or use a fresh one (e.g. ' +
+                            '`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' +
+                            'crawler instance.');
                     }
-                }, 1000);
-                await client.teardown();
-                finished = true;
+                }
+                if (this.statistics.errorTracker.total !== 0) {
+                    const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
+                    this.log.info(`Error analysis:`, {
+                        totalErrors: this.statistics.errorTracker.total,
+                        uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
+                        mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
+                    });
+                }
+                const client = serviceLocator.getStorageBackend();
+                if (client.teardown) {
+                    let finished = false;
+                    setTimeout(() => {
+                        if (!finished) {
+                            this.log.info('Waiting for the storage to write its state to file system.');
+                        }
+                    }, 1000);
+                    await client.teardown();
+                    finished = true;
+                }
+                periodicLogger.stop();
+                this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsSucceeded} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
+                await run.finish();
+                await this.#concurrencySystem.ifOwned((system) => system.stop());
+                this.#hasFinishedBefore = true;
             }
-            periodicLogger.stop();
-            this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
-            this.running = false;
-            this.hasFinishedBefore = true;
-        }
-        return stats;
+            return stats;
+        }));
     }
     /**
      * Gracefully stops the current run of the crawler.
@@ -801,11 +926,12 @@ export class BasicCrawler {
      * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
     stop(reason = 'The crawler has been gracefully stopped.') {
-        if (this.unexpectedStop) {
+        const run = this.#liveRun;
+        if (run === undefined) {
+            this.log.warning('Cannot stop a crawler that is not running.');
             return;
         }
-        this.log.info(reason);
-        this.unexpectedStop = true;
+        run.stop(reason);
     }
     /**
      * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
@@ -816,39 +942,41 @@ export class BasicCrawler {
      * throughout, since a shared one may still be serving other crawlers.
      */
     async pause(timeoutSecs) {
-        if (!this.autoscaledPool) {
+        const run = this.#dispatchingRun;
+        if (run === undefined) {
             this.log.warning('Cannot pause a crawler that is not running.');
             return;
         }
-        await this.autoscaledPool.pause(timeoutSecs);
+        await run.pause(timeoutSecs);
     }
     /**
      * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
      * again. A no-op on a crawler that is not paused.
      */
     resume() {
-        if (!this.autoscaledPool) {
+        const run = this.#dispatchingRun;
+        if (run === undefined) {
             this.log.warning('Cannot resume a crawler that is not running.');
             return;
         }
-        this.autoscaledPool.resume();
+        run.resume();
     }
     /**
      * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
      * if none has been configured or opened yet.
      */
     async getRequestManager() {
-        if (!this.requestManager) {
-            this.requestManager = await this.openOwnedRequestQueue();
+        if (!this.#requestManager) {
+            this.#requestManager = await this.#openOwnedRequestQueue();
         }
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
         // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
         // but guard so we do not re-issue it on every call.
-        if (!this.requestManagerTimeoutsApplied) {
-            this.requestManagerTimeoutsApplied = true;
-            await this.applyRequestManagerTimeouts(this.requestManager);
+        if (!this.#requestManagerTimeoutsApplied) {
+            this.#requestManagerTimeoutsApplied = true;
+            await this.#applyRequestManagerTimeouts(this.#requestManager);
         }
-        return this.requestManager;
+        return this.#requestManager;
     }
     /**
      * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
@@ -858,16 +986,13 @@ export class BasicCrawler {
         return this.getRequestManager();
     }
     /**
-     * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
-     * crawler-owned queue (so it gets purged between repeated `run()` calls).
-     * @private
+     * Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
      */
-    async openOwnedRequestQueue() {
+    async #openOwnedRequestQueue() {
         // The first crawler instance uses the default queue (null identifier);
         // subsequent instances get their own queue via a unique alias so they don't collide.
-        const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
-        const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
-        return this.ownedRequestQueue.set(requestQueue);
+        const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };
+        return RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
     }
     /**
      * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
@@ -876,12 +1001,12 @@ export class BasicCrawler {
      * request from being handed out a second time while it is still being processed — and it works
      * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
      */
-    async applyRequestManagerTimeouts(requestManager) {
+    async #applyRequestManagerTimeouts(requestManager) {
         // A router route may hold a request for longer than the crawler's own timeout, and we cannot know
         // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
         // raise-only, so erring high here is safe.
         const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
-        const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
+        const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
         await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
     }
     /**
@@ -891,7 +1016,7 @@ export class BasicCrawler {
      * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
      * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
      */
-    async validateRequestUserData(source) {
+    async #validateRequestUserData(source) {
         if (typeof source === 'string') {
             return;
         }
@@ -912,12 +1037,12 @@ export class BasicCrawler {
     }
     async useState(defaultValue = {}) {
         const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
-        if (this.identity.hasExplicitId) {
-            const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
+        if (this.#identity.hasExplicitId) {
+            const stateKey = `${BasicCrawler.#CRAWLEE_STATE_KEY}_${this.#identity.id}`;
             return kvs.getAutoSavedValue(stateKey, defaultValue);
         }
-        BasicCrawler.useStateAnonymousIndices.add(this.identity.instanceIndex);
-        if (BasicCrawler.useStateAnonymousIndices.size > 1) {
+        BasicCrawler.#useStateAnonymousIndices.add(this.#identity.instanceIndex);
+        if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
             serviceLocator
                 .getLogger()
                 .warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
@@ -925,33 +1050,35 @@ export class BasicCrawler {
                 'To fix this, provide a unique `id` option to each crawler instance. \n' +
                 'Example: new BasicCrawler({ id: "my-crawler-1", ... })');
         }
-        return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
+        return kvs.getAutoSavedValue(BasicCrawler.#CRAWLEE_STATE_KEY, defaultValue);
     }
-    async getPendingRequestCountApproximation() {
+    async #getPendingRequestCountApproximation() {
         return (await this.requestManager?.getPendingCount()) ?? 0;
     }
-    async calculateEnqueuedRequestLimit(explicitLimit) {
-        if (this.maxRequestsPerCrawl === undefined) {
+    async #calculateEnqueuedRequestLimit(explicitLimit) {
+        if (this.#maxRequestsPerCrawl === undefined) {
             return explicitLimit;
         }
-        const limit = Math.max(0, this.maxRequestsPerCrawl - this.handledRequestsCount - (await this.getPendingRequestCountApproximation()));
+        const limit = Math.max(0, this.#maxRequestsPerCrawl -
+            this.#handledRequestsCount -
+            (await this.#getPendingRequestCountApproximation()));
         return Math.min(limit, explicitLimit ?? Infinity);
     }
-    async handleSkippedRequest(options) {
-        if (options.reason === 'limit') {
-            this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
-                `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
-        }
-        if (options.reason === 'depth') {
-            this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
-        }
-        await this.onSkippedRequest?.(options);
-    }
-    logOncePerRun(key, message) {
-        if (!this.loggedPerRun.has(key)) {
-            this.log.info(message);
-            this.loggedPerRun.add(key);
-        }
+    async #handleSkippedRequest(options) {
+        // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
+        // transaction scope, which rolls back - so the skip bookkeeping must write directly.
+        await withDirectStorageAccess(async () => {
+            // Enqueueing happens outside a run too - `addRequests()` before the first `run()` - so these dedupe
+            // against the logger for the crawler's lifetime rather than per run.
+            if (options.reason === 'limit') {
+                this.log.info('The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
+                    `${this.#maxRequestsPerCrawl} requests and no further requests will be added.`, undefined, { once: true });
+            }
+            if (options.reason === 'depth') {
+                this.log.info(`The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`, undefined, { once: true });
+            }
+            await this.#onSkippedRequest?.(options);
+        });
     }
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
@@ -959,6 +1086,11 @@ export class BasicCrawler {
      * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
      * the `waitForAllRequestsToBeAdded` promise you get in the response object.
      *
+     * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
+     * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
+     * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
+     * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
+     *
      * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
      *
      * @param requests The requests to add
@@ -966,62 +1098,118 @@ export class BasicCrawler {
      */
     async addRequests(requests, options = {}) {
         await this.getRequestManager();
-        const requestLimit = await this.calculateEnqueuedRequestLimit();
-        const skippedBecauseOfRobots = new Set();
-        const skippedBecauseOfMaxCrawlDepth = new Set();
-        const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
-        const maxCrawlDepth = this.maxCrawlDepth;
-        const validateRequestUserData = this.validateRequestUserData.bind(this);
-        ow(requests, ow.object
-            .is((value) => isIterable(value) || isAsyncIterable(value))
-            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
+        if (!isIterable(requests) && !isAsyncIterable(requests)) {
+            throw new Error(`Expected an iterable or async iterable, got ${getObjectType(requests)}`);
+        }
+        parseArgument(options, addRequestsOptionsSchema, 'EnqueueUrlsOptions');
+        // `label`/`userData` apply to every request this call produces, so a single upfront validation
+        // against the label's schema covers them all and fails the whole call fast, rather than failing
+        // lazily once the generator below is drained. Skipped when neither is set - each item still gets
+        // its own per-item validation below, and validating an absent label/userData here would spuriously
+        // check them against a registered default-route schema.
+        if (options.label !== undefined || options.userData !== undefined) {
+            await this.#validateRequestUserData({ label: options.label, userData: options.userData });
+        }
+        const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);
+        const strategy = options.strategy ?? EnqueueStrategy.All;
+        const urlExcludePatternObjects = options.exclude?.length ? constructUrlPatternObjects(options.exclude) : [];
+        const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
+        // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
+        // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
+        const enqueueStrategyPatterns = options.baseUrl ? buildEnqueueStrategyPatterns(options.baseUrl, strategy) : [];
+        const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
+        const maxCrawlDepth = this.#maxCrawlDepth;
+        const validateRequestUserData = this.#validateRequestUserData.bind(this);
+        const allSkipped = [];
+        // A skipped source (which can carry arbitrary userData) is only retained if something reads it -
+        // otherwise the URL alone is enough to build the callback argument and to log with.
+        const hasSkippedRequestCallback = this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
+        const keepSkippedSource = (source) => (hasSkippedRequestCallback ? source : source.url);
+        const reportSkippedRequests = async () => {
+            const skippedRequests = allSkipped.splice(0);
+            if (skippedRequests.length === 0) {
+                return;
+            }
+            const skippedRobotsUrls = skippedRequests
+                .filter((s) => s.reason === 'robotsTxt')
+                .map(({ source }) => (typeof source === 'string' ? source : source.url));
+            if (skippedRobotsUrls.length > 0) {
+                this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
+            }
+            // Only log the limit message when an explicit `limit` was passed (not the internal
+            // `maxRequestsPerCrawl`-derived one), and only once per call.
+            if (options.limit !== undefined && skippedRequests.some((s) => s.reason === 'limit')) {
+                this.log.info(requestLimit === options.limit
+                    ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
+                    : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
+            }
+            await Promise.all(skippedRequests.map(async ({ source, reason }) => {
+                const args = createSkippedRequestArgs(source, reason);
+                await this.#handleSkippedRequest(args);
+                await options.onSkippedRequest?.(args);
+            }));
+        };
         async function* filteredRequests() {
             for await (const request of requests) {
-                const url = typeof request === 'string' ? request : request.url;
-                if (maxCrawlDepth !== undefined && request.crawlDepth > maxCrawlDepth) {
-                    skippedBecauseOfMaxCrawlDepth.add(url);
+                const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
+                if (!requestOptions) {
+                    continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
+                }
+                if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
+                    allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });
                     continue;
                 }
-                if (await isAllowedBasedOnRobotsTxtFile(url)) {
-                    await validateRequestUserData(request);
-                    yield request;
+                if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
+                    allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });
+                    continue;
                 }
-                else {
-                    skippedBecauseOfRobots.add(url);
+                const onSkippedByFilter = (opts) => allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });
+                // Filter by user patterns first (with exclude)...
+                let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
+                // ...then filter by the enqueue strategy (making this an AND check)
+                filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedByFilter);
+                if (filtered.length === 0) {
+                    continue;
+                }
+                let [finalOptions] = filtered;
+                if (options.transformRequestFunction) {
+                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }));
+                    if (transformed.length === 0) {
+                        continue;
+                    }
+                    [finalOptions] = transformed;
                 }
+                await validateRequestUserData(finalOptions);
+                yield new Request(finalOptions);
             }
         }
         const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
-            ...options,
+            forefront: options.forefront,
+            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
+            batchSize: options.batchSize,
+            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
             maxNewRequests: requestLimit,
         });
-        // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit)
-        const skippedBecauseOfLimit = result.requestsOverLimit ?? [];
-        if (skippedBecauseOfRobots.size > 0) {
-            this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
-                skipped: [...skippedBecauseOfRobots],
-            });
-        }
-        if (skippedBecauseOfRobots.size > 0 ||
-            skippedBecauseOfLimit.length > 0 ||
-            skippedBecauseOfMaxCrawlDepth.size > 0) {
-            await Promise.all([...skippedBecauseOfRobots]
-                .map((url) => {
-                return this.handleSkippedRequest({ url, reason: 'robotsTxt' });
-            })
-                .concat(skippedBecauseOfLimit.map((request) => {
-                const url = typeof request === 'string' ? request : request.url;
-                return this.handleSkippedRequest({ url, reason: 'limit' });
-            }), [...skippedBecauseOfMaxCrawlDepth].map((url) => {
-                return this.handleSkippedRequest({ url, reason: 'depth' });
-            })));
-        }
-        return result;
+        // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
+        // explicit `limit` option)
+        for (const request of result.requestsOverLimit ?? []) {
+            allSkipped.push({ source: request, reason: 'limit' });
+        }
+        await reportSkippedRequests();
+        const waitForAllRequestsToBeAdded = result.waitForAllRequestsToBeAdded.then(async (addedRequests) => {
+            await reportSkippedRequests();
+            return addedRequests;
+        });
+        // Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
+        // when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
+        void waitForAllRequestsToBeAdded.catch(() => { });
+        return { ...result, waitForAllRequestsToBeAdded };
     }
     /**
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
      */
     async pushData(data, datasetIdentifier) {
+        tryCancel();
         const dataset = await this.getDataset(datasetIdentifier);
         return dataset.pushData(data);
     }
@@ -1067,6 +1255,7 @@ export class BasicCrawler {
                 const keys = options?.collectAllKeys
                     ? Array.from(new Set(items.flatMap(Object.keys)))
                     : Object.keys(items[0]);
+                const { stringify } = await import('csv-stringify/sync');
                 value = stringify([
                     keys,
                     ...items.map((item) => {
@@ -1074,13 +1263,13 @@ export class BasicCrawler {
                     }),
                 ]);
             }
-            await ensureDir(dirname(path));
+            await mkdir(dirname(path), { recursive: true });
             await writeFile(path, value);
             this.log.info(`Export to ${path} finished!`);
         }
         if (format === 'json') {
-            await ensureDir(dirname(path));
-            await writeJSON(path, items, { spaces: 4 });
+            await mkdir(dirname(path), { recursive: true });
+            await writeFile(path, `${JSON.stringify(items, null, 4)}\n`);
             this.log.info(`Export to ${path} finished!`);
         }
         return items;
@@ -1088,32 +1277,26 @@ export class BasicCrawler {
     /**
      * Initializes the crawler.
      */
-    async _init() {
+    async init() {
+        const run = this.#liveRun;
+        if (run === undefined) {
+            throw new Error('There is no crawl in progress - `run()` is the only thing that starts one.');
+        }
         const eventManager = serviceLocator.getEventManager();
         if (!eventManager.isInitialized()) {
             await eventManager.init();
-            this._closeEvents = true;
+            run.ownsEventManager = true;
         }
         // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
         // floored per request so it will not actually cut them short, but the configured value is then effectively
         // ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
         // navigation timeout only after `super()`.
-        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(undefined);
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(undefined);
         if (this.internalTimeoutMillis < phasesMillis) {
             this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
                 `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
                 `so it does not cut them short.`);
         }
-        // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
-        // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
-        // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
-        this.concurrencySystemDep = this.resolveConcurrencySystem();
-        await this.concurrencySystemDep.ifOwned((system) => system.start());
-        this.autoscaledPool = new AutoscaledPool({
-            ...this.taskLoopOptions,
-            concurrencySystem: this.concurrencySystemDep.value,
-            consumer: this.identity,
-        });
         await this.getRequestManager();
     }
     /**
@@ -1130,9 +1313,9 @@ export class BasicCrawler {
      * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
      * whichever is larger: the configured internal timeout, or this request's combined phase budget.
      */
-    async withRequestTimeout(crawlingContext, work) {
+    async #withRequestTimeout(crawlingContext, work) {
         const { request } = crawlingContext;
-        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(request.label);
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(request.label);
         const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
         await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
     }
@@ -1143,14 +1326,14 @@ export class BasicCrawler {
      * @param label The request's route label, or `undefined` for the default route / no specific request.
      * @param fallbackMillis Timeout to use when no route overrides it.
      */
-    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.requestHandlerTimeoutMillis) {
-        return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
+    #resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
+        return this.#getRouteTimeoutMillis(label) ?? fallbackMillis;
     }
     /**
      * The timeout the router route with the given label asked for, or `undefined` when it did not override one
      * (or the request handler is not a router at all).
      */
-    getRouteTimeoutMillis(label) {
+    #getRouteTimeoutMillis(label) {
         const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
         if (typeof getTimeoutSecs !== 'function') {
             return undefined;
@@ -1159,39 +1342,121 @@ export class BasicCrawler {
         return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
     }
     async runRequestHandler(crawlingContext) {
-        const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
+        const timeoutMillis = this.#resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
         await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
     }
+    /**
+     * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
+     * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
+     * normal return says nothing about success. `handleRequest` commits on success.
+     */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
+    async runInStorageTransaction(callback) {
+        if (!this.#transactionalStorageEnabled) {
+            return callback();
+        }
+        const transaction = createStorageTransaction({
+            policy: this.#storageWritePolicy,
+            commitTimeoutMillis: this.internalTimeoutMillis,
+        });
+        let threw = true;
+        try {
+            const result = await transaction.run(callback);
+            threw = false;
+            return result;
+        }
+        finally {
+            if (transaction.state === 'open') {
+                // `handleRequest` commits or `handleRequestFailure` rolls back on every normal path, so an open transaction on
+                // a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
+                // expected. Either way, discard the unvalidated writes; only the former is worth flagging.
+                if (!threw) {
+                    this.log.error('Internal error: a storage transaction was still open after the request pipeline ' +
+                        'returned normally. Its writes are being discarded. Please report this.');
+                }
+                transaction.rollback();
+            }
+            // Unconditional: `failed` is a terminal state that the branch above never reaches.
+            transaction.dispose();
+        }
+    }
     /**
      * Handles blocked request
      */
-    _throwOnBlockedRequest(statusCode) {
+    throwOnBlockedRequest(statusCode) {
         if (this.retryOnBlocked)
             return;
         if (this.blockedStatusCodes.has(statusCode)) {
             throw new SessionError(`Request blocked - received ${statusCode} status code.`);
         }
     }
-    async isAllowedBasedOnRobotsTxtFile(url) {
-        if (!this.respectRobotsTxtFile) {
+    async #isAllowedBasedOnRobotsTxtFile(url) {
+        if (!this.#respectRobotsTxtFile) {
             return true;
         }
         const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
-        const userAgent = typeof this.respectRobotsTxtFile === 'object' ? this.respectRobotsTxtFile?.userAgent : '*';
+        const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';
+        if (robotsTxtFile) {
+            const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
+            if (crawlDelay !== undefined) {
+                this.#applyCrawlDelay(url, crawlDelay);
+            }
+        }
         return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
     }
+    /**
+     * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
+     *
+     * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
+     * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
+     *  {@link RequestThrottledError} rather than treating the response as a blocked session.
+     */
+    recordDomainRateLimit(url, retryAfterHeader) {
+        if (this.requestManager?.recordPacingSignal({
+            reason: 'rateLimited',
+            url,
+            waitMs: parseRetryAfterHeader(retryAfterHeader) ?? undefined,
+        })) {
+            return true;
+        }
+        const domain = hostnameOrUrl(url);
+        this.log.warningOnce(`"${domain}" responded with HTTP 429 (Too Many Requests), but the crawler's request manager does not ` +
+            'pace that domain, so the response is handled like any other, with no per-domain delay. Set ' +
+            `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
+            '`requestManager`, to honour `Retry-After` and apply exponential backoff instead.');
+        return false;
+    }
+    /** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
+    #applyCrawlDelay(url, delaySeconds) {
+        // robots.txt is per-origin; `hostname` is the closest pacing scope and errs wide (http and https to one
+        // host share a clock).
+        if (this.requestManager?.recordPacingSignal({
+            reason: 'minInterval',
+            url,
+            intervalMs: delaySeconds * 1000,
+            scope: 'hostname',
+        })) {
+            return;
+        }
+        const domain = hostnameOrUrl(url);
+        this.log.warningOnce(`robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but the crawler's request ` +
+            'manager does not pace that domain, so its requests will not be paced. Set ' +
+            `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
+            '`requestManager`.');
+    }
+    /** @internal */
     async getRobotsTxtFileForUrl(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return undefined;
         }
         try {
             const origin = new URL(url).origin;
-            const cachedRobotsTxtFile = this.robotsTxtFileCache.get(origin);
+            const cachedRobotsTxtFile = this.#robotsTxtFileCache.get(origin);
             if (cachedRobotsTxtFile) {
                 return cachedRobotsTxtFile;
             }
             const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
-            this.robotsTxtFileCache.add(origin, robotsTxtFile);
+            this.#robotsTxtFileCache.add(origin, robotsTxtFile);
             return robotsTxtFile;
         }
         catch (e) {
@@ -1199,10 +1464,10 @@ export class BasicCrawler {
             return undefined;
         }
     }
-    async pauseOnMigration() {
-        if (this.autoscaledPool) {
-            // if run wasn't called, this is going to crash
-            await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
+    async #pauseOnMigration() {
+        const run = this.#dispatchingRun;
+        if (run) {
+            await run.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
                 if (err.message.includes('running tasks did not finish')) {
                     this.log.error('The crawler was paused due to migration to another host, ' +
                         "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -1212,61 +1477,21 @@ export class BasicCrawler {
                 }
             });
         }
-        const requestManagerPersistPromise = (async () => {
-            // The request manager persists its read-only loader's state, if it has one that supports persistence
-            // (e.g. a tandem wrapping a `RequestList`). For a plain `RequestQueue`, this is a no-op.
-            if (this.requestManager?.persistState) {
-                if (await this.requestManager.isFinished())
-                    return;
-                await this.requestManager.persistState().catch((err) => {
-                    if (err.message.includes('Cannot persist state.')) {
-                        this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
-                            'invalid configuration. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
-                            'constructor to ensure your crawling state is persisted through host migrations and restarts.');
-                    }
-                    else {
-                        this.log.exception(err, 'An unexpected error occurred when the crawler ' +
-                            "attempted to persist its request list's state.");
-                    }
-                });
-            }
-        })();
-        await Promise.all([requestManagerPersistPromise, this.stats.persistState()]);
+        // Captures the statistics changed while draining above. No PERSIST_STATE event is tied to the drain:
+        // the periodic one is unrelated, and the one the platform emits on migration arrives before any of this.
+        await this.statistics.persistState?.();
     }
     /**
      * Fetches the next request to process from the underlying request provider.
      */
-    async fetchNextRequest() {
+    async #fetchNextRequest() {
         if (this.requestManager === undefined) {
             throw new Error(`fetchNextRequest called on an uninitialized crawler`);
         }
         return this.requestManager.fetchNextRequest();
     }
-    /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
-     */
-    delayRequest(request, source) {
-        const domain = getDomain(request.url);
-        if (!domain || !request) {
-            return false;
-        }
-        const now = Date.now();
-        const lastAccessTime = this.domainAccessedTime.get(domain);
-        if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {
-            this.domainAccessedTime.set(domain, now);
-            return false;
-        }
-        const delay = lastAccessTime + this.sameDomainDelayMillis - now;
-        this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
-        setTimeout(async () => {
-            this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
-            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
-        }, delay);
-        return true;
-    }
-    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async handleRequest(crawlingContext, requestSource, request) {
         // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
         // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
@@ -1274,107 +1499,89 @@ export class BasicCrawler {
         if (crawlingContext[timeoutExpiredKey]?.()) {
             return;
         }
-        const statisticsId = request.id || request.uniqueKey;
-        let isRequestLocked = true;
+        request.state = RequestState.REQUEST_HANDLER;
+        await this.runRequestHandler(crawlingContext);
+        // Commit *before* marking the request as handled, so a commit failure fails the request and
+        // it is retried. This also closes the transaction, so everything below passes through.
+        // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
+        await currentStorageTransaction()?.commit();
+        await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
+        // reclaim session if request finishes successfully
+        request.state = RequestState.DONE;
+        crawlingContext.session.markGood();
+    }
+    /**
+     * Handles a request that failed anywhere in the context pipeline - a middleware (e.g. navigation or its hooks)
+     * or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
+     * error handler has seen the context.
+     */
+    async #handleRequestFailure(rawError, crawlingContext, requestSource, request) {
+        // Roll back *before* any error handler runs - error handlers write to real storage precisely
+        // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
+        currentStorageTransaction()?.rollback();
+        const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
         try {
-            request.state = RequestState.REQUEST_HANDLER;
-            await this.runRequestHandler(crawlingContext);
-            await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-            isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
-            this.stats.finishJob(statisticsId, request.retryCount);
-            // reclaim session if request finishes successfully
-            request.state = RequestState.DONE;
-            crawlingContext.session.markGood();
-        }
-        catch (rawError) {
-            const err = this.unwrapError(rawError);
-            try {
-                request.state = RequestState.ERROR_HANDLER;
-                await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, crawlingContext, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                if (!(err instanceof CriticalError)) {
-                    isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
-                }
-                request.state = RequestState.DONE;
-            }
-            catch (secondaryError) {
-                const unwrappedSecondaryError = this.unwrapError(secondaryError);
-                if (!unwrappedSecondaryError.triggeredFromUserHandler &&
-                    // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
-                    !(unwrappedSecondaryError instanceof CriticalError)) {
-                    const apifySpecific = process.env.APIFY_IS_AT_HOME
-                        ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
-                        : '';
-                    this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
-                        `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
-                }
-                request.state = RequestState.ERROR;
-                throw unwrappedSecondaryError;
-            }
-            // decrease the session score if the request fails (but the error handler did not throw);
-            // skip when the error is a SessionError, which already retired the session
-            if (!(err instanceof SessionError)) {
-                crawlingContext.session.markBad();
+            const unwrappedError = this.#unwrapError(rawError);
+            // A failing or timed out `onSkippedRequest` makes this an ordinary failure, so the request gets retried.
+            const err = unwrappedError instanceof ContextPipelineInterruptedError
+                ? await this.#trySkipRequest(unwrappedError, request, requestSource)
+                : unwrappedError;
+            if (err) {
+                await this.#runErrorHandler(err, crawlingContext, requestSource, request);
             }
         }
         finally {
-            // Safety net - return the request to the queue if nobody managed to mark it as handled
-            // or reclaim it before (e.g. after a CriticalError). Reclaiming a request that is no longer
-            // in progress is a harmless no-op on the storage backend.
-            if (isRequestLocked && requestSource instanceof RequestQueue) {
-                try {
-                    await requestSource.reclaimRequest(request);
-                }
-                catch {
-                    // The request was never in progress, or could not be reclaimed. Either way it's fine.
-                }
-            }
+            resumeRequestTimeout?.();
         }
     }
-    /**
-     * Wrapper around the crawling context's `enqueueLinks` method:
-     * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
-     * @internal
-     */
-    async enqueueLinksWithCrawlDepth(options, request, requestManager) {
-        const transformRequestFunctionWrapper = (requestOptions) => {
-            requestOptions.crawlDepth = request.crawlDepth + 1;
-            if (this.maxCrawlDepth !== undefined && requestOptions.crawlDepth > this.maxCrawlDepth) {
-                // Setting `skippedReason` before returning `false` ensures that `reportSkippedRequests`
-                // reports `'depth'` as the reason (via `request.skippedReason ?? reason` fallback),
-                // rather than the generic `'transform'` reason.
-                requestOptions.skippedReason = 'depth';
-                return false;
+    async #runErrorHandler(err, context, requestSource, request) {
+        const retryCountBefore = request.retryCount;
+        try {
+            request.state = RequestState.ERROR_HANDLER;
+            await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            // The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
+            if (request.state !== RequestState.SKIPPED) {
+                request.state = RequestState.DONE;
             }
-            // After injecting the crawlDepth, we call the user-provided transform function, if there is one.
-            return options.transformRequestFunction?.(requestOptions) ?? requestOptions;
-        };
-        // Create a request-scoped callback that logs enqueueLimit once per request handler call
-        // Only log if an explicit limit was passed to enqueueLinks (not the internal maxRequestsPerCrawl-derived limit)
-        let loggedEnqueueLimitForThisRequest = false;
-        const onSkippedRequest = async (skippedOptions) => {
-            if (skippedOptions.reason === 'enqueueLimit') {
-                if (!loggedEnqueueLimitForThisRequest && options.limit !== undefined) {
-                    this.log.info(`Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`);
-                    loggedEnqueueLimitForThisRequest = true;
+        }
+        catch (secondaryError) {
+            const unwrappedSecondaryError = this.#unwrapError(secondaryError);
+            const timedOut = unwrappedSecondaryError instanceof TimeoutError;
+            request.state = RequestState.ERROR;
+            if (timedOut) {
+                // Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
+                // retry is counted, so that an error handler that always times out does not retry it forever.
+                this.log.warning(unwrappedSecondaryError.message);
+                if (request.retryCount === retryCountBefore) {
+                    request.retryCount++;
                 }
             }
-            await this.handleSkippedRequest(skippedOptions);
-        };
-        // `enqueueLinks` applies `options.label`/`options.userData` to every newly enqueued request, so a single
-        // validation against the label's schema covers them all (a no-op unless the router declares a schema).
-        await this.validateRequestUserData({ label: options.label, userData: options.userData });
-        return await enqueueLinks({
-            requestManager,
-            robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
-            respectRobotsTxtFile: this.respectRobotsTxtFile,
-            onSkippedRequest,
-            limit: await this.calculateEnqueuedRequestLimit(options.limit),
-            // Allow user options to override defaults set above ⤴
-            ...options,
-            transformRequestFunction: transformRequestFunctionWrapper,
-        });
+            else if (!(unwrappedSecondaryError instanceof CriticalError)) {
+                // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
+                const apifySpecific = process.env.APIFY_IS_AT_HOME
+                    ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
+                    : '';
+                this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
+                    `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
+            }
+            // Safety net - the error handler neither marked the request as handled nor reclaimed it (e.g. after a
+            // CriticalError). Reclaiming a request that is no longer in progress is a harmless no-op on the storage backend.
+            try {
+                await addTimeoutToPromise(async () => requestSource.reclaimRequest(request), this.internalTimeoutMillis, `Reclaiming request ${request.url} (${request.id}) timed out.`);
+            }
+            catch {
+                // The request was never in progress, or could not be reclaimed. Either way it's fine.
+            }
+            if (!timedOut) {
+                throw unwrappedSecondaryError;
+            }
+        }
+        // decrease the session score if the request fails (but the error handler did not throw);
+        // skip when the error is a SessionError, which already retired the session
+        if (!this.#errorAbsolvesSession(err)) {
+            context.session?.markBad();
+        }
     }
     /**
      * Generator function that yields requests injected with the given crawl depth.
@@ -1385,6 +1592,11 @@ export class BasicCrawler {
             if (typeof request === 'string') {
                 yield { url: request, crawlDepth: newRequestDepth };
             }
+            else if (request instanceof Request) {
+                const crawlee = (request.userData.__crawlee ??= {});
+                crawlee.crawlDepth ??= newRequestDepth;
+                yield request;
+            }
             else {
                 request.crawlDepth ??= newRequestDepth;
                 yield request;
@@ -1395,7 +1607,7 @@ export class BasicCrawler {
      * Run async callback with given timeout and retry. Returns the result of the callback.
      * @ignore
      */
-    async timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
+    async #timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
         try {
             return await addTimeoutToPromise(handler, timeout, error);
         }
@@ -1403,32 +1615,34 @@ export class BasicCrawler {
             if (retried <= maxRetries) {
                 // we retry on any error, not just timeout
                 this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
-                return this.timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
+                return this.#timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
             }
             throw e;
         }
     }
     /**
-     * Returns true if either RequestList or RequestQueue have a request ready for processing.
+     * Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
+     * wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
      */
-    async isTaskReadyFunction() {
-        return this.requestManager !== undefined && !(await this.requestManager.isEmpty());
-    }
-    /**
-     * Returns true if both RequestList and RequestQueue have all requests finished.
-     */
-    async defaultIsFinishedFunction() {
-        return !this.requestManager || (await this.requestManager.isFinished());
+    async #isTaskReadyFunction() {
+        if (this.requestManager === undefined) {
+            return false;
+        }
+        const state = await this.requestManager.checkReadiness();
+        if (state.status === 'waiting' && state.readyAt !== undefined) {
+            this.#run?.scheduleWake(state.readyAt);
+        }
+        return state.status === 'ready';
     }
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
+     * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
      */
-    unwrapError(error) {
+    #unwrapError(error) {
         if (error instanceof RequestHandlerError ||
             error instanceof ContextPipelineInitializationError ||
             error instanceof ContextPipelineCleanupError) {
-            return this.unwrapError(error.cause);
+            return this.#unwrapError(error.cause);
         }
         return error;
     }
@@ -1437,16 +1651,45 @@ export class BasicCrawler {
      *
      * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async requestFunctionErrorHandler(error, crawlingContext, request, source) {
+        if (error instanceof RequestThrottledError) {
+            // The domain told us to come back later, so the request was never really attempted. Put it back where
+            // it was, without recording a failure - it costs neither a retry nor session reputation.
+            this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
+                id: request.id,
+                url: request.url,
+            });
+            await source.reclaimRequest(request, { forefront: true });
+            return;
+        }
         request.pushErrorMessage(error);
         if (error instanceof CriticalError) {
             throw error;
         }
-        const shouldRetryRequest = this.canRequestBeRetried(request, error);
+        const shouldRetryRequest = this.#canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
-            await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
-            await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
-            error);
+            await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
+            try {
+                await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
+                error);
+            }
+            catch (handlerError) {
+                if (handlerError instanceof ContextPipelineInterruptedError) {
+                    const skipError = await this.#trySkipRequest(handlerError, request, source);
+                    if (!skipError) {
+                        return;
+                    }
+                    // Retry for the original error. Routing `skipError` through `errorHandler` again
+                    // would loop forever with a handler that always skips.
+                    this.log.warning(`Skipping request ${request.url} failed, retrying it instead.`, {
+                        error: skipError.message,
+                    });
+                }
+                else {
+                    throw handlerError;
+                }
+            }
             if (error instanceof SessionError) {
                 crawlingContext.session?.retire();
             }
@@ -1455,13 +1698,13 @@ export class BasicCrawler {
                 const { url, retryCount, id } = request;
                 // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
                 // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
-                const message = this._getMessageFromError(error);
+                const message = this.getMessageFromError(error);
                 this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {
                     id,
                     url,
                     retryCount,
                 });
-                await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+                await source.reclaimRequest(request);
                 return;
             }
         }
@@ -1473,34 +1716,54 @@ export class BasicCrawler {
         // This is to make sure the error snapshot is not duplicated in the errorTrackerRetry and errorTracker objects.
         const { noRetry, maxRetries } = request;
         if (noRetry || !maxRetries) {
-            await this.stats.errorTracker.addAsync(error, crawlingContext);
+            await this.statistics.errorTracker.addAsync(error, crawlingContext);
         }
         else {
-            this.stats.errorTracker.add(error);
+            this.statistics.errorTracker.add(error);
         }
         // If we get here, the request is either not retryable
         // or failed more than retryCount times and will not be retried anymore.
         // Mark the request as failed and do not retry.
         await source.markRequestAsHandled(request);
-        this.stats.failJob(request.id || request.uniqueKey, request.retryCount);
+        this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
         await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
+    /** Settles a skipped request. Returns the error that prevented that, leaving the request untouched if `onSkippedRequest` failed. */
+    async #trySkipRequest(error, request, source) {
+        try {
+            await addTimeoutToPromise(async () => this.#handleSkippedRequest({ request, reason: error.reason, message: error.skipMessage }), this.internalTimeoutMillis, `onSkippedRequest for ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            request.state = RequestState.SKIPPED;
+            request.noRetry = true;
+            await this.#timeoutAndRetry(async () => source.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
+            return undefined;
+        }
+        catch (settleError) {
+            return this.#unwrapError(settleError);
+        }
+    }
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
-        const message = this._getMessageFromError(error, true);
+        const message = this.getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
-        if (this.failedRequestHandler) {
-            await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
+        try {
+            await this.#failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
             error);
         }
+        catch (handlerError) {
+            if (!(handlerError instanceof ContextPipelineInterruptedError))
+                throw handlerError;
+            this.log.warning(`skipRequest() has no effect in failedRequestHandler, request ${url} (${id}) has already failed.`);
+        }
     }
     /**
      * Resolves the most verbose error message from a thrown error
      * @param error The error received
      * @returns The message to be logged
      */
-    _getMessageFromError(error, forceStack = false) {
+    getMessageFromError(error, forceStack = false) {
         if ([TypeError, SyntaxError, ReferenceError].some((type) => error instanceof type)) {
             forceStack = true;
         }
@@ -1508,13 +1771,23 @@ export class BasicCrawler {
         const baseDir = process.cwd();
         const userLine = stackLines.find((line) => line.includes(baseDir) && !line.includes('node_modules'));
         if (error instanceof TimeoutError) {
-            return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error; // stack in timeout errors does not really help
+            // stack in timeout errors does not really help
+            return (process.env.CRAWLEE_VERBOSE_LOG && error.stack) || error.message;
         }
         return process.env.CRAWLEE_VERBOSE_LOG || forceStack
             ? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
             : [error.message || error, userLine].join('\n');
     }
-    canRequestBeRetried(request, error) {
+    /**
+     * Whether the session should be spared for this error - either because it was already retired, or because the
+     * failure says nothing about the session (a rate limit is a property of the domain).
+     */
+    #errorAbsolvesSession(error) {
+        return (error instanceof SessionError ||
+            error instanceof RequestThrottledError ||
+            error instanceof ContextPipelineInterruptedError);
+    }
+    #canRequestBeRetried(request, error) {
         // Request should never be retried, or the error encountered makes it not able to be retried.
         if (request.noRetry || error instanceof NonRetryableError) {
             return false;
@@ -1524,33 +1797,55 @@ export class BasicCrawler {
             return true;
         }
         // Ensure there are more retries available for the request
-        const maxRequestRetries = request.maxRetries ?? this.maxRequestRetries;
+        const maxRequestRetries = request.maxRetries ?? this.#maxRequestRetries;
         return request.retryCount < maxRequestRetries;
     }
     /**
-     * Stops the crawler immediately.
+     * Ends the run in progress immediately, without waiting for the requests already in flight.
      *
-     * This method doesn't wait for currently active requests to finish.
+     * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
+     * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
+     * released by {@link BasicCrawler.destroy|`destroy()`} instead.
      *
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     async teardown() {
-        serviceLocator.getEventManager().emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
-        if (this._closeEvents) {
-            await serviceLocator.getEventManager().close();
+        const eventManager = serviceLocator.getEventManager();
+        const ownsEventManager = this.#run?.ownsEventManager === true;
+        // First, so that the pool writes its final state once - its teardown stops it listening before the
+        // persistence event below goes out.
+        await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown());
+        // When this crawler initialized the event manager, its close() call emits
+        // the final persistence event after the crawler-specific state has been
+        // saved. External event managers still need an explicit event here.
+        if (!ownsEventManager) {
+            eventManager.emit(EventType.PERSIST_STATE, { isMigrating: false });
         }
-        await this.sessionPoolDep.ifOwned((pool) => pool.teardown());
-        await this.autoscaledPool?.abort();
-        await this.concurrencySystemDep?.ifOwned((system) => system.stop());
+        if (ownsEventManager) {
+            await eventManager.close();
+        }
+        await this.#run?.end();
+    }
+    /**
+     * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
+     * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
+     * could otherwise have run again.
+     */
+    async destroy() {
+        // An abandoned run - one whose `run()` never returned - would otherwise keep dispatching requests.
+        await this.#run?.end();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.destroy();
     }
-    _getCookieHeaderFromRequest(request) {
+    getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {
             this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
             return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);
         }
         return request.headers?.Cookie || request.headers?.cookie || '';
     }
-    requestMatchesEnqueueStrategy(request) {
+    #requestMatchesEnqueueStrategy(request) {
         // If `skipNavigation` was used, just return `true`
         try {
             // eslint-disable-next-line @typescript-eslint/no-unused-expressions
@@ -1563,8 +1858,7 @@ export class BasicCrawler {
             throw err;
         }
         const { url, loadedUrl } = request;
-        // eslint-disable-next-line dot-notation -- private access
-        const strategy = request['enqueueStrategy'];
+        const strategy = request.enqueueStrategy;
         // No strategy set, so we assume it matches, or it was added outside of enqueueLinks
         if (!strategy) {
             return true;
@@ -1599,6 +1893,10 @@ export class BasicCrawler {
         }
     }
 }
+/** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
+function hostnameOrUrl(url) {
+    return URL.canParse(url) ? new URL(url).hostname : url;
+}
 export function createBasicRouter(routes) {
     return Router.create(routes);
 }