@crawlee/basic 4.0.0-rc.0 → 4.0.0-rc.1

package/internals/basic-crawler.js

@@ -1,16 +1,30 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
-import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
+import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
 import { getDomain } from 'tldts';
 import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
-import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
+import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
-import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
+import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
+import { mergeCookies } from './cookie_utils.js';
+import { ContextPipeline } from './crawlers/context_pipeline.js';
+import { CrawlingRequest, RequestState } from './crawling_request.js';
+import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
+import { Statistics } from './crawlers/statistics.js';
+import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue_links/index.js';
+import { parseRetryAfterHeader } from './http.js';
+import { CrawlerRun } from './crawler-run.js';
+import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
+import { Router, validateUserData } from './router.js';
+import { BLOCKED_STATUS_CODES } from './session_pool/consts.js';
+import { Session } from './session_pool/session.js';
+import { SessionPool } from './session_pool/session_pool.js';
+import { ThrottlingRequestManager } from './throttling_request_manager.js';
 class LazyDefaultHttpClient extends BaseHttpClient {
     #delegatePromise;
     constructor(options) {
@@ -40,7 +54,6 @@ class LazyDefaultHttpClient extends BaseHttpClient {
  * @ignore
  */
 const SAFE_MIGRATION_WAIT_MILLIS = 20000;
-const deferredCleanupKey = Symbol('deferredCleanup');
 // The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
 export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
 const urlPatternSchema = z.union([
@@ -71,14 +84,13 @@ const addRequestsOptionsSchema = z.looseObject({
     onSkippedRequest: schemas.anyFunction.optional(),
 });
 export class BasicCrawler {
-    static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
+    static #CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
     /**
      * Tracks the number of crawler instances created. The first crawler uses the default
      * request queue; subsequent ones get their own queue via a unique alias so they don't
      * collide.
      */
-    // kept as TS-private: tests reset the counter at runtime
-    static instanceCount = 0;
+    static #instanceCount = 0;
     /**
      * Tracks crawler instances that accessed shared state without having an explicit id.
      * Used to detect and warn about multiple crawlers sharing the same state.
@@ -93,12 +105,15 @@ export class BasicCrawler {
     get statistics() {
         return this.#statisticsDep.value;
     }
+    #requestManager;
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
-    requestManager;
+    get requestManager() {
+        return this.#requestManager;
+    }
     /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
     #sessionPoolDep;
     /**
@@ -108,12 +123,6 @@ export class BasicCrawler {
     get sessionPool() {
         return this.#sessionPoolDep.value;
     }
-    /**
-     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
-     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
-     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
-     */
-    #ownedRequestQueue = OwnedOrInjected.resolve();
     /**
      * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
      * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
@@ -126,31 +135,36 @@ export class BasicCrawler {
      * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
      */
     #resolveConcurrencySystem;
-    /** As resolved by `init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    #concurrencySystemDep;
+    /** The last run started on this instance. It outlives the run, so anything that drives one goes through `#liveRun`. */
+    #run;
+    #concurrencySystem;
+    /** The run in progress, if any — a finished run is history and cannot be driven. */
+    get #liveRun() {
+        return this.#run?.isLive === true ? this.#run : undefined;
+    }
+    /** The run whose task loop is dispatching, if any — only that one can be paused and resumed. */
+    get #dispatchingRun() {
+        return this.#liveRun?.isDispatching === true ? this.#liveRun : undefined;
+    }
+    /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
+    get running() {
+        return this.#liveRun !== undefined;
+    }
     /**
-     * The concurrency governor this run is booking its requests against — either the
+     * The concurrency governor the run in progress is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
      * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
      *
-     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
-     * is also rebuilt for every run, so the instance is not stable across runs.
+     * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
+     * during a run rather than caching it across runs.
      *
      * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
      * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem() {
-        return this.#concurrencySystemDep?.maybeValue;
+        return this.#concurrencySystem?.value;
     }
-    /**
-     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
-     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
-     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
-     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
-     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
-     */
-    #autoscaledPool;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -168,40 +182,38 @@ export class BasicCrawler {
      * pipelines expect the basic crawler fields to already be present in the context at runtime.
      *
      * Context built with this pipeline can be passed into multiple crawler pipelines at once.
-     * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
      */
-    get basicContextPipeline() {
-        if (this.#basicContextPipeline === undefined) {
-            this.#basicContextPipeline = this.buildBasicContextPipeline();
-        }
+    get #basicPipeline() {
+        this.#basicContextPipeline ??= this.#buildBasicContextPipeline();
         return this.#basicContextPipeline;
     }
     #contextPipeline;
+    /** @internal */
     get contextPipeline() {
         if (this.#contextPipeline === undefined) {
-            this.#contextPipeline = this.buildFinalContextPipeline();
+            this.#contextPipeline = this.#buildFinalContextPipeline();
         }
         return this.#contextPipeline;
     }
-    running = false;
-    hasFinishedBefore = false;
-    #unexpectedStop = false;
+    #hasFinishedBefore = false;
+    /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
+    get hasFinishedBefore() {
+        return this.#hasFinishedBefore;
+    }
     #log;
     get log() {
         return this.#log;
     }
     requestHandler;
-    errorHandler;
-    failedRequestHandler;
-    // kept as TS-private: tests read it at runtime
-    requestHandlerTimeoutMillis;
+    #errorHandler;
+    #failedRequestHandler;
+    #requestHandlerTimeoutMillis;
     internalTimeoutMillis;
-    maxRequestRetries;
-    maxCrawlDepth;
-    #sameDomainDelaySecs;
-    maxRequestsPerCrawl;
-    get handledRequestsCount() {
-        return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;
+    #maxRequestRetries;
+    #maxCrawlDepth;
+    #maxRequestsPerCrawl;
+    get #handledRequestsCount() {
+        return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;
     }
     #statusMessageLoggingInterval;
     #statusMessageCallback;
@@ -214,8 +226,7 @@ export class BasicCrawler {
      * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
      * settable here.
      */
-    // kept as TS-private: tests mutate it at runtime
-    taskLoopOptions;
+    #taskLoopOptions;
     httpClient;
     retryOnBlocked;
     #respectRobotsTxtFile;
@@ -223,12 +234,13 @@ export class BasicCrawler {
     #transactionalStorageEnabled;
     /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
     #storageWritePolicy;
-    onSkippedRequest;
-    #closeEvents;
-    #loggedPerRun = new Set();
+    #onSkippedRequest;
     #robotsTxtFileCache;
-    identity;
+    #identity;
     #contextPipelineOptions;
+    /**
+     * @internal
+     */
     static optionsShape = {
         contextPipelineBuilder: schemas.anyObject.optional(),
         extendContext: schemas.anyFunction.optional(),
@@ -270,6 +282,7 @@ export class BasicCrawler {
         // AutoscaledPool shorthands
         minConcurrency: schemas.anyNumber.optional(),
         maxConcurrency: schemas.anyNumber.optional(),
+        initialConcurrency: schemas.anyNumber.optional(),
         maxRequestsPerMinute: schemas.anyNumber
             .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
             .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
@@ -278,12 +291,12 @@ export class BasicCrawler {
         statistics: schemas.anyObject.optional(),
         id: z.string().optional(),
     };
-    static optionsSchema = z.strictObject(BasicCrawler.optionsShape);
+    static #optionsSchema = z.strictObject(BasicCrawler.optionsShape);
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
     constructor(options = {}) {
-        const parsedOptions = parseArgument(options, BasicCrawler.optionsSchema, 'BasicCrawlerOptions');
+        const parsedOptions = parseArgument(options, BasicCrawler.#optionsSchema, 'BasicCrawlerOptions');
         const { 
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
         requestList, 
@@ -292,15 +305,18 @@ export class BasicCrawler {
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
+        minConcurrency, maxConcurrency, initialConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
         // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
         // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
         // hammering a site.
         if (concurrencySystem !== undefined &&
-            (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
-            throw new Error('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
-                '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
-                'replaces. Pass them to the `ConcurrencySystem` constructor instead.');
+            (minConcurrency !== undefined ||
+                maxConcurrency !== undefined ||
+                initialConcurrency !== undefined ||
+                maxRequestsPerMinute !== undefined)) {
+            throw new Error('The `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts ' +
+                'cannot be combined with `concurrencySystem` - they configure the default `ConcurrencySystem` ' +
+                'that a supplied one replaces. Pass them to the `ConcurrencySystem` constructor instead.');
         }
         // Create per-crawler service locator if custom services were provided.
         // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
@@ -328,50 +344,58 @@ export class BasicCrawler {
             this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
             // Initialize the Configuration instance to avoid lazy loading in the components
             serviceLocator.getConfiguration();
-            const instanceIndex = BasicCrawler.instanceCount++;
-            this.identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
-            if (requestManager !== undefined) {
-                if (requestList !== undefined || requestQueue !== undefined) {
-                    throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
-                }
-                // Both would pace the same domains, from different keys and with no idea of one another.
-                if (sameDomainDelaySecs > 0 && supportsDomainThrottling(requestManager)) {
-                    throw new Error('The `sameDomainDelaySecs` option cannot be combined with a `requestManager` that throttles ' +
-                        'per domain on its own. Configure the delay on the manager instead, via the ' +
-                        '`minCrawlDelaySecs` option of `ThrottlingRequestManager`.');
-                }
-                this.requestManager = requestManager;
+            const instanceIndex = BasicCrawler.#instanceCount++;
+            this.#identity = { instanceIndex, hasExplicitId: id !== undefined, id: id ?? String(instanceIndex) };
+            if (requestManager !== undefined && (requestList !== undefined || requestQueue !== undefined)) {
+                throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
             }
-            else if (requestList !== undefined && requestQueue !== undefined) {
-                // Combine the read-only list with the writable queue into a tandem.
-                this.requestManager = new RequestManagerTandem(requestList, requestQueue);
+            const suppliedManager = requestManager ?? requestQueue;
+            // Offered before building a pacer of our own: anything that paces takes the floor - through any
+            // number of wrappers, since they all forward - so no domain ends up with two clocks.
+            const floorTaken = sameDomainDelaySecs > 0 &&
+                (suppliedManager?.recordPacingSignal({
+                    reason: 'minIntervalEverywhere',
+                    intervalMs: sameDomainDelaySecs * 1000,
+                    // What `sameDomainDelaySecs` has always meant: one clock per site, subdomains included.
+                    scope: 'registrableDomain',
+                }) ??
+                    false);
+            const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken;
+            // Built here rather than at first use so it can sit *inside* the tandem below, which is where a
+            // loader's transferred requests pass through it.
+            const writableManager = pacerNeeded
+                ? new ThrottlingRequestManager({
+                    domains: 'all',
+                    minCrawlDelaySecs: sameDomainDelaySecs,
+                    throttleBy: 'registrableDomain',
+                    persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.#identity.id}`,
+                    // A factory, because the default queue is only opened on first use.
+                    inner: suppliedManager ?? (() => this.#openOwnedRequestQueue()),
+                })
+                : suppliedManager;
+            if (requestList !== undefined) {
+                // The list is read first, while new requests still have somewhere writable to go.
+                this.#requestManager = new RequestManagerTandem(requestList, writableManager ?? (() => this.#openOwnedRequestQueue()));
             }
-            else if (requestQueue !== undefined) {
+            else if (writableManager !== undefined) {
                 // A RequestQueue is itself a request manager.
-                this.requestManager = requestQueue;
-            }
-            else if (requestList !== undefined) {
-                // A lone read-only `requestList` (deprecated option) is combined with a lazily-opened default queue
-                // into a tandem, so that its requests are read first and new ones can still be enqueued during the
-                // crawl. The queue is opened on first use; the tandem also forwards `persistState()` to the loader.
-                this.requestManager = new RequestManagerTandem(requestList, () => this.openOwnedRequestQueue());
+                this.#requestManager = writableManager;
             }
             this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
             this.proxyConfiguration = proxyConfiguration;
             this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
             this.#statusMessageCallback = statusMessageCallback;
             this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
-            this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
             this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
             this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
             this.requestHandler = requestHandler ?? this.router;
-            this.failedRequestHandler = failedRequestHandler;
-            this.errorHandler = errorHandler;
+            this.#failedRequestHandler = failedRequestHandler;
+            this.#errorHandler = errorHandler;
             if (requestHandlerTimeoutSecs) {
-                this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+                this.#requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
             }
             else {
-                this.requestHandlerTimeoutMillis = 60_000;
+                this.#requestHandlerTimeoutMillis = 60_000;
             }
             this.retryOnBlocked = retryOnBlocked;
             this.#respectRobotsTxtFile = respectRobotsTxtFile;
@@ -379,14 +403,13 @@ export class BasicCrawler {
             const transactionalStorageOption = transactionalStorage;
             this.#transactionalStorageEnabled = transactionalStorageOption !== false;
             this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
-            this.onSkippedRequest = onSkippedRequest;
+            this.#onSkippedRequest = onSkippedRequest;
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
                 serviceLocator.getConfiguration().internalTimeoutMillis ??
-                    Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
-            this.maxRequestRetries = maxRequestRetries;
-            this.maxCrawlDepth = maxCrawlDepth;
-            this.#sameDomainDelaySecs = sameDomainDelaySecs;
+                    Math.max(this.#requestHandlerTimeoutMillis * 2, 300e3);
+            this.#maxRequestRetries = maxRequestRetries;
+            this.#maxCrawlDepth = maxCrawlDepth;
             this.#statisticsDep = OwnedOrInjected.resolve(statistics, 
             // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
             // only be satisfied by an injected instance carrying the matching `state`, so this branch does
@@ -394,7 +417,7 @@ export class BasicCrawler {
             () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
-                id: this.identity.id,
+                id: this.#identity.id,
             }));
             if (sessionPool && proxyConfiguration) {
                 this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
@@ -405,19 +428,21 @@ export class BasicCrawler {
             this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
                 createSessionFunction: async (opts) => new Session({
                     ...opts?.sessionOptions,
-                    proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
+                    proxyInfo: this.proxyConfiguration
+                        ? await this.proxyConfiguration.newProxyInfo(opts?.sessionOptions?.proxyInfo)
+                        : opts?.sessionOptions?.proxyInfo,
                 }),
             }));
             this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
             const maxSignedInteger = 2 ** 31 - 1;
-            if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
-                this.log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
+            if (this.#requestHandlerTimeoutMillis > maxSignedInteger) {
+                this.log.warning(`requestHandlerTimeoutMillis ${this.#requestHandlerTimeoutMillis}` +
                     ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
-                this.requestHandlerTimeoutMillis = maxSignedInteger;
+                this.#requestHandlerTimeoutMillis = maxSignedInteger;
             }
             this.internalTimeoutMillis = Math.min(this.internalTimeoutMillis, maxSignedInteger);
-            this.maxRequestsPerCrawl = maxRequestsPerCrawl;
-            const isMaxPagesExceeded = () => this.maxRequestsPerCrawl && this.maxRequestsPerCrawl <= this.handledRequestsCount;
+            this.#maxRequestsPerCrawl = maxRequestsPerCrawl;
+            const isMaxPagesExceeded = () => this.#maxRequestsPerCrawl && this.#maxRequestsPerCrawl <= this.#handledRequestsCount;
             // eslint-disable-next-line prefer-const
             let { isFinishedFunction, isTaskReadyFunction } = taskLoopOptions;
             // override even if `isFinishedFunction` provided by user - `keepAlive` has higher priority
@@ -429,90 +454,74 @@ export class BasicCrawler {
                     const source = this.requestManager;
                     if (!source)
                         throw new Error('Request provider is not initialized!');
-                    const request = await this.resolveRequest();
+                    const request = await this.#resolveRequest();
                     if (!request) {
                         return;
                     }
                     // Started here, rather than in `handleRequest`, so that a failure during context pipeline
                     // initialization (e.g. a browser page timing out before the request handler ever runs) is
-                    // still accounted for by `failJob` below - which is a no-op without a matching `startJob`.
-                    this.statistics.startJob(request.id || request.uniqueKey);
+                    // still accounted for by `recordRequestFailure` below - which is a no-op without a
+                    // matching `recordRequestStart`.
+                    this.statistics.recordRequestStart(request.id || request.uniqueKey);
                     const crawlingContext = { request };
                     try {
                         // The transaction spans the whole pipeline call, covering the navigation hooks
-                        // and `extendContext` too; `handleRequest` drives its outcome explicitly.
+                        // and `extendContext` too; `handleRequest` commits it and `handleRequestFailure` rolls it back.
                         await this.runInStorageTransaction(async () => 
                         // Navigation, the navigation hooks and the request handler are timed individually, but the
                         // phases between them are not, so a request could still get stuck indefinitely. This is the
                         // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
-                        await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
-                            .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request))));
+                        await this.#withRequestTimeout(crawlingContext, this.#basicPipeline.chain(this.contextPipeline).call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request), (error) => this.#handleRequestFailure(error, crawlingContext, source, request))));
                     }
                     catch (error) {
-                        // ContextPipelineInterruptedError means the request was intentionally skipped
-                        // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
-                        if (error instanceof ContextPipelineInterruptedError) {
-                            this.statistics.discardJob(request.id || request.uniqueKey);
-                            await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                            return;
-                        }
-                        // If the error happened during pipeline initialization (e.g., navigation timeout, session/proxy error,
-                        // i.e. not in user's requestHandler), handle it through the normal error flow. A bare `TimeoutError`
-                        // here is the internal timeout above firing - anything else thrown inside the pipeline arrives wrapped.
-                        const isPipelineError = error instanceof ContextPipelineInitializationError ||
-                            error instanceof SessionError ||
-                            error instanceof TimeoutError;
-                        if (isPipelineError) {
-                            const unwrappedError = this.unwrapError(error);
+                        // The internal timeout above fired, or `handleRequestFailure` threw a `SessionError` - both
+                        // take the normal error flow. Anything else is fatal.
+                        if (error instanceof SessionError || crawlingContext[timeoutExpiredKey]?.()) {
+                            const unwrappedError = this.#unwrapError(error);
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
-                            // SessionError already retired the session in `requestFunctionErrorHandler`;
-                            // skip `markBad` to avoid double-counting usage/error score.
-                            if (!this.errorAbsolvesSession(unwrappedError)) {
+                            if (!this.#errorAbsolvesSession(unwrappedError)) {
                                 crawlingContext.session?.markBad();
                             }
                             return;
                         }
-                        throw this.unwrapError(error);
-                    }
-                    finally {
-                        // Run request-scoped deferred cleanups only after the whole request lifecycle - including the user's error handler - has finished.
-                        const deferredCleanup = crawlingContext[deferredCleanupKey] ?? [];
-                        await Promise.all(deferredCleanup.map((fn) => fn().catch((cleanupError) => this.log.debug('Error in deferred cleanup', { error: cleanupError }))));
+                        throw this.#unwrapError(error);
                     }
                 },
                 isTaskReadyFunction: async () => {
                     if (isMaxPagesExceeded()) {
-                        this.logOncePerRun('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
-                            `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
+                        this.#liveRun?.logOnce('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
+                            `${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
                         return false;
                     }
-                    if (this.#unexpectedStop) {
-                        this.logOncePerRun('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
+                    if (this.#run?.stopRequested) {
+                        this.#liveRun?.logOnce('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
                             'Ongoing requests will be allowed to complete.');
                         return false;
                     }
-                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this.isTaskReadyFunction();
+                    return isTaskReadyFunction ? await isTaskReadyFunction() : await this.#isTaskReadyFunction();
                 },
                 isFinishedFunction: async () => {
                     if (isMaxPagesExceeded()) {
-                        this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.maxRequestsPerCrawl} requests ` +
+                        this.log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${this.#maxRequestsPerCrawl} requests ` +
                             'and all requests that were in progress at that time have now finished. ' +
-                            `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
+                            `In total, the crawler processed ${this.#handledRequestsCount} requests and will shut down.`);
                         return true;
                     }
-                    if (this.#unexpectedStop) {
+                    if (this.#run?.stopRequested) {
                         this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
                         return true;
                     }
-                    // Checked here because this runs only once nothing is in flight, which is exactly when a
-                    // crawl that cannot progress looks indistinguishable from one that is merely waiting.
-                    if (!keepAlive && supportsDomainThrottling(this.requestManager)) {
-                        await this.requestManager.assertNoStalledDomains();
+                    // `maybeFinish()` calls this only once nothing is in flight (`autoscaled_pool.ts`) - the point
+                    // where a crawl that cannot progress becomes distinguishable from one that is merely waiting,
+                    // and the only place where throwing does not abandon requests mid-processing.
+                    const state = await this.requestManager?.checkReadiness();
+                    // Under `keepAlive`, outliving a domain that will not let us through is the whole point.
+                    if (state?.status === 'stalled' && !keepAlive) {
+                        throw new PersistentRateLimitError(`Giving up: ${state.reason}`);
                     }
                     const isFinished = isFinishedFunction
                         ? await isFinishedFunction()
-                        : await this.defaultIsFinishedFunction();
+                        : state === undefined || state.status === 'finished';
                     if (isFinished) {
                         const reason = isFinishedFunction
                             ? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
@@ -523,11 +532,14 @@ export class BasicCrawler {
                 },
                 log: this.log,
             };
-            this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
+            this.#taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
             this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
                 minConcurrency,
                 maxConcurrency,
                 maxTasksPerMinute: maxRequestsPerMinute,
+                // Spread conditionally - an explicit `undefined` would clobber a subclass default, see
+                // `HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS`.
+                ...(initialConcurrency !== undefined && { desiredConcurrency: initialConcurrency }),
                 log: this.log,
             }));
         }
@@ -537,7 +549,7 @@ export class BasicCrawler {
     }
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
@@ -561,35 +573,21 @@ export class BasicCrawler {
      * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
      * This handles base context creation, session resolution, and context helpers.
      */
-    buildBasicContextPipeline() {
+    #buildBasicContextPipeline() {
         return ContextPipeline.create()
-            .compose({ action: this.checkRobotsTxt.bind(this) })
-            .compose({ action: (context) => this.createBaseContext(context) })
-            .compose({ action: this.resolveSession.bind(this) })
-            .compose({ action: this.createContextHelpers.bind(this) });
+            .compose(this.#checkRobotsTxt.bind(this))
+            .compose(this.#createBaseContext.bind(this))
+            .compose(this.#resolveSession.bind(this))
+            .compose(this.#createContextHelpers.bind(this));
     }
-    async checkRobotsTxt({ request }) {
-        if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
+    async #checkRobotsTxt({ request }) {
+        if (!(await this.#isAllowedBasedOnRobotsTxtFile(request.url))) {
             this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
-            request.state = RequestState.SKIPPED;
-            request.noRetry = true;
-            await this.handleSkippedRequest({
-                url: request.url,
-                reason: 'robotsTxt',
-            });
-            throw new ContextPipelineInterruptedError(`Skipping request ${request.url} as disallowed by robots.txt`);
+            throw new ContextPipelineInterruptedError('robotsTxt');
         }
         return {};
     }
-    /**
-     * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
-     * Subclasses should override this to add their own pipeline stages.
-     */
-    buildContextPipeline() {
-        return ContextPipeline.create();
-    }
-    createBaseContext(context) {
-        const deferredCleanup = [];
+    #createBaseContext(context, onCleanup) {
         return {
             id: cryptoRandomObjectId(10),
             log: this.log,
@@ -597,7 +595,23 @@ export class BasicCrawler {
             useState: this.useState.bind(this),
             getKeyValueStore: async (identifier) => KeyValueStore.open(identifier),
             registerDeferredCleanup: (cleanup) => {
-                deferredCleanup.push(cleanup);
+                onCleanup(async () => {
+                    try {
+                        await cleanup();
+                    }
+                    catch (error) {
+                        this.log.debug('Error in deferred cleanup', { error });
+                    }
+                });
+            },
+            afterStorageCommit: (callback) => {
+                const transaction = currentStorageTransaction();
+                if (transaction === undefined) {
+                    throw new Error('afterStorageCommit() needs an active storage transaction, and there is none. ' +
+                        "With `transactionalStorage: false` the request's writes are applied as they are " +
+                        'made, so they throw at the call site instead of at commit time.');
+                }
+                transaction.afterCommit(callback);
             },
             extendTimeout: (secs) => {
                 const extraMillis = secs * 1000;
@@ -610,29 +624,43 @@ export class BasicCrawler {
                 if (context[navigationDeadlineKey] !== undefined) {
                     context[navigationDeadlineKey] += extraMillis;
                 }
+                // Extension failure must not fail the request that asked for more time.
+                this.requestManager?.extendRequestProcessingTimeSecs?.(context.request, secs)?.catch((error) => {
+                    this.log.debug('Extending the request processing time failed', { url: context.request.url, error });
+                });
+            },
+            skipRequest: (message) => {
+                this.log.debug(`Skipping request ${context.request.url} (${context.request.id})${message ? `: ${message}` : ''}`);
+                throw new ContextPipelineInterruptedError('manual', message);
             },
-            [deferredCleanupKey]: deferredCleanup,
         };
     }
-    async resolveRequest() {
-        const request = await this.timeoutAndRetry(this.fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-        // Reset loadedUrl so an old one is not carried over to retries.
-        if (request) {
-            request.loadedUrl = undefined;
+    /**
+     * Fetches the next request and rebuilds it as a {@link CrawlingRequest}. A copy rather than the manager's
+     * instance: managers key on `uniqueKey`/`id`, and it is this copy that flows back into `reclaimRequest` /
+     * `markRequestAsHandled`, so nothing observes the difference.
+     */
+    async #resolveRequest() {
+        const fetched = await this.#timeoutAndRetry(this.#fetchNextRequest.bind(this), this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        if (!fetched) {
+            return null;
         }
+        const request = CrawlingRequest.fromSchema(fetched);
+        // Reset loadedUrl so an old one is not carried over to retries.
+        request.loadedUrl = undefined;
         return request;
     }
-    async resolveSession({ request }) {
-        const session = await this.timeoutAndRetry(async () => {
+    async #resolveSession({ request }) {
+        const session = await this.#timeoutAndRetry(async () => {
             const existingSession = await this.sessionPool.getSession(request.sessionId);
             if (!existingSession) {
-                throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
+                throw new MissingSessionError(request.sessionId);
             }
             return existingSession;
         }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
         return { session, proxyInfo: session?.proxyInfo };
     }
-    async createContextHelpers({ request, session }) {
+    async #createContextHelpers({ request, session }) {
         const addRequests = async (requests, options = {}) => {
             const newCrawlDepth = request.crawlDepth + 1;
             const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
@@ -641,9 +669,9 @@ export class BasicCrawler {
         const sendRequest = createSendRequest(this.httpClient, request, session);
         return { addRequests, sendRequest };
     }
-    buildFinalContextPipeline() {
+    #buildFinalContextPipeline() {
         const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
-            this.buildContextPipeline());
+            ContextPipeline.create());
         // `extendContext` runs *before* the subclass navigation pipeline (which includes the
         // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
         // request handler alike. The trade-off is that `extendContext` cannot access
@@ -658,26 +686,21 @@ export class BasicCrawler {
         let contextPipeline;
         if (extendContext !== undefined) {
             contextPipeline = ContextPipeline.create()
-                .compose({ action: async (context) => await extendContext(context) })
+                .compose(async (context) => await extendContext(context))
                 .chain(subclassPipeline);
         }
         else {
             contextPipeline = subclassPipeline;
         }
-        contextPipeline = contextPipeline.compose({
-            action: async (context) => {
-                const { request } = context;
-                if (request && !this.requestMatchesEnqueueStrategy(request)) {
-                    // eslint-disable-next-line dot-notation
-                    const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
-                    this.log.debug(message);
-                    request.noRetry = true;
-                    request.state = RequestState.SKIPPED;
-                    await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });
-                    throw new ContextPipelineInterruptedError(message);
-                }
-                return context;
-            },
+        contextPipeline = contextPipeline.compose(async (context) => {
+            const { request } = context;
+            if (request && !this.#requestMatchesEnqueueStrategy(request)) {
+                // eslint-disable-next-line dot-notation
+                const message = `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`;
+                this.log.debug(message);
+                throw new ContextPipelineInterruptedError('redirect');
+            }
+            return context;
         });
         return contextPipeline;
     }
@@ -688,7 +711,7 @@ export class BasicCrawler {
      * @param error The error to check.
      */
     isProxyError(error) {
-        return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error)?.includes(x));
+        return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error).includes(x));
     }
     /**
      * Sets the status message for the current crawler run.
@@ -701,19 +724,21 @@ export class BasicCrawler {
      */
     setStatusMessage(message, options = {}) {
         const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
-        this.log.logWithLevel(LogLevel[options.level ?? 'DEBUG'], message, data);
+        // Each allowed level has its own method on the logger, so this goes through them rather than through
+        // `logWithLevel`, which is abstract and therefore cannot be instrumented.
+        this.log[{ DEBUG: 'debug', INFO: 'info', WARNING: 'warning', ERROR: 'error' }[options.level ?? 'DEBUG']](message, data);
         // Broadcast the status message through the event system. Consumers (e.g. the Apify SDK) can
         // subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.
         // Setting the status message is not a storage concern, so we intentionally don't route it
         // through the storage client anymore.
         serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
-            crawlerId: this.identity.id,
+            crawlerId: this.#identity.id,
             message,
             isStatusMessageTerminal: options.isStatusMessageTerminal,
             level: options.level,
         });
     }
-    getPeriodicLogger() {
+    #getPeriodicLogger() {
         let previousState = { ...this.statistics.state };
         const getOperationMode = () => {
             const { requestsFailed } = this.statistics.state;
@@ -733,7 +758,7 @@ export class BasicCrawler {
             }
             else {
                 const total = await this.requestManager?.getTotalCount();
-                message = `Crawled ${this.statistics.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
+                message = `Crawled ${this.statistics.state.requestsSucceeded}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
             if (this.#statusMessageCallback) {
                 await this.#statusMessageCallback({
@@ -757,120 +782,141 @@ export class BasicCrawler {
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
      *
+     * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
+     * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
+     * if that is what you want.
+     *
      * @param [requests] The requests to add.
-     * @param [options] Options for the request queue.
+     * @param [options] Options for adding the initial requests.
      */
     async run(requests, options) {
-        if (this.running) {
-            throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
-        }
-        const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
-        if (this.hasFinishedBefore) {
-            // When executing the run method for the second time explicitly,
-            // we need to purge the RQ to allow processing the same requests again — this is important so users can
-            // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
-            // ignored — as a failed request is still handled.
-            // By default (`purgeRequestQueue` unset), only the queue we opened ourselves is purged.
-            // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
-            // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
-            const shouldPurge = purgeRequestQueue !== false;
-            const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
-            if (shouldPurge) {
-                await managerToPurge?.purge?.();
-                // The per-domain queues a `sameDomainDelaySecs` wrapper created are the crawler's own, whatever
-                // sits underneath them - so they are emptied even when the manager they wrap is spared. Purging
-                // the wrapper itself has already covered them.
-                if (this.requestManager instanceof ThrottlingRequestManager && managerToPurge !== this.requestManager) {
-                    await this.requestManager.purgeDomainQueues();
-                }
+        // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
+        return withDirectStorageAccess(async () => timeoutStorage.exit(async () => {
+            if (this.running) {
+                throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
             }
-            // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
-            await this.#statisticsDep.ifOwned(async (stats) => {
-                stats.reset();
-                await stats.resetStore();
-            });
-            await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());
-        }
-        this.#unexpectedStop = false;
-        this.running = true;
-        this.#loggedPerRun.clear();
-        await purgeDefaultStorages({
-            onlyPurgeOnce: true,
-            storageBackend: serviceLocator.getStorageBackend(),
-            configuration: serviceLocator.getConfiguration(),
-        });
-        if (requests) {
-            await this.addRequests(requests, addRequestsOptions);
-        }
-        try {
-            await this.init();
-            await this.statistics.startCapturing();
-        }
-        catch (error) {
-            // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
-            await this.teardown().catch((teardownError) => {
-                this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
+            if (this.#hasFinishedBefore) {
+                // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+                await this.#statisticsDep.ifOwned(async (stats) => {
+                    stats.reset();
+                    await stats.resetStore();
+                });
+                // A session pool the crawler built follows the same blank-slate rule as its statistics.
+                await this.#sessionPoolDep.ifOwned(async (pool) => {
+                    pool.reset();
+                    await pool.resetStore();
+                });
+            }
+            // An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
+            // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
+            // run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
+            this.#concurrencySystem = this.#resolveConcurrencySystem();
+            const run = new CrawlerRun({
+                log: this.log,
+                concurrencySystem: this.#concurrencySystem.value,
+                taskLoopOptions: this.#taskLoopOptions,
+                consumer: this.#identity,
             });
-            // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
-            this.running = false;
-            throw error;
-        }
-        const periodicLogger = this.getPeriodicLogger();
-        this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
-        const sigintHandler = async () => {
-            this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
-            await this.pauseOnMigration();
-            await this.#autoscaledPool.abort();
-        };
-        // Attach a listener to handle migration and aborting events gracefully.
-        const boundPauseOnMigration = this.pauseOnMigration.bind(this);
-        process.once('SIGINT', sigintHandler);
-        const eventManager = serviceLocator.getEventManager();
-        eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
-        eventManager.on(EventType.ABORTING, boundPauseOnMigration);
-        let stats = {};
-        try {
-            await this.#autoscaledPool.run();
-        }
-        finally {
-            await this.statistics.stopCapturing();
-            await this.teardown();
-            process.off('SIGINT', sigintHandler);
-            eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
-            eventManager.off(EventType.ABORTING, boundPauseOnMigration);
-            const finalStats = this.statistics.calculate();
-            stats = {
-                requestsFinished: this.statistics.state.requestsFinished,
-                requestsFailed: this.statistics.state.requestsFailed,
-                retryHistogram: this.statistics.requestRetryHistogram,
-                ...finalStats,
-            };
-            this.log.info('Final request statistics:', stats);
-            if (this.statistics.errorTracker.total !== 0) {
-                const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
-                this.log.info(`Error analysis:`, {
-                    totalErrors: this.statistics.errorTracker.total,
-                    uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
-                    mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
+            this.#run = run;
+            try {
+                await purgeDefaultStorages({
+                    onlyPurgeOnce: true,
+                    storageBackend: serviceLocator.getStorageBackend(),
+                    configuration: serviceLocator.getConfiguration(),
                 });
+                if (requests) {
+                    await this.addRequests(requests, options);
+                }
+                // An injected governor is the caller's to run; an owned one is started here and stopped again
+                // below if the startup fails. Either way the task loop rejects one that is not running.
+                await this.#concurrencySystem.ifOwned((system) => system.start());
+                await this.init();
+                await this.statistics.startCapturing();
+            }
+            catch (error) {
+                // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
+                await this.teardown().catch((teardownError) => {
+                    this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
+                });
+                // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
+                await run.finish();
+                await this.#concurrencySystem.ifOwned((system) => system.stop());
+                throw error;
+            }
+            const periodicLogger = this.#getPeriodicLogger();
+            this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
+            const sigintHandler = async () => {
+                this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
+                await this.#pauseOnMigration();
+                await run.abort();
+            };
+            // Attach a listener to handle migration and aborting events gracefully.
+            const boundPauseOnMigration = this.#pauseOnMigration.bind(this);
+            process.once('SIGINT', sigintHandler);
+            const eventManager = serviceLocator.getEventManager();
+            eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
+            eventManager.on(EventType.ABORTING, boundPauseOnMigration);
+            let stats = {};
+            try {
+                await run.dispatchRequests();
             }
-            const client = serviceLocator.getStorageBackend();
-            if (client.teardown) {
-                let finished = false;
-                setTimeout(() => {
-                    if (!finished) {
-                        this.log.info('Waiting for the storage to write its state to file system.');
+            finally {
+                await this.statistics.stopCapturing();
+                await this.teardown();
+                process.off('SIGINT', sigintHandler);
+                eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
+                eventManager.off(EventType.ABORTING, boundPauseOnMigration);
+                const finalStats = this.statistics.calculate();
+                stats = {
+                    requestsSucceeded: this.statistics.state.requestsSucceeded,
+                    requestsFailed: this.statistics.state.requestsFailed,
+                    retryHistogram: this.statistics.requestRetryHistogram,
+                    ...finalStats,
+                };
+                this.log.info('Final request statistics:', stats);
+                // A crawl that did nothing while the manager holds only handled requests is a mistake whoever
+                // handled them - this run, another crawler on the same queue, or a previous process. Starting
+                // against handled requests is not: that is what resuming a crawl looks like.
+                if (stats.requestsSucceeded + stats.requestsFailed === 0) {
+                    // Never let the diagnostic itself break the run.
+                    const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
+                    if (alreadyHandled > 0) {
+                        this.log.warningOnce('This crawl processed no requests - the request manager holds ' +
+                            `${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` +
+                            'already handled, and a failed request counts as handled too. Nothing ' +
+                            'empties a queue between runs, so to crawl them again, purge it ' +
+                            '(`await queue.purge()`) or use a fresh one (e.g. ' +
+                            '`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' +
+                            'crawler instance.');
                     }
-                }, 1000);
-                await client.teardown();
-                finished = true;
+                }
+                if (this.statistics.errorTracker.total !== 0) {
+                    const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
+                    this.log.info(`Error analysis:`, {
+                        totalErrors: this.statistics.errorTracker.total,
+                        uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
+                        mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
+                    });
+                }
+                const client = serviceLocator.getStorageBackend();
+                if (client.teardown) {
+                    let finished = false;
+                    setTimeout(() => {
+                        if (!finished) {
+                            this.log.info('Waiting for the storage to write its state to file system.');
+                        }
+                    }, 1000);
+                    await client.teardown();
+                    finished = true;
+                }
+                periodicLogger.stop();
+                this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsSucceeded} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
+                await run.finish();
+                await this.#concurrencySystem.ifOwned((system) => system.stop());
+                this.#hasFinishedBefore = true;
             }
-            periodicLogger.stop();
-            this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
-            this.running = false;
-            this.hasFinishedBefore = true;
-        }
-        return stats;
+            return stats;
+        }));
     }
     /**
      * Gracefully stops the current run of the crawler.
@@ -880,11 +926,12 @@ export class BasicCrawler {
      * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
     stop(reason = 'The crawler has been gracefully stopped.') {
-        if (this.#unexpectedStop) {
+        const run = this.#liveRun;
+        if (run === undefined) {
+            this.log.warning('Cannot stop a crawler that is not running.');
             return;
         }
-        this.log.info(reason);
-        this.#unexpectedStop = true;
+        run.stop(reason);
     }
     /**
      * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
@@ -895,52 +942,41 @@ export class BasicCrawler {
      * throughout, since a shared one may still be serving other crawlers.
      */
     async pause(timeoutSecs) {
-        if (!this.#autoscaledPool) {
+        const run = this.#dispatchingRun;
+        if (run === undefined) {
             this.log.warning('Cannot pause a crawler that is not running.');
             return;
         }
-        await this.#autoscaledPool.pause(timeoutSecs);
+        await run.pause(timeoutSecs);
     }
     /**
      * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
      * again. A no-op on a crawler that is not paused.
      */
     resume() {
-        if (!this.#autoscaledPool) {
+        const run = this.#dispatchingRun;
+        if (run === undefined) {
             this.log.warning('Cannot resume a crawler that is not running.');
             return;
         }
-        this.#autoscaledPool.resume();
+        run.resume();
     }
     /**
      * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
      * if none has been configured or opened yet.
      */
     async getRequestManager() {
-        if (!this.requestManager) {
-            this.requestManager = await this.openOwnedRequestQueue();
-        }
-        // Wrapped here rather than in the constructor, because the manager being wrapped may only be opened at
-        // this point - and because everything that enqueues goes through here first, so nothing slips past the
-        // wrapper into the queue it hides.
-        if (this.#sameDomainDelaySecs > 0 && !supportsDomainThrottling(this.requestManager)) {
-            this.requestManager = new ThrottlingRequestManager({
-                inner: this.requestManager,
-                domains: 'all',
-                minCrawlDelaySecs: this.#sameDomainDelaySecs,
-                // What `sameDomainDelaySecs` has always meant: one clock for a site, subdomains included.
-                throttleBy: 'registrableDomain',
-                persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.identity.id}`,
-            });
+        if (!this.#requestManager) {
+            this.#requestManager = await this.#openOwnedRequestQueue();
         }
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
         // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
         // but guard so we do not re-issue it on every call.
         if (!this.#requestManagerTimeoutsApplied) {
             this.#requestManagerTimeoutsApplied = true;
-            await this.applyRequestManagerTimeouts(this.requestManager);
+            await this.#applyRequestManagerTimeouts(this.#requestManager);
         }
-        return this.requestManager;
+        return this.#requestManager;
     }
     /**
      * @deprecated Use {@link BasicCrawler.getRequestManager|`getRequestManager()`} instead. This returns the
@@ -950,16 +986,13 @@ export class BasicCrawler {
         return this.getRequestManager();
     }
     /**
-     * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
-     * crawler-owned queue (so it gets purged between repeated `run()` calls).
-     * @private
+     * Opens the default {@link RequestQueue} — the crawler's own, read from when the caller supplied nothing.
      */
-    async openOwnedRequestQueue() {
+    async #openOwnedRequestQueue() {
         // The first crawler instance uses the default queue (null identifier);
         // subsequent instances get their own queue via a unique alias so they don't collide.
-        const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
-        const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
-        return this.#ownedRequestQueue.set(requestQueue);
+        const identifier = this.#identity.instanceIndex === 0 ? null : { alias: `__default_${this.#identity.id}__` };
+        return RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
     }
     /**
      * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
@@ -968,12 +1001,12 @@ export class BasicCrawler {
      * request from being handed out a second time while it is still being processed — and it works
      * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
      */
-    async applyRequestManagerTimeouts(requestManager) {
+    async #applyRequestManagerTimeouts(requestManager) {
         // A router route may hold a request for longer than the crawler's own timeout, and we cannot know
         // which routes a run will hit, so reserve for the longest one any route asked for. The hint is
         // raise-only, so erring high here is safe.
         const maxRouteTimeoutSecs = this.requestHandler.getMaxTimeoutSecs?.() ?? 0;
-        const handlerTimeoutSecs = Math.max(this.requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
+        const handlerTimeoutSecs = Math.max(this.#requestHandlerTimeoutMillis / 1000, maxRouteTimeoutSecs);
         await requestManager.setExpectedRequestProcessingTimeSecs?.(Math.max(handlerTimeoutSecs + 5, 60));
     }
     /**
@@ -983,7 +1016,7 @@ export class BasicCrawler {
      * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
      * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
      */
-    async validateRequestUserData(source) {
+    async #validateRequestUserData(source) {
         if (typeof source === 'string') {
             return;
         }
@@ -1004,11 +1037,11 @@ export class BasicCrawler {
     }
     async useState(defaultValue = {}) {
         const kvs = await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
-        if (this.identity.hasExplicitId) {
-            const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
+        if (this.#identity.hasExplicitId) {
+            const stateKey = `${BasicCrawler.#CRAWLEE_STATE_KEY}_${this.#identity.id}`;
             return kvs.getAutoSavedValue(stateKey, defaultValue);
         }
-        BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);
+        BasicCrawler.#useStateAnonymousIndices.add(this.#identity.instanceIndex);
         if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
             serviceLocator
                 .getLogger()
@@ -1017,38 +1050,36 @@ export class BasicCrawler {
                 'To fix this, provide a unique `id` option to each crawler instance. \n' +
                 'Example: new BasicCrawler({ id: "my-crawler-1", ... })');
         }
-        return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
+        return kvs.getAutoSavedValue(BasicCrawler.#CRAWLEE_STATE_KEY, defaultValue);
     }
-    async getPendingRequestCountApproximation() {
+    async #getPendingRequestCountApproximation() {
         return (await this.requestManager?.getPendingCount()) ?? 0;
     }
-    async calculateEnqueuedRequestLimit(explicitLimit) {
-        if (this.maxRequestsPerCrawl === undefined) {
+    async #calculateEnqueuedRequestLimit(explicitLimit) {
+        if (this.#maxRequestsPerCrawl === undefined) {
             return explicitLimit;
         }
-        const limit = Math.max(0, this.maxRequestsPerCrawl - this.handledRequestsCount - (await this.getPendingRequestCountApproximation()));
+        const limit = Math.max(0, this.#maxRequestsPerCrawl -
+            this.#handledRequestsCount -
+            (await this.#getPendingRequestCountApproximation()));
         return Math.min(limit, explicitLimit ?? Infinity);
     }
-    async handleSkippedRequest(options) {
+    async #handleSkippedRequest(options) {
         // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
         // transaction scope, which rolls back - so the skip bookkeeping must write directly.
         await withDirectStorageAccess(async () => {
+            // Enqueueing happens outside a run too - `addRequests()` before the first `run()` - so these dedupe
+            // against the logger for the crawler's lifetime rather than per run.
             if (options.reason === 'limit') {
-                this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
-                    `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
+                this.log.info('The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
+                    `${this.#maxRequestsPerCrawl} requests and no further requests will be added.`, undefined, { once: true });
             }
             if (options.reason === 'depth') {
-                this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
+                this.log.info(`The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`, undefined, { once: true });
             }
-            await this.onSkippedRequest?.(options);
+            await this.#onSkippedRequest?.(options);
         });
     }
-    logOncePerRun(key, message, level = 'info') {
-        if (!this.#loggedPerRun.has(key)) {
-            this.log[level](message);
-            this.#loggedPerRun.add(key);
-        }
-    }
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -1077,25 +1108,47 @@ export class BasicCrawler {
         // its own per-item validation below, and validating an absent label/userData here would spuriously
         // check them against a registered default-route schema.
         if (options.label !== undefined || options.userData !== undefined) {
-            await this.validateRequestUserData({ label: options.label, userData: options.userData });
+            await this.#validateRequestUserData({ label: options.label, userData: options.userData });
         }
-        const requestLimit = await this.calculateEnqueuedRequestLimit(options.limit);
+        const requestLimit = await this.#calculateEnqueuedRequestLimit(options.limit);
         const strategy = options.strategy ?? EnqueueStrategy.All;
-        const urlExcludePatternObjects = options.exclude?.length
-            ? constructUrlPatternObjects(options.exclude)
-            : [];
-        const urlPatternObjects = options.include?.length
-            ? constructUrlPatternObjects(options.include)
-            : [];
+        const urlExcludePatternObjects = options.exclude?.length ? constructUrlPatternObjects(options.exclude) : [];
+        const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
         // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
         // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
-        const enqueueStrategyPatterns = options.baseUrl
-            ? buildEnqueueStrategyPatterns(options.baseUrl, strategy)
-            : [];
-        const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
-        const maxCrawlDepth = this.maxCrawlDepth;
-        const validateRequestUserData = this.validateRequestUserData.bind(this);
+        const enqueueStrategyPatterns = options.baseUrl ? buildEnqueueStrategyPatterns(options.baseUrl, strategy) : [];
+        const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
+        const maxCrawlDepth = this.#maxCrawlDepth;
+        const validateRequestUserData = this.#validateRequestUserData.bind(this);
         const allSkipped = [];
+        // A skipped source (which can carry arbitrary userData) is only retained if something reads it -
+        // otherwise the URL alone is enough to build the callback argument and to log with.
+        const hasSkippedRequestCallback = this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;
+        const keepSkippedSource = (source) => (hasSkippedRequestCallback ? source : source.url);
+        const reportSkippedRequests = async () => {
+            const skippedRequests = allSkipped.splice(0);
+            if (skippedRequests.length === 0) {
+                return;
+            }
+            const skippedRobotsUrls = skippedRequests
+                .filter((s) => s.reason === 'robotsTxt')
+                .map(({ source }) => (typeof source === 'string' ? source : source.url));
+            if (skippedRobotsUrls.length > 0) {
+                this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
+            }
+            // Only log the limit message when an explicit `limit` was passed (not the internal
+            // `maxRequestsPerCrawl`-derived one), and only once per call.
+            if (options.limit !== undefined && skippedRequests.some((s) => s.reason === 'limit')) {
+                this.log.info(requestLimit === options.limit
+                    ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
+                    : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
+            }
+            await Promise.all(skippedRequests.map(async ({ source, reason }) => {
+                const args = createSkippedRequestArgs(source, reason);
+                await this.#handleSkippedRequest(args);
+                await options.onSkippedRequest?.(args);
+            }));
+        };
         async function* filteredRequests() {
             for await (const request of requests) {
                 const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
@@ -1103,24 +1156,24 @@ export class BasicCrawler {
                     continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
                 }
                 if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
-                    allSkipped.push({ url: requestOptions.url, reason: 'depth' });
+                    allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });
                     continue;
                 }
                 if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
-                    allSkipped.push({ url: requestOptions.url, reason: 'robotsTxt' });
+                    allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });
                     continue;
                 }
-                const onSkippedFilterUrl = (url) => allSkipped.push({ url, reason: 'filters' });
+                const onSkippedByFilter = (opts) => allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });
                 // Filter by user patterns first (with exclude)...
-                let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedFilterUrl);
+                let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
                 // ...then filter by the enqueue strategy (making this an AND check)
-                filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedFilterUrl);
+                filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedByFilter);
                 if (filtered.length === 0) {
                     continue;
                 }
                 let [finalOptions] = filtered;
                 if (options.transformRequestFunction) {
-                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ url: r.url, reason: r.skippedReason ?? 'transform' }));
+                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ source: keepSkippedSource(r), reason: 'transform' }));
                     if (transformed.length === 0) {
                         continue;
                     }
@@ -1140,31 +1193,23 @@ export class BasicCrawler {
         // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
         // explicit `limit` option)
         for (const request of result.requestsOverLimit ?? []) {
-            allSkipped.push({ url: typeof request === 'string' ? request : request.url, reason: 'limit' });
+            allSkipped.push({ source: request, reason: 'limit' });
         }
-        if (allSkipped.length > 0) {
-            const skippedRobotsUrls = allSkipped.filter((s) => s.reason === 'robotsTxt').map((s) => s.url);
-            if (skippedRobotsUrls.length > 0) {
-                this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
-            }
-            // Only log the limit message when an explicit `limit` was passed (not the internal
-            // `maxRequestsPerCrawl`-derived one), and only once per call.
-            if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
-                this.log.info(requestLimit === options.limit
-                    ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
-                    : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
-            }
-            await Promise.all(allSkipped.map(async ({ url, reason }) => {
-                await this.handleSkippedRequest({ url, reason });
-                await options.onSkippedRequest?.({ url, reason });
-            }));
-        }
-        return result;
+        await reportSkippedRequests();
+        const waitForAllRequestsToBeAdded = result.waitForAllRequestsToBeAdded.then(async (addedRequests) => {
+            await reportSkippedRequests();
+            return addedRequests;
+        });
+        // Keep callback failures observable to callers that await this promise without emitting an unhandled rejection
+        // when callers intentionally leave background additions running, matching `drainRequestBatches` behavior.
+        void waitForAllRequestsToBeAdded.catch(() => { });
+        return { ...result, waitForAllRequestsToBeAdded };
     }
     /**
      * Pushes data to the specified {@link Dataset}, or the default crawler {@link Dataset} by calling {@link Dataset.pushData}.
      */
     async pushData(data, datasetIdentifier) {
+        tryCancel();
         const dataset = await this.getDataset(datasetIdentifier);
         return dataset.pushData(data);
     }
@@ -1233,31 +1278,25 @@ export class BasicCrawler {
      * Initializes the crawler.
      */
     async init() {
+        const run = this.#liveRun;
+        if (run === undefined) {
+            throw new Error('There is no crawl in progress - `run()` is the only thing that starts one.');
+        }
         const eventManager = serviceLocator.getEventManager();
         if (!eventManager.isInitialized()) {
             await eventManager.init();
-            this.#closeEvents = true;
+            run.ownsEventManager = true;
         }
         // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
         // floored per request so it will not actually cut them short, but the configured value is then effectively
         // ignored, which is worth flagging. Checked here (not in the constructor) because a subclass sets its
         // navigation timeout only after `super()`.
-        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(undefined);
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(undefined);
         if (this.internalTimeoutMillis < phasesMillis) {
             this.log.warning(`CRAWLEE_INTERNAL_TIMEOUT (${this.internalTimeoutMillis / 1000}s) is shorter than the navigation ` +
                 `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
                 `so it does not cut them short.`);
         }
-        // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
-        // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
-        // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
-        this.#concurrencySystemDep = this.#resolveConcurrencySystem();
-        await this.#concurrencySystemDep.ifOwned((system) => system.start());
-        this.#autoscaledPool = new AutoscaledPool({
-            ...this.taskLoopOptions,
-            concurrencySystem: this.#concurrencySystemDep.value,
-            consumer: this.identity,
-        });
         await this.getRequestManager();
     }
     /**
@@ -1274,9 +1313,9 @@ export class BasicCrawler {
      * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
      * whichever is larger: the configured internal timeout, or this request's combined phase budget.
      */
-    async withRequestTimeout(crawlingContext, work) {
+    async #withRequestTimeout(crawlingContext, work) {
         const { request } = crawlingContext;
-        const phasesMillis = this.getNavigationTimeoutMillis() + this.resolveRequestHandlerTimeoutMillis(request.label);
+        const phasesMillis = this.getNavigationTimeoutMillis() + this.#resolveRequestHandlerTimeoutMillis(request.label);
         const timeoutMillis = Math.max(this.internalTimeoutMillis, phasesMillis);
         await raceWithTimeout(crawlingContext, work, { timeoutMillis, requestId: request.id });
     }
@@ -1287,14 +1326,14 @@ export class BasicCrawler {
      * @param label The request's route label, or `undefined` for the default route / no specific request.
      * @param fallbackMillis Timeout to use when no route overrides it.
      */
-    resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.requestHandlerTimeoutMillis) {
-        return this.getRouteTimeoutMillis(label) ?? fallbackMillis;
+    #resolveRequestHandlerTimeoutMillis(label, fallbackMillis = this.#requestHandlerTimeoutMillis) {
+        return this.#getRouteTimeoutMillis(label) ?? fallbackMillis;
     }
     /**
      * The timeout the router route with the given label asked for, or `undefined` when it did not override one
      * (or the request handler is not a router at all).
      */
-    getRouteTimeoutMillis(label) {
+    #getRouteTimeoutMillis(label) {
         const getTimeoutSecs = this.requestHandler.getTimeoutSecs;
         if (typeof getTimeoutSecs !== 'function') {
             return undefined;
@@ -1303,14 +1342,15 @@ export class BasicCrawler {
         return timeoutSecs === undefined ? undefined : timeoutSecs * 1000;
     }
     async runRequestHandler(crawlingContext) {
-        const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
+        const timeoutMillis = this.#resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
         await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
     }
     /**
      * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
-     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
-     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
+     * normal return says nothing about success. `handleRequest` commits on success.
      */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- accessed by tests
     async runInStorageTransaction(callback) {
         if (!this.#transactionalStorageEnabled) {
             return callback();
@@ -1327,7 +1367,7 @@ export class BasicCrawler {
         }
         finally {
             if (transaction.state === 'open') {
-                // `handleRequest` commits or rolls back on every normal path, so an open transaction on
+                // `handleRequest` commits or `handleRequestFailure` rolls back on every normal path, so an open transaction on
                 // a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
                 // expected. Either way, discard the unvalidated writes; only the former is worth flagging.
                 if (!threw) {
@@ -1350,7 +1390,7 @@ export class BasicCrawler {
             throw new SessionError(`Request blocked - received ${statusCode} status code.`);
         }
     }
-    async isAllowedBasedOnRobotsTxtFile(url) {
+    async #isAllowedBasedOnRobotsTxtFile(url) {
         if (!this.#respectRobotsTxtFile) {
             return true;
         }
@@ -1359,45 +1399,52 @@ export class BasicCrawler {
         if (robotsTxtFile) {
             const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
             if (crawlDelay !== undefined) {
-                this.applyCrawlDelay(url, crawlDelay);
+                this.#applyCrawlDelay(url, crawlDelay);
             }
         }
         return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
     }
     /**
-     * Records an HTTP 429 against the URL's domain so the request manager can pace the retry.
+     * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
      *
      * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
-     * @returns `true` if a manager took responsibility for the delay, in which case the caller should throw
+     * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
      *  {@link RequestThrottledError} rather than treating the response as a blocked session.
      */
     recordDomainRateLimit(url, retryAfterHeader) {
-        if (supportsDomainThrottling(this.requestManager) &&
-            this.requestManager.recordDomainDelay(url, parseRetryAfterHeader(retryAfterHeader))) {
+        if (this.requestManager?.recordPacingSignal({
+            reason: 'rateLimited',
+            url,
+            waitMs: parseRetryAfterHeader(retryAfterHeader) ?? undefined,
+        })) {
             return true;
         }
         const domain = hostnameOrUrl(url);
-        this.logOncePerRun(`rateLimitNotThrottled:${domain}`, `"${domain}" responded with HTTP 429 (Too Many Requests), but nothing is set up to back off from it, ` +
-            'so the response is handled like any other, with no per-domain delay. ' +
-            `Pass a \`ThrottlingRequestManager\` as \`requestManager\` and include "${domain}" in its \`domains\` ` +
-            'option to honour `Retry-After` and apply exponential backoff instead.', 'warning');
+        this.log.warningOnce(`"${domain}" responded with HTTP 429 (Too Many Requests), but the crawler's request manager does not ` +
+            'pace that domain, so the response is handled like any other, with no per-domain delay. Set ' +
+            `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
+            '`requestManager`, to honour `Retry-After` and apply exponential backoff instead.');
         return false;
     }
-    /**
-     * Hands a robots.txt `Crawl-delay` to the request manager, warning if nothing is able to honour it.
-     *
-     * The warning is driven by whether the delay was actually accepted rather than by the type of the manager,
-     * because a manager that does throttle still drops the delay for a domain missing from its `domains` list.
-     */
-    applyCrawlDelay(url, delaySeconds) {
-        if (supportsDomainThrottling(this.requestManager) && this.requestManager.setCrawlDelay(url, delaySeconds)) {
+    /** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
+    #applyCrawlDelay(url, delaySeconds) {
+        // robots.txt is per-origin; `hostname` is the closest pacing scope and errs wide (http and https to one
+        // host share a clock).
+        if (this.requestManager?.recordPacingSignal({
+            reason: 'minInterval',
+            url,
+            intervalMs: delaySeconds * 1000,
+            scope: 'hostname',
+        })) {
             return;
         }
         const domain = hostnameOrUrl(url);
-        this.logOncePerRun(`crawlDelayIgnored:${domain}`, `robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but nothing is set up to honour it, ` +
-            'so requests to that domain will not be paced. Pass a `ThrottlingRequestManager` as `requestManager` ' +
-            `and include "${domain}" in its \`domains\` option to enforce the delay.`, 'warning');
+        this.log.warningOnce(`robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but the crawler's request ` +
+            'manager does not pace that domain, so its requests will not be paced. Set ' +
+            `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
+            '`requestManager`.');
     }
+    /** @internal */
     async getRobotsTxtFileForUrl(url) {
         if (!this.#respectRobotsTxtFile) {
             return undefined;
@@ -1417,10 +1464,10 @@ export class BasicCrawler {
             return undefined;
         }
     }
-    async pauseOnMigration() {
-        if (this.#autoscaledPool) {
-            // if run wasn't called, this is going to crash
-            await this.#autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
+    async #pauseOnMigration() {
+        const run = this.#dispatchingRun;
+        if (run) {
+            await run.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
                 if (err.message.includes('running tasks did not finish')) {
                     this.log.error('The crawler was paused due to migration to another host, ' +
                         "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -1430,37 +1477,21 @@ export class BasicCrawler {
                 }
             });
         }
-        const requestManagerPersistPromise = (async () => {
-            // The request manager persists its read-only loader's state, if it has one that supports persistence
-            // (e.g. a tandem wrapping a `RequestList`). For a plain `RequestQueue`, this is a no-op.
-            if (this.requestManager?.persistState) {
-                if (await this.requestManager.isFinished())
-                    return;
-                await this.requestManager.persistState().catch((err) => {
-                    if (err.message.includes('Cannot persist state.')) {
-                        this.log.error("The crawler attempted to persist its request list's state and failed due to missing or " +
-                            'invalid configuration. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
-                            'constructor to ensure your crawling state is persisted through host migrations and restarts.');
-                    }
-                    else {
-                        this.log.exception(err, 'An unexpected error occurred when the crawler ' +
-                            "attempted to persist its request list's state.");
-                    }
-                });
-            }
-        })();
-        await Promise.all([requestManagerPersistPromise, this.statistics.persistState?.()]);
+        // Captures the statistics changed while draining above. No PERSIST_STATE event is tied to the drain:
+        // the periodic one is unrelated, and the one the platform emits on migration arrives before any of this.
+        await this.statistics.persistState?.();
     }
     /**
      * Fetches the next request to process from the underlying request provider.
      */
-    async fetchNextRequest() {
+    async #fetchNextRequest() {
         if (this.requestManager === undefined) {
             throw new Error(`fetchNextRequest called on an uninitialized crawler`);
         }
         return this.requestManager.fetchNextRequest();
     }
-    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async handleRequest(crawlingContext, requestSource, request) {
         // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
         // which already failed the request in `runTaskFunction`. Bail before running the handler so it does not
@@ -1468,68 +1499,88 @@ export class BasicCrawler {
         if (crawlingContext[timeoutExpiredKey]?.()) {
             return;
         }
-        const statisticsId = request.id || request.uniqueKey;
+        request.state = RequestState.REQUEST_HANDLER;
+        await this.runRequestHandler(crawlingContext);
+        // Commit *before* marking the request as handled, so a commit failure fails the request and
+        // it is retried. This also closes the transaction, so everything below passes through.
         // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
-        const transaction = currentStorageTransaction();
-        let isRequestLocked = true;
+        await currentStorageTransaction()?.commit();
+        await this.#timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        this.statistics.recordRequestSuccess(request.id || request.uniqueKey, request.retryCount);
+        // reclaim session if request finishes successfully
+        request.state = RequestState.DONE;
+        crawlingContext.session.markGood();
+    }
+    /**
+     * Handles a request that failed anywhere in the context pipeline - a middleware (e.g. navigation or its hooks)
+     * or `handleRequest`. Runs inside the pipeline call, so the cleanups (e.g. closing the page) only fire after the
+     * error handler has seen the context.
+     */
+    async #handleRequestFailure(rawError, crawlingContext, requestSource, request) {
+        // Roll back *before* any error handler runs - error handlers write to real storage precisely
+        // because the transaction is already closed. A no-op when the commit in `handleRequest` succeeded.
+        currentStorageTransaction()?.rollback();
+        const resumeRequestTimeout = crawlingContext[suspendTimeoutKey]?.();
         try {
-            request.state = RequestState.REQUEST_HANDLER;
-            await this.runRequestHandler(crawlingContext);
-            // Commit *before* marking the request as handled, so a commit failure fails the request and
-            // it is retried. This also closes the transaction, so everything below passes through.
-            await transaction?.commit();
-            await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-            isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
-            this.statistics.finishJob(statisticsId, request.retryCount);
-            // reclaim session if request finishes successfully
-            request.state = RequestState.DONE;
-            crawlingContext.session.markGood();
-        }
-        catch (rawError) {
-            // Roll back *before* any error handler runs - error handlers write to real storage precisely
-            // because the transaction is already closed. A no-op when the commit above succeeded.
-            transaction?.rollback();
-            const err = this.unwrapError(rawError);
-            try {
-                request.state = RequestState.ERROR_HANDLER;
-                await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, crawlingContext, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                if (!(err instanceof CriticalError)) {
-                    isRequestLocked = false; // requestFunctionErrorHandler calls either markRequestAsHandled or reclaimRequest
-                }
+            const unwrappedError = this.#unwrapError(rawError);
+            // A failing or timed out `onSkippedRequest` makes this an ordinary failure, so the request gets retried.
+            const err = unwrappedError instanceof ContextPipelineInterruptedError
+                ? await this.#trySkipRequest(unwrappedError, request, requestSource)
+                : unwrappedError;
+            if (err) {
+                await this.#runErrorHandler(err, crawlingContext, requestSource, request);
+            }
+        }
+        finally {
+            resumeRequestTimeout?.();
+        }
+    }
+    async #runErrorHandler(err, context, requestSource, request) {
+        const retryCountBefore = request.retryCount;
+        try {
+            request.state = RequestState.ERROR_HANDLER;
+            await addTimeoutToPromise(async () => this.requestFunctionErrorHandler(err, context, request, requestSource), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            // The error handler may have set `SKIPPED`; TS keeps the narrowing from the assignment above.
+            if (request.state !== RequestState.SKIPPED) {
                 request.state = RequestState.DONE;
             }
-            catch (secondaryError) {
-                const unwrappedSecondaryError = this.unwrapError(secondaryError);
-                if (!unwrappedSecondaryError.triggeredFromUserHandler &&
-                    // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
-                    !(unwrappedSecondaryError instanceof CriticalError)) {
-                    const apifySpecific = process.env.APIFY_IS_AT_HOME
-                        ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
-                        : '';
-                    this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
-                        `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
+        }
+        catch (secondaryError) {
+            const unwrappedSecondaryError = this.#unwrapError(secondaryError);
+            const timedOut = unwrappedSecondaryError instanceof TimeoutError;
+            request.state = RequestState.ERROR;
+            if (timedOut) {
+                // Not worth crashing the crawler over. The request's state is unknown, so it gets retried - and the
+                // retry is counted, so that an error handler that always times out does not retry it forever.
+                this.log.warning(unwrappedSecondaryError.message);
+                if (request.retryCount === retryCountBefore) {
+                    request.retryCount++;
                 }
-                request.state = RequestState.ERROR;
-                throw unwrappedSecondaryError;
             }
-            // decrease the session score if the request fails (but the error handler did not throw);
-            // skip when the error is a SessionError, which already retired the session
-            if (!this.errorAbsolvesSession(err)) {
-                crawlingContext.session.markBad();
+            else if (!(unwrappedSecondaryError instanceof CriticalError)) {
+                // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
+                const apifySpecific = process.env.APIFY_IS_AT_HOME
+                    ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
+                    : '';
+                this.log.exception(unwrappedSecondaryError, 'An exception occurred during handling of failed request. ' +
+                    `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
             }
-        }
-        finally {
-            // Safety net - return the request to the queue if nobody managed to mark it as handled
-            // or reclaim it before (e.g. after a CriticalError). Reclaiming a request that is no longer
-            // in progress is a harmless no-op on the storage backend.
-            if (isRequestLocked && requestSource instanceof RequestQueue) {
-                try {
-                    await requestSource.reclaimRequest(request);
-                }
-                catch {
-                    // The request was never in progress, or could not be reclaimed. Either way it's fine.
-                }
+            // Safety net - the error handler neither marked the request as handled nor reclaimed it (e.g. after a
+            // CriticalError). Reclaiming a request that is no longer in progress is a harmless no-op on the storage backend.
+            try {
+                await addTimeoutToPromise(async () => requestSource.reclaimRequest(request), this.internalTimeoutMillis, `Reclaiming request ${request.url} (${request.id}) timed out.`);
+            }
+            catch {
+                // The request was never in progress, or could not be reclaimed. Either way it's fine.
             }
+            if (!timedOut) {
+                throw unwrappedSecondaryError;
+            }
+        }
+        // decrease the session score if the request fails (but the error handler did not throw);
+        // skip when the error is a SessionError, which already retired the session
+        if (!this.#errorAbsolvesSession(err)) {
+            context.session?.markBad();
         }
     }
     /**
@@ -1541,6 +1592,11 @@ export class BasicCrawler {
             if (typeof request === 'string') {
                 yield { url: request, crawlDepth: newRequestDepth };
             }
+            else if (request instanceof Request) {
+                const crawlee = (request.userData.__crawlee ??= {});
+                crawlee.crawlDepth ??= newRequestDepth;
+                yield request;
+            }
             else {
                 request.crawlDepth ??= newRequestDepth;
                 yield request;
@@ -1551,7 +1607,7 @@ export class BasicCrawler {
      * Run async callback with given timeout and retry. Returns the result of the callback.
      * @ignore
      */
-    async timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
+    async #timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
         try {
             return await addTimeoutToPromise(handler, timeout, error);
         }
@@ -1559,32 +1615,34 @@ export class BasicCrawler {
             if (retried <= maxRetries) {
                 // we retry on any error, not just timeout
                 this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
-                return this.timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
+                return this.#timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
             }
             throw e;
         }
     }
     /**
-     * Returns true if either RequestList or RequestQueue have a request ready for processing.
+     * Whether the request manager has a request ready for processing. A manager that is only `waiting` also gets a
+     * wake-up scheduled, so a paced crawl resumes on its clock rather than on the task loop's polling interval.
      */
-    async isTaskReadyFunction() {
-        return this.requestManager !== undefined && !(await this.requestManager.isEmpty());
-    }
-    /**
-     * Returns true if both RequestList and RequestQueue have all requests finished.
-     */
-    async defaultIsFinishedFunction() {
-        return !this.requestManager || (await this.requestManager.isFinished());
+    async #isTaskReadyFunction() {
+        if (this.requestManager === undefined) {
+            return false;
+        }
+        const state = await this.requestManager.checkReadiness();
+        if (state.status === 'waiting' && state.readyAt !== undefined) {
+            this.#run?.scheduleWake(state.readyAt);
+        }
+        return state.status === 'ready';
     }
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
+     * RequestHandlerError, ContextPipelineInitializationError and ContextPipelineCleanupError wrap the actual error.
      */
-    unwrapError(error) {
+    #unwrapError(error) {
         if (error instanceof RequestHandlerError ||
             error instanceof ContextPipelineInitializationError ||
             error instanceof ContextPipelineCleanupError) {
-            return this.unwrapError(error.cause);
+            return this.#unwrapError(error.cause);
         }
         return error;
     }
@@ -1593,26 +1651,45 @@ export class BasicCrawler {
      *
      * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async requestFunctionErrorHandler(error, crawlingContext, request, source) {
         if (error instanceof RequestThrottledError) {
-            // The domain told us to come back later, so the request was never really attempted. Put it back
-            // without recording a failure - it costs neither a retry nor session reputation.
+            // The domain told us to come back later, so the request was never really attempted. Put it back where
+            // it was, without recording a failure - it costs neither a retry nor session reputation.
             this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
                 id: request.id,
                 url: request.url,
             });
-            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+            await source.reclaimRequest(request, { forefront: true });
             return;
         }
         request.pushErrorMessage(error);
         if (error instanceof CriticalError) {
             throw error;
         }
-        const shouldRetryRequest = this.canRequestBeRetried(request, error);
+        const shouldRetryRequest = this.#canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
             await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
-            await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
-            error);
+            try {
+                await this.#errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
+                error);
+            }
+            catch (handlerError) {
+                if (handlerError instanceof ContextPipelineInterruptedError) {
+                    const skipError = await this.#trySkipRequest(handlerError, request, source);
+                    if (!skipError) {
+                        return;
+                    }
+                    // Retry for the original error. Routing `skipError` through `errorHandler` again
+                    // would loop forever with a handler that always skips.
+                    this.log.warning(`Skipping request ${request.url} failed, retrying it instead.`, {
+                        error: skipError.message,
+                    });
+                }
+                else {
+                    throw handlerError;
+                }
+            }
             if (error instanceof SessionError) {
                 crawlingContext.session?.retire();
             }
@@ -1627,7 +1704,7 @@ export class BasicCrawler {
                     url,
                     retryCount,
                 });
-                await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+                await source.reclaimRequest(request);
                 return;
             }
         }
@@ -1648,18 +1725,38 @@ export class BasicCrawler {
         // or failed more than retryCount times and will not be retried anymore.
         // Mark the request as failed and do not retry.
         await source.markRequestAsHandled(request);
-        this.statistics.failJob(request.id || request.uniqueKey, request.retryCount);
+        this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
         await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
+    /** Settles a skipped request. Returns the error that prevented that, leaving the request untouched if `onSkippedRequest` failed. */
+    async #trySkipRequest(error, request, source) {
+        try {
+            await addTimeoutToPromise(async () => this.#handleSkippedRequest({ request, reason: error.reason, message: error.skipMessage }), this.internalTimeoutMillis, `onSkippedRequest for ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            request.state = RequestState.SKIPPED;
+            request.noRetry = true;
+            await this.#timeoutAndRetry(async () => source.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
+            return undefined;
+        }
+        catch (settleError) {
+            return this.#unwrapError(settleError);
+        }
+    }
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
         const message = this.getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
-        if (this.failedRequestHandler) {
-            await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
+        try {
+            await this.#failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
             error);
         }
+        catch (handlerError) {
+            if (!(handlerError instanceof ContextPipelineInterruptedError))
+                throw handlerError;
+            this.log.warning(`skipRequest() has no effect in failedRequestHandler, request ${url} (${id}) has already failed.`);
+        }
     }
     /**
      * Resolves the most verbose error message from a thrown error
@@ -1674,7 +1771,8 @@ export class BasicCrawler {
         const baseDir = process.cwd();
         const userLine = stackLines.find((line) => line.includes(baseDir) && !line.includes('node_modules'));
         if (error instanceof TimeoutError) {
-            return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error; // stack in timeout errors does not really help
+            // stack in timeout errors does not really help
+            return (process.env.CRAWLEE_VERBOSE_LOG && error.stack) || error.message;
         }
         return process.env.CRAWLEE_VERBOSE_LOG || forceStack
             ? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
@@ -1684,10 +1782,12 @@ export class BasicCrawler {
      * Whether the session should be spared for this error - either because it was already retired, or because the
      * failure says nothing about the session (a rate limit is a property of the domain).
      */
-    errorAbsolvesSession(error) {
-        return error instanceof SessionError || error instanceof RequestThrottledError;
+    #errorAbsolvesSession(error) {
+        return (error instanceof SessionError ||
+            error instanceof RequestThrottledError ||
+            error instanceof ContextPipelineInterruptedError);
     }
-    canRequestBeRetried(request, error) {
+    #canRequestBeRetried(request, error) {
         // Request should never be retried, or the error encountered makes it not able to be retried.
         if (request.noRetry || error instanceof NonRetryableError) {
             return false;
@@ -1697,29 +1797,46 @@ export class BasicCrawler {
             return true;
         }
         // Ensure there are more retries available for the request
-        const maxRequestRetries = request.maxRetries ?? this.maxRequestRetries;
+        const maxRequestRetries = request.maxRetries ?? this.#maxRequestRetries;
         return request.retryCount < maxRequestRetries;
     }
     /**
-     * Stops the crawler immediately.
+     * Ends the run in progress immediately, without waiting for the requests already in flight.
      *
-     * This method doesn't wait for currently active requests to finish.
+     * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
+     * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
+     * released by {@link BasicCrawler.destroy|`destroy()`} instead.
      *
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     async teardown() {
+        const eventManager = serviceLocator.getEventManager();
+        const ownsEventManager = this.#run?.ownsEventManager === true;
+        // First, so that the pool writes its final state once - its teardown stops it listening before the
+        // persistence event below goes out.
+        await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown());
         // When this crawler initialized the event manager, its close() call emits
         // the final persistence event after the crawler-specific state has been
         // saved. External event managers still need an explicit event here.
-        if (!this.#closeEvents) {
-            serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
+        if (!ownsEventManager) {
+            eventManager.emit(EventType.PERSIST_STATE, { isMigrating: false });
         }
-        await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this.#closeEvents ?? false }));
-        if (this.#closeEvents) {
-            await serviceLocator.getEventManager().close();
+        if (ownsEventManager) {
+            await eventManager.close();
         }
-        await this.#autoscaledPool?.abort();
-        await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
+        await this.#run?.end();
+    }
+    /**
+     * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
+     * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
+     * could otherwise have run again.
+     */
+    async destroy() {
+        // An abandoned run - one whose `run()` never returned - would otherwise keep dispatching requests.
+        await this.#run?.end();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.destroy();
     }
     getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {
@@ -1728,7 +1845,7 @@ export class BasicCrawler {
         }
         return request.headers?.Cookie || request.headers?.cookie || '';
     }
-    requestMatchesEnqueueStrategy(request) {
+    #requestMatchesEnqueueStrategy(request) {
         // If `skipNavigation` was used, just return `true`
         try {
             // eslint-disable-next-line @typescript-eslint/no-unused-expressions
@@ -1741,8 +1858,7 @@ export class BasicCrawler {
             throw err;
         }
         const { url, loadedUrl } = request;
-        // eslint-disable-next-line dot-notation -- private access
-        const strategy = request['enqueueStrategy'];
+        const strategy = request.enqueueStrategy;
         // No strategy set, so we assume it matches, or it was added outside of enqueueLinks
         if (!strategy) {
             return true;