@crawlee/basic 4.0.0-beta.98 → 4.0.0-rc.0

package/internals/basic-crawler.js

@@ -1,21 +1,21 @@
-import { writeFile } from 'node:fs/promises';
+import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, } from '@crawlee/core';
-import { FetchHttpClient } from '@crawlee/http-client';
-import { isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
-import { stringify } from 'csv-stringify/sync';
-import { ensureDir, writeJSON } from 'fs-extra/esm';
-import ow, { ArgumentError } from 'ow';
+import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, parseArgument, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, supportsDomainThrottling, Router, schemas, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
+import { isAsyncIterable, isIterable, ROTATE_PROXY_ERRORS } from '@crawlee/utils/internal';
+import { RobotsTxtFile } from '@crawlee/utils';
 import { getDomain } from 'tldts';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { addTimeoutToPromise, extendTimeout, TimeoutError } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
 import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
-class LazyDefaultHttpClient {
-    _delegatePromise;
+class LazyDefaultHttpClient extends BaseHttpClient {
+    #delegatePromise;
     constructor(options) {
-        this._delegatePromise = import('@crawlee/impit-client')
+        super(options);
+        this.#delegatePromise = import('@crawlee/impit-client')
             .then(({ ImpitHttpClient }) => new ImpitHttpClient(options))
             .catch(() => {
             (options?.logger ?? log).warning('Optional dependency @crawlee/impit-client is not installed. ' +
@@ -23,8 +23,11 @@ class LazyDefaultHttpClient {
             return new FetchHttpClient(options);
         });
     }
+    fetch() {
+        throw new Error('LazyDefaultHttpClient delegates `sendRequest` entirely; `fetch` is never called.');
+    }
     async sendRequest(...args) {
-        return (await this._delegatePromise).sendRequest(...args);
+        return (await this.#delegatePromise).sendRequest(...args);
     }
 }
 /**
@@ -40,6 +43,33 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;
 const deferredCleanupKey = Symbol('deferredCleanup');
 // The request timeout plumbing (the window helper, the context symbols, and the race) lives in its own module.
 export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
+const urlPatternSchema = z.union([
+    z.string(),
+    z.instanceof(RegExp),
+    schemas.objectWithKeys(['glob']),
+    schemas.objectWithKeys(['regexp']),
+]);
+// `looseObject` (rather than `strictObject`) lets subclasses forward their own extraction-only options
+// (e.g. `selector`) straight through without having to strip them out first.
+const addRequestsOptionsSchema = z.looseObject({
+    forefront: z.boolean().optional(),
+    cache: z.boolean().optional(),
+    waitForAllRequestsToBeAdded: z.boolean().optional(),
+    batchSize: schemas.anyNumber.optional(),
+    waitBetweenBatchesMillis: schemas.anyNumber.optional(),
+    maxNewRequests: schemas.anyNumber.optional(),
+    limit: schemas.anyNumber.optional(),
+    baseUrl: z.string().optional(),
+    userData: schemas.anyObject.optional(),
+    label: z.string().optional(),
+    sessionId: z.string().optional(),
+    skipNavigation: z.boolean().optional(),
+    include: schemas.arrayOf(urlPatternSchema, 'URL patterns').min(1).optional(),
+    exclude: schemas.arrayOf(urlPatternSchema, 'URL patterns').optional(),
+    transformRequestFunction: schemas.anyFunction.optional(),
+    strategy: z.enum(EnqueueStrategy).optional(),
+    onSkippedRequest: schemas.anyFunction.optional(),
+});
 export class BasicCrawler {
     static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
     /**
@@ -47,16 +77,22 @@ export class BasicCrawler {
      * request queue; subsequent ones get their own queue via a unique alias so they don't
      * collide.
      */
+    // kept as TS-private: tests reset the counter at runtime
     static instanceCount = 0;
     /**
      * Tracks crawler instances that accessed shared state without having an explicit id.
      * Used to detect and warn about multiple crawlers sharing the same state.
      */
-    static useStateAnonymousIndices = new Set();
+    static #useStateAnonymousIndices = new Set();
+    /** Backs the {@link BasicCrawler.statistics|`statistics`} getter. */
+    #statisticsDep;
     /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+     * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    stats;
+    get statistics() {
+        return this.#statisticsDep.value;
+    }
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
@@ -64,34 +100,34 @@ export class BasicCrawler {
      */
     requestManager;
     /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
-    sessionPoolDep;
+    #sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool() {
-        return this.sessionPoolDep.value;
+        return this.#sessionPoolDep.value;
     }
     /**
      * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
      * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
      * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
      */
-    ownedRequestQueue = OwnedOrInjected.resolve();
+    #ownedRequestQueue = OwnedOrInjected.resolve();
     /**
      * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
      * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
      * so it only needs to be applied once, at the first async access of the manager.
      */
-    requestManagerTimeoutsApplied = false;
+    #requestManagerTimeoutsApplied = false;
     /**
      * Resolves the governor for one run: either the injected
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
      * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
      */
-    resolveConcurrencySystem;
-    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    concurrencySystemDep;
+    #resolveConcurrencySystem;
+    /** As resolved by `init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
+    #concurrencySystemDep;
     /**
      * The concurrency governor this run is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
@@ -105,7 +141,7 @@ export class BasicCrawler {
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem() {
-        return this.concurrencySystemDep?.maybeValue;
+        return this.#concurrencySystemDep?.maybeValue;
     }
     /**
      * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
@@ -114,7 +150,7 @@ export class BasicCrawler {
      * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
      * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
      */
-    autoscaledPool;
+    #autoscaledPool;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -125,7 +161,7 @@ export class BasicCrawler {
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     router = Router.create();
-    _basicContextPipeline;
+    #basicContextPipeline;
     /**
      * The basic part of the context pipeline. Unlike the subclass pipeline, this
      * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
@@ -135,21 +171,21 @@ export class BasicCrawler {
      * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
      */
     get basicContextPipeline() {
-        if (this._basicContextPipeline === undefined) {
-            this._basicContextPipeline = this.buildBasicContextPipeline();
+        if (this.#basicContextPipeline === undefined) {
+            this.#basicContextPipeline = this.buildBasicContextPipeline();
         }
-        return this._basicContextPipeline;
+        return this.#basicContextPipeline;
     }
-    _contextPipeline;
+    #contextPipeline;
     get contextPipeline() {
-        if (this._contextPipeline === undefined) {
-            this._contextPipeline = this.buildFinalContextPipeline();
+        if (this.#contextPipeline === undefined) {
+            this.#contextPipeline = this.buildFinalContextPipeline();
         }
-        return this._contextPipeline;
+        return this.#contextPipeline;
     }
     running = false;
     hasFinishedBefore = false;
-    unexpectedStop = false;
+    #unexpectedStop = false;
     #log;
     get log() {
         return this.#log;
@@ -157,100 +193,114 @@ export class BasicCrawler {
     requestHandler;
     errorHandler;
     failedRequestHandler;
+    // kept as TS-private: tests read it at runtime
     requestHandlerTimeoutMillis;
     internalTimeoutMillis;
     maxRequestRetries;
     maxCrawlDepth;
-    sameDomainDelayMillis;
-    domainAccessedTime;
+    #sameDomainDelaySecs;
     maxRequestsPerCrawl;
     get handledRequestsCount() {
-        return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
+        return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;
     }
-    statusMessageLoggingInterval;
-    statusMessageCallback;
+    #statusMessageLoggingInterval;
+    #statusMessageCallback;
     blockedStatusCodes = new Set();
     additionalHttpErrorStatusCodes;
-    ignoreHttpErrorStatusCodes;
+    #ignoreHttpErrorStatusCodes;
     /**
      * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
      * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
      * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
      * settable here.
      */
+    // kept as TS-private: tests mutate it at runtime
     taskLoopOptions;
     httpClient;
     retryOnBlocked;
-    respectRobotsTxtFile;
+    #respectRobotsTxtFile;
+    /** Whether `runInStorageTransaction()` opens a transaction at all. */
+    #transactionalStorageEnabled;
+    /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
+    #storageWritePolicy;
     onSkippedRequest;
-    _closeEvents;
-    loggedPerRun = new Set();
-    robotsTxtFileCache;
+    #closeEvents;
+    #loggedPerRun = new Set();
+    #robotsTxtFileCache;
     identity;
-    contextPipelineOptions;
+    #contextPipelineOptions;
     static optionsShape = {
-        contextPipelineBuilder: ow.optional.object,
-        extendContext: ow.optional.function,
-        requestList: ow.optional.object.validate(validators.requestList),
-        requestQueue: ow.optional.object.validate(validators.requestQueue),
+        contextPipelineBuilder: schemas.anyObject.optional(),
+        extendContext: schemas.anyFunction.optional(),
+        requestList: validators.requestList.optional(),
+        requestQueue: validators.requestQueue.optional(),
+        requestManager: validators.requestManager.optional(),
         // Subclasses override this function instead of passing it
         // in constructor, so this validation needs to apply only
         // if the user creates an instance of BasicCrawler directly.
-        requestHandler: ow.optional.function,
-        requestHandlerTimeoutSecs: ow.optional.number,
-        errorHandler: ow.optional.function,
-        failedRequestHandler: ow.optional.function,
-        maxRequestRetries: ow.optional.number,
-        sameDomainDelaySecs: ow.optional.number,
-        maxRequestsPerCrawl: ow.optional.number,
-        maxCrawlDepth: ow.optional.number,
-        taskLoopOptions: ow.optional.object,
-        concurrencySystem: ow.optional.object,
-        sessionPool: ow.optional.object.validate(validators.sessionPool),
-        proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
-        statusMessageLoggingInterval: ow.optional.number,
-        statusMessageCallback: ow.optional.function,
-        additionalHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
-        ignoreHttpErrorStatusCodes: ow.optional.array.ofType(ow.number),
-        blockedStatusCodes: ow.optional.array.ofType(ow.number),
-        retryOnBlocked: ow.optional.boolean,
-        respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object),
-        onSkippedRequest: ow.optional.function,
-        httpClient: ow.optional.object,
-        configuration: ow.optional.object,
-        storageBackend: ow.optional.object,
-        eventManager: ow.optional.object,
-        logger: ow.optional.object,
+        requestHandler: schemas.anyFunction.optional(),
+        requestHandlerTimeoutSecs: schemas.anyNumber.optional(),
+        errorHandler: schemas.anyFunction.optional(),
+        failedRequestHandler: schemas.anyFunction.optional(),
+        maxRequestRetries: schemas.anyNumber.default(3),
+        sameDomainDelaySecs: schemas.anyNumber.default(0),
+        maxRequestsPerCrawl: schemas.anyNumber.optional(),
+        maxCrawlDepth: schemas.anyNumber.optional(),
+        // No zod default — subclasses provide their own fallback (e.g. HTTP-optimized pool options).
+        taskLoopOptions: schemas.anyObject.optional(),
+        concurrencySystem: schemas.anyObject.optional(),
+        sessionPool: validators.sessionPool.optional(),
+        proxyConfiguration: validators.proxyConfiguration.optional(),
+        statusMessageLoggingInterval: schemas.anyNumber.default(10),
+        statusMessageCallback: schemas.anyFunction.optional(),
+        additionalHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
+        ignoreHttpErrorStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').default(() => []),
+        blockedStatusCodes: schemas.arrayOf(schemas.anyNumber, 'numbers').optional(),
+        retryOnBlocked: z.boolean().default(false),
+        respectRobotsTxtFile: z.union([z.boolean(), schemas.anyObject]).default(false),
+        transactionalStorage: z
+            .union([z.boolean(), z.strictObject({ requestQueue: z.enum(['deferred', 'writeThrough']).optional() })])
+            .optional(),
+        onSkippedRequest: schemas.anyFunction.optional(),
+        httpClient: schemas.httpClient.optional(),
+        configuration: z.instanceof(Configuration).optional(),
+        storageBackend: validators.storageBackend.optional(),
+        eventManager: z.instanceof(EventManager).optional(),
+        logger: validators.logger.optional(),
         // AutoscaledPool shorthands
-        minConcurrency: ow.optional.number,
-        maxConcurrency: ow.optional.number,
-        maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
-        keepAlive: ow.optional.boolean,
-        statisticsOptions: ow.optional.object,
-        id: ow.optional.string,
+        minConcurrency: schemas.anyNumber.optional(),
+        maxConcurrency: schemas.anyNumber.optional(),
+        maxRequestsPerMinute: schemas.anyNumber
+            .refine((value) => Number.isInteger(value) || value === Infinity, 'Expected an integer or infinite number')
+            .refine((value) => value >= 1, 'Expected a number greater than or equal to 1')
+            .optional(),
+        keepAlive: z.boolean().optional(),
+        statistics: schemas.anyObject.optional(),
+        id: z.string().optional(),
     };
+    static optionsSchema = z.strictObject(BasicCrawler.optionsShape);
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
     constructor(options = {}) {
-        ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
+        const parsedOptions = parseArgument(options, BasicCrawler.optionsSchema, 'BasicCrawlerOptions');
         const { 
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
         requestList, 
         // oxlint-disable-next-line typescript/no-deprecated -- still accepted and folded into `requestManager` for back-compat
-        requestQueue, requestManager, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes = [], ignoreHttpErrorStatusCodes = [], 
+        requestQueue, requestManager, maxRequestRetries, sameDomainDelaySecs, maxRequestsPerCrawl, maxCrawlDepth, taskLoopOptions = {}, concurrencySystem, keepAlive, sessionPool, proxyConfiguration, additionalHttpErrorStatusCodes, ignoreHttpErrorStatusCodes, 
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statisticsOptions, httpClient, id, } = options;
+        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked, respectRobotsTxtFile, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval, statusMessageCallback, statistics, httpClient, id, } = parsedOptions;
         // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
         // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
         // hammering a site.
         if (concurrencySystem !== undefined &&
             (minConcurrency !== undefined || maxConcurrency !== undefined || maxRequestsPerMinute !== undefined)) {
-            throw new ArgumentError('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
+            throw new Error('The `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts cannot be combined with ' +
                 '`concurrencySystem` - they configure the default `ConcurrencySystem` that a supplied one ' +
-                'replaces. Pass them to the `ConcurrencySystem` constructor instead.', this.constructor);
+                'replaces. Pass them to the `ConcurrencySystem` constructor instead.');
         }
         // Create per-crawler service locator if custom services were provided.
         // This wraps every method on the crawler instance so that calls to the global `serviceLocator`
@@ -262,14 +312,18 @@ export class BasicCrawler {
             eventManager ||
             logger ||
             (configuration !== undefined && configuration !== serviceLocator.getConfiguration())) {
-            const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageBackend, logger);
+            // Inherit the ambient locator's already-set services for anything not explicitly
+            // provided - e.g. only passing a `logger` must not detach the crawler from a globally
+            // configured storage backend.
+            const ambientServices = serviceLocator.getServicesIfSet();
+            const scopedServiceLocator = new ServiceLocator(configuration ?? ambientServices.configuration, eventManager ?? ambientServices.eventManager, storageBackend ?? ambientServices.storageBackend, logger ?? ambientServices.logger);
             serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);
         }
         try {
             serviceLocatorScope.enterScope();
-            this.contextPipelineOptions = {
-                contextPipelineBuilder: options.contextPipelineBuilder,
-                extendContext: options.extendContext,
+            this.#contextPipelineOptions = {
+                contextPipelineBuilder: parsedOptions.contextPipelineBuilder,
+                extendContext: parsedOptions.extendContext,
             };
             this.#log = serviceLocator.getLogger().child({ prefix: this.constructor.name });
             // Initialize the Configuration instance to avoid lazy loading in the components
@@ -280,6 +334,12 @@ export class BasicCrawler {
                 if (requestList !== undefined || requestQueue !== undefined) {
                     throw new Error('The `requestManager` option cannot be used in conjunction with `requestList` and/or `requestQueue`');
                 }
+                // Both would pace the same domains, from different keys and with no idea of one another.
+                if (sameDomainDelaySecs > 0 && supportsDomainThrottling(requestManager)) {
+                    throw new Error('The `sameDomainDelaySecs` option cannot be combined with a `requestManager` that throttles ' +
+                        'per domain on its own. Configure the delay on the manager instead, via the ' +
+                        '`minCrawlDelaySecs` option of `ThrottlingRequestManager`.');
+                }
                 this.requestManager = requestManager;
             }
             else if (requestList !== undefined && requestQueue !== undefined) {
@@ -298,13 +358,12 @@ export class BasicCrawler {
             }
             this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
             this.proxyConfiguration = proxyConfiguration;
-            this.statusMessageLoggingInterval = statusMessageLoggingInterval;
-            this.statusMessageCallback = statusMessageCallback;
-            this.domainAccessedTime = new Map();
-            this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
+            this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
+            this.#statusMessageCallback = statusMessageCallback;
+            this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
             this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
             this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
-            this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
+            this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
             this.requestHandler = requestHandler ?? this.router;
             this.failedRequestHandler = failedRequestHandler;
             this.errorHandler = errorHandler;
@@ -315,7 +374,11 @@ export class BasicCrawler {
                 this.requestHandlerTimeoutMillis = 60_000;
             }
             this.retryOnBlocked = retryOnBlocked;
-            this.respectRobotsTxtFile = respectRobotsTxtFile;
+            this.#respectRobotsTxtFile = respectRobotsTxtFile;
+            // The cast undoes ow's assertion signature, which mangles `boolean | object` unions.
+            const transactionalStorageOption = transactionalStorage;
+            this.#transactionalStorageEnabled = transactionalStorageOption !== false;
+            this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
             this.onSkippedRequest = onSkippedRequest;
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
@@ -323,20 +386,23 @@ export class BasicCrawler {
                     Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
-            this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
-            this.stats = new Statistics({
+            this.#sameDomainDelaySecs = sameDomainDelaySecs;
+            this.#statisticsDep = OwnedOrInjected.resolve(statistics, 
+            // A crawler-built default tracks the built-in fields only. A non-empty `StatisticStateExtension` can
+            // only be satisfied by an injected instance carrying the matching `state`, so this branch does
+            // not run in that case - hence the cast.
+            () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
                 id: this.identity.id,
-                ...statisticsOptions,
-            });
+            }));
             if (sessionPool && proxyConfiguration) {
                 this.log.warning('Both `sessionPool` and `proxyConfiguration` were provided to the crawler. ' +
                     'The `proxyConfiguration` is ignored - sessions from the supplied pool keep whatever ' +
                     '`proxyInfo` they were created with. Configure proxies on the pool instead, ' +
                     'e.g. via `addSession({ proxyInfo })` or a custom `createSessionFunction`.');
             }
-            this.sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
+            this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
                 createSessionFunction: async (opts) => new Session({
                     ...opts?.sessionOptions,
                     proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
@@ -364,27 +430,30 @@ export class BasicCrawler {
                     if (!source)
                         throw new Error('Request provider is not initialized!');
                     const request = await this.resolveRequest();
-                    if (!request || this.delayRequest(request, source)) {
+                    if (!request) {
                         return;
                     }
                     // Started here, rather than in `handleRequest`, so that a failure during context pipeline
                     // initialization (e.g. a browser page timing out before the request handler ever runs) is
                     // still accounted for by `failJob` below - which is a no-op without a matching `startJob`.
-                    this.stats.startJob(request.id || request.uniqueKey);
+                    this.statistics.startJob(request.id || request.uniqueKey);
                     const crawlingContext = { request };
                     try {
+                        // The transaction spans the whole pipeline call, covering the navigation hooks
+                        // and `extendContext` too; `handleRequest` drives its outcome explicitly.
+                        await this.runInStorageTransaction(async () => 
                         // Navigation, the navigation hooks and the request handler are timed individually, but the
                         // phases between them are not, so a request could still get stuck indefinitely. This is the
                         // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
                         await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
                             .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request)));
+                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request))));
                     }
                     catch (error) {
                         // ContextPipelineInterruptedError means the request was intentionally skipped
                         // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
                         if (error instanceof ContextPipelineInterruptedError) {
-                            this.stats.discardJob(request.id || request.uniqueKey);
+                            this.statistics.discardJob(request.id || request.uniqueKey);
                             await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
                             return;
                         }
@@ -399,7 +468,7 @@ export class BasicCrawler {
                             await this.requestFunctionErrorHandler(unwrappedError, crawlingContext, request, this.requestManager);
                             // SessionError already retired the session in `requestFunctionErrorHandler`;
                             // skip `markBad` to avoid double-counting usage/error score.
-                            if (!(unwrappedError instanceof SessionError)) {
+                            if (!this.errorAbsolvesSession(unwrappedError)) {
                                 crawlingContext.session?.markBad();
                             }
                             return;
@@ -418,7 +487,7 @@ export class BasicCrawler {
                             `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
                         return false;
                     }
-                    if (this.unexpectedStop) {
+                    if (this.#unexpectedStop) {
                         this.logOncePerRun('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
                             'Ongoing requests will be allowed to complete.');
                         return false;
@@ -432,10 +501,15 @@ export class BasicCrawler {
                             `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
                         return true;
                     }
-                    if (this.unexpectedStop) {
+                    if (this.#unexpectedStop) {
                         this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
                         return true;
                     }
+                    // Checked here because this runs only once nothing is in flight, which is exactly when a
+                    // crawl that cannot progress looks indistinguishable from one that is merely waiting.
+                    if (!keepAlive && supportsDomainThrottling(this.requestManager)) {
+                        await this.requestManager.assertNoStalledDomains();
+                    }
                     const isFinished = isFinishedFunction
                         ? await isFinishedFunction()
                         : await this.defaultIsFinishedFunction();
@@ -450,7 +524,7 @@ export class BasicCrawler {
                 log: this.log,
             };
             this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
-            this.resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
+            this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
                 minConcurrency,
                 maxConcurrency,
                 maxTasksPerMinute: maxRequestsPerMinute,
@@ -479,7 +553,7 @@ export class BasicCrawler {
      * @returns `true` if the status code is considered an error, `false` otherwise
      */
     isErrorStatusCode(status) {
-        const excludeError = this.ignoreHttpErrorStatusCodes.has(status);
+        const excludeError = this.#ignoreHttpErrorStatusCodes.has(status);
         const includeError = this.additionalHttpErrorStatusCodes.has(status);
         return (status >= 500 && !excludeError) || includeError;
     }
@@ -559,20 +633,16 @@ export class BasicCrawler {
         return { session, proxyInfo: session?.proxyInfo };
     }
     async createContextHelpers({ request, session }) {
-        const enqueueLinksWrapper = async (options) => {
-            const requestManager = await this.getRequestManager();
-            return await this.enqueueLinksWithCrawlDepth(options, request, requestManager);
-        };
         const addRequests = async (requests, options = {}) => {
             const newCrawlDepth = request.crawlDepth + 1;
             const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
-            await this.addRequests(requestsGenerator, options);
+            return await this.addRequests(requestsGenerator, options);
         };
         const sendRequest = createSendRequest(this.httpClient, request, session);
-        return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };
+        return { addRequests, sendRequest };
     }
     buildFinalContextPipeline() {
-        const subclassPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??
+        const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
             this.buildContextPipeline());
         // `extendContext` runs *before* the subclass navigation pipeline (which includes the
         // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
@@ -584,7 +654,7 @@ export class BasicCrawler {
         // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
         // casts below are sound because `buildFinalContextPipeline` is declared to return the fully
         // resolved `ExtendedContext` (= `Context & ContextExtension`).
-        const { extendContext } = this.contextPipelineOptions;
+        const { extendContext } = this.#contextPipelineOptions;
         let contextPipeline;
         if (extendContext !== undefined) {
             contextPipeline = ContextPipeline.create()
@@ -618,7 +688,7 @@ export class BasicCrawler {
      * @param error The error to check.
      */
     isProxyError(error) {
-        return ROTATE_PROXY_ERRORS.some((x) => this._getMessageFromError(error)?.includes(x));
+        return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error)?.includes(x));
     }
     /**
      * Sets the status message for the current crawler run.
@@ -636,7 +706,7 @@ export class BasicCrawler {
         // subscribe to `EventType.STATUS_MESSAGE` and propagate it to their status-reporting backend.
         // Setting the status message is not a storage concern, so we intentionally don't route it
         // through the storage client anymore.
-        serviceLocator.getEventManager().emit("statusMessage" /* EventType.STATUS_MESSAGE */, {
+        serviceLocator.getEventManager().emit(EventType.STATUS_MESSAGE, {
             crawlerId: this.identity.id,
             message,
             isStatusMessageTerminal: options.isStatusMessageTerminal,
@@ -644,11 +714,11 @@ export class BasicCrawler {
         });
     }
     getPeriodicLogger() {
-        let previousState = { ...this.stats.state };
+        let previousState = { ...this.statistics.state };
         const getOperationMode = () => {
-            const { requestsFailed } = this.stats.state;
+            const { requestsFailed } = this.statistics.state;
             const { requestsFailed: previousRequestsFailed } = previousState;
-            previousState = { ...this.stats.state };
+            previousState = { ...this.statistics.state };
             const failedDelta = requestsFailed - previousRequestsFailed;
             if (failedDelta > 0) {
                 return { mode: 'ERROR', failedDelta };
@@ -659,16 +729,16 @@ export class BasicCrawler {
             const { mode: operationMode, failedDelta } = getOperationMode();
             let message;
             if (operationMode === 'ERROR') {
-                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
+                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.#statusMessageLoggingInterval} seconds.`;
             }
             else {
                 const total = await this.requestManager?.getTotalCount();
-                message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
+                message = `Crawled ${this.statistics.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
-            if (this.statusMessageCallback) {
-                await this.statusMessageCallback({
+            if (this.#statusMessageCallback) {
+                await this.#statusMessageCallback({
                     crawler: this,
-                    state: this.stats.state,
+                    state: this.statistics.state,
                     previousState,
                     message,
                 });
@@ -676,7 +746,7 @@ export class BasicCrawler {
             }
             this.setStatusMessage(message);
         };
-        const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);
+        const interval = setInterval(log, this.#statusMessageLoggingInterval * 1e3);
         return { log, stop: () => clearInterval(interval) };
     }
     /**
@@ -704,17 +774,26 @@ export class BasicCrawler {
             // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
             // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
             const shouldPurge = purgeRequestQueue !== false;
-            const managerToPurge = this.ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
-            if (managerToPurge?.purge && shouldPurge) {
-                await managerToPurge.purge();
+            const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
+            if (shouldPurge) {
+                await managerToPurge?.purge?.();
+                // The per-domain queues a `sameDomainDelaySecs` wrapper created are the crawler's own, whatever
+                // sits underneath them - so they are emptied even when the manager they wrap is spared. Purging
+                // the wrapper itself has already covered them.
+                if (this.requestManager instanceof ThrottlingRequestManager && managerToPurge !== this.requestManager) {
+                    await this.requestManager.purgeDomainQueues();
+                }
             }
-            this.stats.reset();
-            await this.stats.resetStore();
-            await this.sessionPoolDep.ifOwned((pool) => pool.resetStore());
+            // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+            await this.#statisticsDep.ifOwned(async (stats) => {
+                stats.reset();
+                await stats.resetStore();
+            });
+            await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());
         }
-        this.unexpectedStop = false;
+        this.#unexpectedStop = false;
         this.running = true;
-        this.loggedPerRun.clear();
+        this.#loggedPerRun.clear();
         await purgeDefaultStorages({
             onlyPurgeOnce: true,
             storageBackend: serviceLocator.getStorageBackend(),
@@ -724,8 +803,8 @@ export class BasicCrawler {
             await this.addRequests(requests, addRequestsOptions);
         }
         try {
-            await this._init();
-            await this.stats.startCapturing();
+            await this.init();
+            await this.statistics.startCapturing();
         }
         catch (error) {
             // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
@@ -741,38 +820,38 @@ export class BasicCrawler {
         const sigintHandler = async () => {
             this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
             await this.pauseOnMigration();
-            await this.autoscaledPool.abort();
+            await this.#autoscaledPool.abort();
         };
         // Attach a listener to handle migration and aborting events gracefully.
         const boundPauseOnMigration = this.pauseOnMigration.bind(this);
         process.once('SIGINT', sigintHandler);
         const eventManager = serviceLocator.getEventManager();
-        eventManager.on("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
-        eventManager.on("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
+        eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
+        eventManager.on(EventType.ABORTING, boundPauseOnMigration);
         let stats = {};
         try {
-            await this.autoscaledPool.run();
+            await this.#autoscaledPool.run();
         }
         finally {
+            await this.statistics.stopCapturing();
             await this.teardown();
-            await this.stats.stopCapturing();
             process.off('SIGINT', sigintHandler);
-            eventManager.off("migrating" /* EventType.MIGRATING */, boundPauseOnMigration);
-            eventManager.off("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
-            const finalStats = this.stats.calculate();
+            eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
+            eventManager.off(EventType.ABORTING, boundPauseOnMigration);
+            const finalStats = this.statistics.calculate();
             stats = {
-                requestsFinished: this.stats.state.requestsFinished,
-                requestsFailed: this.stats.state.requestsFailed,
-                retryHistogram: this.stats.requestRetryHistogram,
+                requestsFinished: this.statistics.state.requestsFinished,
+                requestsFailed: this.statistics.state.requestsFailed,
+                retryHistogram: this.statistics.requestRetryHistogram,
                 ...finalStats,
             };
             this.log.info('Final request statistics:', stats);
-            if (this.stats.errorTracker.total !== 0) {
+            if (this.statistics.errorTracker.total !== 0) {
                 const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
                 this.log.info(`Error analysis:`, {
-                    totalErrors: this.stats.errorTracker.total,
-                    uniqueErrors: this.stats.errorTracker.getUniqueErrorCount(),
-                    mostCommonErrors: this.stats.errorTracker.getMostPopularErrors(3).map(prettify),
+                    totalErrors: this.statistics.errorTracker.total,
+                    uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
+                    mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
                 });
             }
             const client = serviceLocator.getStorageBackend();
@@ -787,7 +866,7 @@ export class BasicCrawler {
                 finished = true;
             }
             periodicLogger.stop();
-            this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
+            this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
             this.running = false;
             this.hasFinishedBefore = true;
         }
@@ -801,11 +880,11 @@ export class BasicCrawler {
      * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
     stop(reason = 'The crawler has been gracefully stopped.') {
-        if (this.unexpectedStop) {
+        if (this.#unexpectedStop) {
             return;
         }
         this.log.info(reason);
-        this.unexpectedStop = true;
+        this.#unexpectedStop = true;
     }
     /**
      * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
@@ -816,22 +895,22 @@ export class BasicCrawler {
      * throughout, since a shared one may still be serving other crawlers.
      */
     async pause(timeoutSecs) {
-        if (!this.autoscaledPool) {
+        if (!this.#autoscaledPool) {
             this.log.warning('Cannot pause a crawler that is not running.');
             return;
         }
-        await this.autoscaledPool.pause(timeoutSecs);
+        await this.#autoscaledPool.pause(timeoutSecs);
     }
     /**
      * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
      * again. A no-op on a crawler that is not paused.
      */
     resume() {
-        if (!this.autoscaledPool) {
+        if (!this.#autoscaledPool) {
             this.log.warning('Cannot resume a crawler that is not running.');
             return;
         }
-        this.autoscaledPool.resume();
+        this.#autoscaledPool.resume();
     }
     /**
      * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
@@ -841,11 +920,24 @@ export class BasicCrawler {
         if (!this.requestManager) {
             this.requestManager = await this.openOwnedRequestQueue();
         }
+        // Wrapped here rather than in the constructor, because the manager being wrapped may only be opened at
+        // this point - and because everything that enqueues goes through here first, so nothing slips past the
+        // wrapper into the queue it hides.
+        if (this.#sameDomainDelaySecs > 0 && !supportsDomainThrottling(this.requestManager)) {
+            this.requestManager = new ThrottlingRequestManager({
+                inner: this.requestManager,
+                domains: 'all',
+                minCrawlDelaySecs: this.#sameDomainDelaySecs,
+                // What `sameDomainDelaySecs` has always meant: one clock for a site, subdomains included.
+                throttleBy: 'registrableDomain',
+                persistStateKey: `CRAWLEE_THROTTLED_DOMAINS_${this.identity.id}`,
+            });
+        }
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
         // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
         // but guard so we do not re-issue it on every call.
-        if (!this.requestManagerTimeoutsApplied) {
-            this.requestManagerTimeoutsApplied = true;
+        if (!this.#requestManagerTimeoutsApplied) {
+            this.#requestManagerTimeoutsApplied = true;
             await this.applyRequestManagerTimeouts(this.requestManager);
         }
         return this.requestManager;
@@ -867,7 +959,7 @@ export class BasicCrawler {
         // subsequent instances get their own queue via a unique alias so they don't collide.
         const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
         const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
-        return this.ownedRequestQueue.set(requestQueue);
+        return this.#ownedRequestQueue.set(requestQueue);
     }
     /**
      * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
@@ -916,8 +1008,8 @@ export class BasicCrawler {
             const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
             return kvs.getAutoSavedValue(stateKey, defaultValue);
         }
-        BasicCrawler.useStateAnonymousIndices.add(this.identity.instanceIndex);
-        if (BasicCrawler.useStateAnonymousIndices.size > 1) {
+        BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);
+        if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
             serviceLocator
                 .getLogger()
                 .warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
@@ -938,19 +1030,23 @@ export class BasicCrawler {
         return Math.min(limit, explicitLimit ?? Infinity);
     }
     async handleSkippedRequest(options) {
-        if (options.reason === 'limit') {
-            this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
-                `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
-        }
-        if (options.reason === 'depth') {
-            this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
-        }
-        await this.onSkippedRequest?.(options);
+        // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
+        // transaction scope, which rolls back - so the skip bookkeeping must write directly.
+        await withDirectStorageAccess(async () => {
+            if (options.reason === 'limit') {
+                this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
+                    `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
+            }
+            if (options.reason === 'depth') {
+                this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
+            }
+            await this.onSkippedRequest?.(options);
+        });
     }
-    logOncePerRun(key, message) {
-        if (!this.loggedPerRun.has(key)) {
-            this.log.info(message);
-            this.loggedPerRun.add(key);
+    logOncePerRun(key, message, level = 'info') {
+        if (!this.#loggedPerRun.has(key)) {
+            this.log[level](message);
+            this.#loggedPerRun.add(key);
         }
     }
     /**
@@ -959,6 +1055,11 @@ export class BasicCrawler {
      * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
      * the `waitForAllRequestsToBeAdded` promise you get in the response object.
      *
+     * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
+     * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
+     * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
+     * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
+     *
      * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
      *
      * @param requests The requests to add
@@ -966,55 +1067,97 @@ export class BasicCrawler {
      */
     async addRequests(requests, options = {}) {
         await this.getRequestManager();
-        const requestLimit = await this.calculateEnqueuedRequestLimit();
-        const skippedBecauseOfRobots = new Set();
-        const skippedBecauseOfMaxCrawlDepth = new Set();
+        if (!isIterable(requests) && !isAsyncIterable(requests)) {
+            throw new Error(`Expected an iterable or async iterable, got ${getObjectType(requests)}`);
+        }
+        parseArgument(options, addRequestsOptionsSchema, 'EnqueueUrlsOptions');
+        // `label`/`userData` apply to every request this call produces, so a single upfront validation
+        // against the label's schema covers them all and fails the whole call fast, rather than failing
+        // lazily once the generator below is drained. Skipped when neither is set - each item still gets
+        // its own per-item validation below, and validating an absent label/userData here would spuriously
+        // check them against a registered default-route schema.
+        if (options.label !== undefined || options.userData !== undefined) {
+            await this.validateRequestUserData({ label: options.label, userData: options.userData });
+        }
+        const requestLimit = await this.calculateEnqueuedRequestLimit(options.limit);
+        const strategy = options.strategy ?? EnqueueStrategy.All;
+        const urlExcludePatternObjects = options.exclude?.length
+            ? constructUrlPatternObjects(options.exclude)
+            : [];
+        const urlPatternObjects = options.include?.length
+            ? constructUrlPatternObjects(options.include)
+            : [];
+        // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
+        // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
+        const enqueueStrategyPatterns = options.baseUrl
+            ? buildEnqueueStrategyPatterns(options.baseUrl, strategy)
+            : [];
         const isAllowedBasedOnRobotsTxtFile = this.isAllowedBasedOnRobotsTxtFile.bind(this);
         const maxCrawlDepth = this.maxCrawlDepth;
         const validateRequestUserData = this.validateRequestUserData.bind(this);
-        ow(requests, ow.object
-            .is((value) => isIterable(value) || isAsyncIterable(value))
-            .message((value) => `Expected an iterable or async iterable, got ${getObjectType(value)}`));
+        const allSkipped = [];
         async function* filteredRequests() {
             for await (const request of requests) {
-                const url = typeof request === 'string' ? request : request.url;
-                if (maxCrawlDepth !== undefined && request.crawlDepth > maxCrawlDepth) {
-                    skippedBecauseOfMaxCrawlDepth.add(url);
+                const [requestOptions] = createRequestOptions([typeof request === 'string' ? request : request], { ...options, strategy });
+                if (!requestOptions) {
+                    continue; // invalid URL, silently dropped (matches `createRequestOptions`'s own filtering)
+                }
+                if (maxCrawlDepth !== undefined && requestOptions.crawlDepth > maxCrawlDepth) {
+                    allSkipped.push({ url: requestOptions.url, reason: 'depth' });
                     continue;
                 }
-                if (await isAllowedBasedOnRobotsTxtFile(url)) {
-                    await validateRequestUserData(request);
-                    yield request;
+                if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {
+                    allSkipped.push({ url: requestOptions.url, reason: 'robotsTxt' });
+                    continue;
                 }
-                else {
-                    skippedBecauseOfRobots.add(url);
+                const onSkippedFilterUrl = (url) => allSkipped.push({ url, reason: 'filters' });
+                // Filter by user patterns first (with exclude)...
+                let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedFilterUrl);
+                // ...then filter by the enqueue strategy (making this an AND check)
+                filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedFilterUrl);
+                if (filtered.length === 0) {
+                    continue;
+                }
+                let [finalOptions] = filtered;
+                if (options.transformRequestFunction) {
+                    const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) => allSkipped.push({ url: r.url, reason: r.skippedReason ?? 'transform' }));
+                    if (transformed.length === 0) {
+                        continue;
+                    }
+                    [finalOptions] = transformed;
                 }
+                await validateRequestUserData(finalOptions);
+                yield new Request(finalOptions);
             }
         }
         const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
-            ...options,
+            forefront: options.forefront,
+            waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
+            batchSize: options.batchSize,
+            waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
             maxNewRequests: requestLimit,
         });
-        // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit)
-        const skippedBecauseOfLimit = result.requestsOverLimit ?? [];
-        if (skippedBecauseOfRobots.size > 0) {
-            this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, {
-                skipped: [...skippedBecauseOfRobots],
-            });
-        }
-        if (skippedBecauseOfRobots.size > 0 ||
-            skippedBecauseOfLimit.length > 0 ||
-            skippedBecauseOfMaxCrawlDepth.size > 0) {
-            await Promise.all([...skippedBecauseOfRobots]
-                .map((url) => {
-                return this.handleSkippedRequest({ url, reason: 'robotsTxt' });
-            })
-                .concat(skippedBecauseOfLimit.map((request) => {
-                const url = typeof request === 'string' ? request : request.url;
-                return this.handleSkippedRequest({ url, reason: 'limit' });
-            }), [...skippedBecauseOfMaxCrawlDepth].map((url) => {
-                return this.handleSkippedRequest({ url, reason: 'depth' });
-            })));
+        // Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an
+        // explicit `limit` option)
+        for (const request of result.requestsOverLimit ?? []) {
+            allSkipped.push({ url: typeof request === 'string' ? request : request.url, reason: 'limit' });
+        }
+        if (allSkipped.length > 0) {
+            const skippedRobotsUrls = allSkipped.filter((s) => s.reason === 'robotsTxt').map((s) => s.url);
+            if (skippedRobotsUrls.length > 0) {
+                this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`, { skipped: skippedRobotsUrls });
+            }
+            // Only log the limit message when an explicit `limit` was passed (not the internal
+            // `maxRequestsPerCrawl`-derived one), and only once per call.
+            if (options.limit !== undefined && allSkipped.some((s) => s.reason === 'limit')) {
+                this.log.info(requestLimit === options.limit
+                    ? `Skipping requests in this call due to the enqueueLinks limit of ${options.limit}.`
+                    : `Skipping requests in this call due to the remaining maxRequestsPerCrawl budget of ${requestLimit}, which is lower than the enqueueLinks limit of ${options.limit}.`);
+            }
+            await Promise.all(allSkipped.map(async ({ url, reason }) => {
+                await this.handleSkippedRequest({ url, reason });
+                await options.onSkippedRequest?.({ url, reason });
+            }));
         }
         return result;
     }
@@ -1067,6 +1210,7 @@ export class BasicCrawler {
                 const keys = options?.collectAllKeys
                     ? Array.from(new Set(items.flatMap(Object.keys)))
                     : Object.keys(items[0]);
+                const { stringify } = await import('csv-stringify/sync');
                 value = stringify([
                     keys,
                     ...items.map((item) => {
@@ -1074,13 +1218,13 @@ export class BasicCrawler {
                     }),
                 ]);
             }
-            await ensureDir(dirname(path));
+            await mkdir(dirname(path), { recursive: true });
             await writeFile(path, value);
             this.log.info(`Export to ${path} finished!`);
         }
         if (format === 'json') {
-            await ensureDir(dirname(path));
-            await writeJSON(path, items, { spaces: 4 });
+            await mkdir(dirname(path), { recursive: true });
+            await writeFile(path, `${JSON.stringify(items, null, 4)}\n`);
             this.log.info(`Export to ${path} finished!`);
         }
         return items;
@@ -1088,11 +1232,11 @@ export class BasicCrawler {
     /**
      * Initializes the crawler.
      */
-    async _init() {
+    async init() {
         const eventManager = serviceLocator.getEventManager();
         if (!eventManager.isInitialized()) {
             await eventManager.init();
-            this._closeEvents = true;
+            this.#closeEvents = true;
         }
         // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
         // floored per request so it will not actually cut them short, but the configured value is then effectively
@@ -1107,11 +1251,11 @@ export class BasicCrawler {
         // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
         // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
         // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
-        this.concurrencySystemDep = this.resolveConcurrencySystem();
-        await this.concurrencySystemDep.ifOwned((system) => system.start());
-        this.autoscaledPool = new AutoscaledPool({
+        this.#concurrencySystemDep = this.#resolveConcurrencySystem();
+        await this.#concurrencySystemDep.ifOwned((system) => system.start());
+        this.#autoscaledPool = new AutoscaledPool({
             ...this.taskLoopOptions,
-            concurrencySystem: this.concurrencySystemDep.value,
+            concurrencySystem: this.#concurrencySystemDep.value,
             consumer: this.identity,
         });
         await this.getRequestManager();
@@ -1162,10 +1306,44 @@ export class BasicCrawler {
         const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
         await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
     }
+    /**
+     * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
+     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
+     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     */
+    async runInStorageTransaction(callback) {
+        if (!this.#transactionalStorageEnabled) {
+            return callback();
+        }
+        const transaction = createStorageTransaction({
+            policy: this.#storageWritePolicy,
+            commitTimeoutMillis: this.internalTimeoutMillis,
+        });
+        let threw = true;
+        try {
+            const result = await transaction.run(callback);
+            threw = false;
+            return result;
+        }
+        finally {
+            if (transaction.state === 'open') {
+                // `handleRequest` commits or rolls back on every normal path, so an open transaction on
+                // a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
+                // expected. Either way, discard the unvalidated writes; only the former is worth flagging.
+                if (!threw) {
+                    this.log.error('Internal error: a storage transaction was still open after the request pipeline ' +
+                        'returned normally. Its writes are being discarded. Please report this.');
+                }
+                transaction.rollback();
+            }
+            // Unconditional: `failed` is a terminal state that the branch above never reaches.
+            transaction.dispose();
+        }
+    }
     /**
      * Handles blocked request
      */
-    _throwOnBlockedRequest(statusCode) {
+    throwOnBlockedRequest(statusCode) {
         if (this.retryOnBlocked)
             return;
         if (this.blockedStatusCodes.has(statusCode)) {
@@ -1173,25 +1351,65 @@ export class BasicCrawler {
         }
     }
     async isAllowedBasedOnRobotsTxtFile(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return true;
         }
         const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
-        const userAgent = typeof this.respectRobotsTxtFile === 'object' ? this.respectRobotsTxtFile?.userAgent : '*';
+        const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';
+        if (robotsTxtFile) {
+            const crawlDelay = robotsTxtFile.getCrawlDelay(userAgent);
+            if (crawlDelay !== undefined) {
+                this.applyCrawlDelay(url, crawlDelay);
+            }
+        }
         return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
     }
+    /**
+     * Records an HTTP 429 against the URL's domain so the request manager can pace the retry.
+     *
+     * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
+     * @returns `true` if a manager took responsibility for the delay, in which case the caller should throw
+     *  {@link RequestThrottledError} rather than treating the response as a blocked session.
+     */
+    recordDomainRateLimit(url, retryAfterHeader) {
+        if (supportsDomainThrottling(this.requestManager) &&
+            this.requestManager.recordDomainDelay(url, parseRetryAfterHeader(retryAfterHeader))) {
+            return true;
+        }
+        const domain = hostnameOrUrl(url);
+        this.logOncePerRun(`rateLimitNotThrottled:${domain}`, `"${domain}" responded with HTTP 429 (Too Many Requests), but nothing is set up to back off from it, ` +
+            'so the response is handled like any other, with no per-domain delay. ' +
+            `Pass a \`ThrottlingRequestManager\` as \`requestManager\` and include "${domain}" in its \`domains\` ` +
+            'option to honour `Retry-After` and apply exponential backoff instead.', 'warning');
+        return false;
+    }
+    /**
+     * Hands a robots.txt `Crawl-delay` to the request manager, warning if nothing is able to honour it.
+     *
+     * The warning is driven by whether the delay was actually accepted rather than by the type of the manager,
+     * because a manager that does throttle still drops the delay for a domain missing from its `domains` list.
+     */
+    applyCrawlDelay(url, delaySeconds) {
+        if (supportsDomainThrottling(this.requestManager) && this.requestManager.setCrawlDelay(url, delaySeconds)) {
+            return;
+        }
+        const domain = hostnameOrUrl(url);
+        this.logOncePerRun(`crawlDelayIgnored:${domain}`, `robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but nothing is set up to honour it, ` +
+            'so requests to that domain will not be paced. Pass a `ThrottlingRequestManager` as `requestManager` ' +
+            `and include "${domain}" in its \`domains\` option to enforce the delay.`, 'warning');
+    }
     async getRobotsTxtFileForUrl(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return undefined;
         }
         try {
             const origin = new URL(url).origin;
-            const cachedRobotsTxtFile = this.robotsTxtFileCache.get(origin);
+            const cachedRobotsTxtFile = this.#robotsTxtFileCache.get(origin);
             if (cachedRobotsTxtFile) {
                 return cachedRobotsTxtFile;
             }
             const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
-            this.robotsTxtFileCache.add(origin, robotsTxtFile);
+            this.#robotsTxtFileCache.add(origin, robotsTxtFile);
             return robotsTxtFile;
         }
         catch (e) {
@@ -1200,9 +1418,9 @@ export class BasicCrawler {
         }
     }
     async pauseOnMigration() {
-        if (this.autoscaledPool) {
+        if (this.#autoscaledPool) {
             // if run wasn't called, this is going to crash
-            await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
+            await this.#autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
                 if (err.message.includes('running tasks did not finish')) {
                     this.log.error('The crawler was paused due to migration to another host, ' +
                         "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -1231,7 +1449,7 @@ export class BasicCrawler {
                 });
             }
         })();
-        await Promise.all([requestManagerPersistPromise, this.stats.persistState()]);
+        await Promise.all([requestManagerPersistPromise, this.statistics.persistState?.()]);
     }
     /**
      * Fetches the next request to process from the underlying request provider.
@@ -1242,30 +1460,6 @@ export class BasicCrawler {
         }
         return this.requestManager.fetchNextRequest();
     }
-    /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
-     */
-    delayRequest(request, source) {
-        const domain = getDomain(request.url);
-        if (!domain || !request) {
-            return false;
-        }
-        const now = Date.now();
-        const lastAccessTime = this.domainAccessedTime.get(domain);
-        if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {
-            this.domainAccessedTime.set(domain, now);
-            return false;
-        }
-        const delay = lastAccessTime + this.sameDomainDelayMillis - now;
-        this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
-        setTimeout(async () => {
-            this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
-            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
-        }, delay);
-        return true;
-    }
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
     async handleRequest(crawlingContext, requestSource, request) {
         // An earlier phase we cannot cancel (e.g. a slow `extendContext`) may have run past the internal timeout,
@@ -1275,18 +1469,26 @@ export class BasicCrawler {
             return;
         }
         const statisticsId = request.id || request.uniqueKey;
+        // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
+        const transaction = currentStorageTransaction();
         let isRequestLocked = true;
         try {
             request.state = RequestState.REQUEST_HANDLER;
             await this.runRequestHandler(crawlingContext);
+            // Commit *before* marking the request as handled, so a commit failure fails the request and
+            // it is retried. This also closes the transaction, so everything below passes through.
+            await transaction?.commit();
             await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
             isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
-            this.stats.finishJob(statisticsId, request.retryCount);
+            this.statistics.finishJob(statisticsId, request.retryCount);
             // reclaim session if request finishes successfully
             request.state = RequestState.DONE;
             crawlingContext.session.markGood();
         }
         catch (rawError) {
+            // Roll back *before* any error handler runs - error handlers write to real storage precisely
+            // because the transaction is already closed. A no-op when the commit above succeeded.
+            transaction?.rollback();
             const err = this.unwrapError(rawError);
             try {
                 request.state = RequestState.ERROR_HANDLER;
@@ -1312,7 +1514,7 @@ export class BasicCrawler {
             }
             // decrease the session score if the request fails (but the error handler did not throw);
             // skip when the error is a SessionError, which already retired the session
-            if (!(err instanceof SessionError)) {
+            if (!this.errorAbsolvesSession(err)) {
                 crawlingContext.session.markBad();
             }
         }
@@ -1330,52 +1532,6 @@ export class BasicCrawler {
             }
         }
     }
-    /**
-     * Wrapper around the crawling context's `enqueueLinks` method:
-     * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
-     * @internal
-     */
-    async enqueueLinksWithCrawlDepth(options, request, requestManager) {
-        const transformRequestFunctionWrapper = (requestOptions) => {
-            requestOptions.crawlDepth = request.crawlDepth + 1;
-            if (this.maxCrawlDepth !== undefined && requestOptions.crawlDepth > this.maxCrawlDepth) {
-                // Setting `skippedReason` before returning `false` ensures that `reportSkippedRequests`
-                // reports `'depth'` as the reason (via `request.skippedReason ?? reason` fallback),
-                // rather than the generic `'transform'` reason.
-                requestOptions.skippedReason = 'depth';
-                return false;
-            }
-            // After injecting the crawlDepth, we call the user-provided transform function, if there is one.
-            return options.transformRequestFunction?.(requestOptions) ?? requestOptions;
-        };
-        // Create a request-scoped callback that logs enqueueLimit once per request handler call
-        // Only log if an explicit limit was passed to enqueueLinks (not the internal maxRequestsPerCrawl-derived limit)
-        let loggedEnqueueLimitForThisRequest = false;
-        const onSkippedRequest = async (skippedOptions) => {
-            if (skippedOptions.reason === 'enqueueLimit') {
-                if (!loggedEnqueueLimitForThisRequest && options.limit !== undefined) {
-                    this.log.info(`Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`);
-                    loggedEnqueueLimitForThisRequest = true;
-                }
-            }
-            await this.handleSkippedRequest(skippedOptions);
-        };
-        // `enqueueLinks` applies `options.label`/`options.userData` to every newly enqueued request, so a single
-        // validation against the label's schema covers them all (a no-op unless the router declares a schema).
-        await this.validateRequestUserData({ label: options.label, userData: options.userData });
-        return await enqueueLinks({
-            requestManager,
-            robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
-            respectRobotsTxtFile: this.respectRobotsTxtFile,
-            onSkippedRequest,
-            limit: await this.calculateEnqueuedRequestLimit(options.limit),
-            // Allow user options to override defaults set above ⤴
-            ...options,
-            transformRequestFunction: transformRequestFunctionWrapper,
-        });
-    }
     /**
      * Generator function that yields requests injected with the given crawl depth.
      * @internal
@@ -1438,13 +1594,23 @@ export class BasicCrawler {
      * @param request The request object, passed separately to circumvent potential dynamic logic in crawlingContext.request
      */
     async requestFunctionErrorHandler(error, crawlingContext, request, source) {
+        if (error instanceof RequestThrottledError) {
+            // The domain told us to come back later, so the request was never really attempted. Put it back
+            // without recording a failure - it costs neither a retry nor session reputation.
+            this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
+                id: request.id,
+                url: request.url,
+            });
+            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+            return;
+        }
         request.pushErrorMessage(error);
         if (error instanceof CriticalError) {
             throw error;
         }
         const shouldRetryRequest = this.canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
-            await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
+            await this.statistics.errorTrackerRetry.addAsync(error, crawlingContext);
             await this.errorHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
             error);
             if (error instanceof SessionError) {
@@ -1455,7 +1621,7 @@ export class BasicCrawler {
                 const { url, retryCount, id } = request;
                 // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
                 // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
-                const message = this._getMessageFromError(error);
+                const message = this.getMessageFromError(error);
                 this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {
                     id,
                     url,
@@ -1473,22 +1639,22 @@ export class BasicCrawler {
         // This is to make sure the error snapshot is not duplicated in the errorTrackerRetry and errorTracker objects.
         const { noRetry, maxRetries } = request;
         if (noRetry || !maxRetries) {
-            await this.stats.errorTracker.addAsync(error, crawlingContext);
+            await this.statistics.errorTracker.addAsync(error, crawlingContext);
         }
         else {
-            this.stats.errorTracker.add(error);
+            this.statistics.errorTracker.add(error);
         }
         // If we get here, the request is either not retryable
         // or failed more than retryCount times and will not be retried anymore.
         // Mark the request as failed and do not retry.
         await source.markRequestAsHandled(request);
-        this.stats.failJob(request.id || request.uniqueKey, request.retryCount);
+        this.statistics.failJob(request.id || request.uniqueKey, request.retryCount);
         await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
     async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
-        const message = this._getMessageFromError(error, true);
+        const message = this.getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
         if (this.failedRequestHandler) {
             await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
@@ -1500,7 +1666,7 @@ export class BasicCrawler {
      * @param error The error received
      * @returns The message to be logged
      */
-    _getMessageFromError(error, forceStack = false) {
+    getMessageFromError(error, forceStack = false) {
         if ([TypeError, SyntaxError, ReferenceError].some((type) => error instanceof type)) {
             forceStack = true;
         }
@@ -1514,6 +1680,13 @@ export class BasicCrawler {
             ? (error.stack ?? [error.message || error, ...stackLines].join('\n'))
             : [error.message || error, userLine].join('\n');
     }
+    /**
+     * Whether the session should be spared for this error - either because it was already retired, or because the
+     * failure says nothing about the session (a rate limit is a property of the domain).
+     */
+    errorAbsolvesSession(error) {
+        return error instanceof SessionError || error instanceof RequestThrottledError;
+    }
     canRequestBeRetried(request, error) {
         // Request should never be retried, or the error encountered makes it not able to be retried.
         if (request.noRetry || error instanceof NonRetryableError) {
@@ -1535,15 +1708,20 @@ export class BasicCrawler {
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     async teardown() {
-        serviceLocator.getEventManager().emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
-        if (this._closeEvents) {
+        // When this crawler initialized the event manager, its close() call emits
+        // the final persistence event after the crawler-specific state has been
+        // saved. External event managers still need an explicit event here.
+        if (!this.#closeEvents) {
+            serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
+        }
+        await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown({ persistState: this.#closeEvents ?? false }));
+        if (this.#closeEvents) {
             await serviceLocator.getEventManager().close();
         }
-        await this.sessionPoolDep.ifOwned((pool) => pool.teardown());
-        await this.autoscaledPool?.abort();
-        await this.concurrencySystemDep?.ifOwned((system) => system.stop());
+        await this.#autoscaledPool?.abort();
+        await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
     }
-    _getCookieHeaderFromRequest(request) {
+    getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {
             this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
             return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);
@@ -1599,6 +1777,10 @@ export class BasicCrawler {
         }
     }
 }
+/** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
+function hostnameOrUrl(url) {
+    return URL.canParse(url) ? new URL(url).hostname : url;
+}
 export function createBasicRouter(routes) {
     return Router.create(routes);
 }