@crawlee/basic 4.0.0-beta.104 → 4.0.0-beta.106

package/internals/basic-crawler.d.ts

@@ -1,7 +1,8 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
 import { ConcurrencySystem, ContextPipeline, Dataset, RequestQueue } from '@crawlee/core';
 import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
+import { type BasePredicate } from 'ow';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
 import { TimeoutError } from '@apify/timeout';
 export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
@@ -314,6 +315,20 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      *
      */
     id?: string;
+    /**
+     * Makes the storage writes performed while handling a request atomic with respect to the request
+     * succeeding: they are recorded in a {@link StorageTransaction} spanning the whole request
+     * lifecycle and only applied when the request handler succeeds, so a thrown handler leaves no partial
+     * writes behind and a retry does not double-write. Reads within the handler see its own writes.
+     *
+     * `false` disables the mechanism entirely; an object overrides the per-storage-type
+     * {@link StorageWritePolicy} (e.g. `{ requestQueue: 'deferred' }` for all-or-nothing enqueues).
+     * {@link withDirectStorageAccess} is the per-call-site escape hatch; `useState()` is deliberately
+     * *not* transactional.
+     *
+     * @default true
+     */
+    transactionalStorage?: boolean | Partial<StorageWritePolicy>;
     /**
      * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
      * By default, status codes >= 500 trigger errors.
@@ -418,13 +433,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * collide.
      */
     private static instanceCount;
-    /**
-     * Tracks crawler instances that accessed shared state without having an explicit id.
-     * Used to detect and warn about multiple crawlers sharing the same state.
-     */
-    private static useStateAnonymousIndices;
-    /** Backs the {@link BasicCrawler.stats|`stats`} getter. */
-    private statsDep;
     /**
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
@@ -436,33 +444,11 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
     protected requestManager?: IRequestManager;
-    /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
-    private sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool(): ISessionPool;
-    /**
-     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
-     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
-     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
-     */
-    private ownedRequestQueue;
-    /**
-     * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
-     * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
-     * so it only needs to be applied once, at the first async access of the manager.
-     */
-    private requestManagerTimeoutsApplied;
-    /**
-     * Resolves the governor for one run: either the injected
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
-     * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
-     */
-    private readonly resolveConcurrencySystem;
-    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    private concurrencySystemDep?;
     /**
      * The concurrency governor this run is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
@@ -476,14 +462,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem(): IConcurrencySystem | undefined;
-    /**
-     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
-     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
-     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
-     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
-     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
-     */
-    private autoscaledPool?;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -494,7 +472,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     readonly router: RouterHandler<Context, Routes>;
-    private _basicContextPipeline?;
     /**
      * The basic part of the context pipeline. Unlike the subclass pipeline, this
      * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
@@ -506,11 +483,9 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     get basicContextPipeline(): ContextPipeline<{
         request: Request;
     }, CrawlingContext>;
-    private _contextPipeline?;
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
     running: boolean;
     hasFinishedBefore: boolean;
-    private unexpectedStop;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
     protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
@@ -519,15 +494,10 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     protected readonly internalTimeoutMillis: number;
     protected readonly maxRequestRetries: number;
     protected readonly maxCrawlDepth?: number;
-    private sameDomainDelayMillis;
-    private domainAccessedTime;
     protected readonly maxRequestsPerCrawl?: number;
     private get handledRequestsCount();
-    private statusMessageLoggingInterval;
-    private statusMessageCallback?;
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
-    private ignoreHttpErrorStatusCodes;
     /**
      * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
      * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
@@ -537,50 +507,45 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
-    private respectRobotsTxtFile;
     protected readonly onSkippedRequest?: SkippedRequestCallback;
-    private _closeEvents?;
-    private loggedPerRun;
-    private readonly robotsTxtFileCache;
     protected readonly identity: CrawlerIdentity;
-    private readonly contextPipelineOptions;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
-        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        extendContext: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestList: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        requestQueue: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        requestHandler: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        requestHandlerTimeoutSecs: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        errorHandler: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        failedRequestHandler: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestRetries: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        sameDomainDelaySecs: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestsPerCrawl: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxCrawlDepth: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        taskLoopOptions: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        concurrencySystem: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        sessionPool: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        proxyConfiguration: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        statusMessageLoggingInterval: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        statusMessageCallback: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -588,33 +553,34 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
 // @ts-ignore optional peer dependency or compatibility with es2022
         blockedStatusCodes: import("ow").ArrayPredicate<number>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        retryOnBlocked: import("ow").BooleanPredicate & BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
+        transactionalStorage: BasePredicate<boolean | Partial<StorageWritePolicy> | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+        onSkippedRequest: import("ow").Predicate<Function> & BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        httpClient: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        configuration: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        storageBackend: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        eventManager: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        logger: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        minConcurrency: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxConcurrency: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+        maxRequestsPerMinute: import("ow").NumberPredicate & BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        keepAlive: import("ow").BooleanPredicate & BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        statistics: import("ow").ObjectPredicate<object> & BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        id: import("ow").StringPredicate & BasePredicate<string | undefined>;
     };
     /**
      * All `BasicCrawler` parameters are passed via an options object.
@@ -773,7 +739,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * Initializes the crawler.
      */
-    protected _init(): Promise<void>;
+    protected init(): Promise<void>;
     /**
      * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
      * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
@@ -801,10 +767,16 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     private getRouteTimeoutMillis;
     protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
+    /**
+     * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
+     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
+     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     */
+    private runInStorageTransaction;
     /**
      * Handles blocked request
      */
-    protected _throwOnBlockedRequest(statusCode: number): void;
+    protected throwOnBlockedRequest(statusCode: number): void;
     private isAllowedBasedOnRobotsTxtFile;
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
     private pauseOnMigration;
@@ -863,7 +835,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @param error The error received
      * @returns The message to be logged
      */
-    protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
+    protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
     private canRequestBeRetried;
     /**
      * Stops the crawler immediately.
@@ -873,7 +845,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     teardown(): Promise<void>;
-    protected _getCookieHeaderFromRequest(request: Request): string;
+    protected getCookieHeaderFromRequest(request: Request): string;
     private requestMatchesEnqueueStrategy;
 }
 export interface CreateContextOptions {

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, } from '@crawlee/core';
+import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ConcurrencySystem, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createStorageTransaction, CriticalError, currentStorageTransaction, Dataset, enqueueLinks, EnqueueStrategy, getObjectType, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
 import { stringify } from 'csv-stringify/sync';
@@ -12,9 +12,9 @@ import { cryptoRandomObjectId } from '@apify/utilities';
 import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
 class LazyDefaultHttpClient {
-    _delegatePromise;
+    #delegatePromise;
     constructor(options) {
-        this._delegatePromise = import('@crawlee/impit-client')
+        this.#delegatePromise = import('@crawlee/impit-client')
             .then(({ ImpitHttpClient }) => new ImpitHttpClient(options))
             .catch(() => {
             (options?.logger ?? log).warning('Optional dependency @crawlee/impit-client is not installed. ' +
@@ -23,7 +23,7 @@ class LazyDefaultHttpClient {
         });
     }
     async sendRequest(...args) {
-        return (await this._delegatePromise).sendRequest(...args);
+        return (await this.#delegatePromise).sendRequest(...args);
     }
 }
 /**
@@ -46,20 +46,21 @@ export class BasicCrawler {
      * request queue; subsequent ones get their own queue via a unique alias so they don't
      * collide.
      */
+    // kept as TS-private: tests reset the counter at runtime
     static instanceCount = 0;
     /**
      * Tracks crawler instances that accessed shared state without having an explicit id.
      * Used to detect and warn about multiple crawlers sharing the same state.
      */
-    static useStateAnonymousIndices = new Set();
+    static #useStateAnonymousIndices = new Set();
     /** Backs the {@link BasicCrawler.stats|`stats`} getter. */
-    statsDep;
+    #statsDep;
     /**
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
     get stats() {
-        return this.statsDep.value;
+        return this.#statsDep.value;
     }
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
@@ -68,34 +69,34 @@ export class BasicCrawler {
      */
     requestManager;
     /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
-    sessionPoolDep;
+    #sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool() {
-        return this.sessionPoolDep.value;
+        return this.#sessionPoolDep.value;
     }
     /**
      * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
      * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
      * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
      */
-    ownedRequestQueue = OwnedOrInjected.resolve();
+    #ownedRequestQueue = OwnedOrInjected.resolve();
     /**
      * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
      * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
      * so it only needs to be applied once, at the first async access of the manager.
      */
-    requestManagerTimeoutsApplied = false;
+    #requestManagerTimeoutsApplied = false;
     /**
      * Resolves the governor for one run: either the injected
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
      * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
      */
-    resolveConcurrencySystem;
-    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    concurrencySystemDep;
+    #resolveConcurrencySystem;
+    /** As resolved by `init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
+    #concurrencySystemDep;
     /**
      * The concurrency governor this run is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
@@ -109,7 +110,7 @@ export class BasicCrawler {
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem() {
-        return this.concurrencySystemDep?.maybeValue;
+        return this.#concurrencySystemDep?.maybeValue;
     }
     /**
      * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
@@ -118,7 +119,7 @@ export class BasicCrawler {
      * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
      * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
      */
-    autoscaledPool;
+    #autoscaledPool;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -129,7 +130,7 @@ export class BasicCrawler {
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     router = Router.create();
-    _basicContextPipeline;
+    #basicContextPipeline;
     /**
      * The basic part of the context pipeline. Unlike the subclass pipeline, this
      * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
@@ -139,21 +140,21 @@ export class BasicCrawler {
      * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
      */
     get basicContextPipeline() {
-        if (this._basicContextPipeline === undefined) {
-            this._basicContextPipeline = this.buildBasicContextPipeline();
+        if (this.#basicContextPipeline === undefined) {
+            this.#basicContextPipeline = this.buildBasicContextPipeline();
         }
-        return this._basicContextPipeline;
+        return this.#basicContextPipeline;
     }
-    _contextPipeline;
+    #contextPipeline;
     get contextPipeline() {
-        if (this._contextPipeline === undefined) {
-            this._contextPipeline = this.buildFinalContextPipeline();
+        if (this.#contextPipeline === undefined) {
+            this.#contextPipeline = this.buildFinalContextPipeline();
         }
-        return this._contextPipeline;
+        return this.#contextPipeline;
     }
     running = false;
     hasFinishedBefore = false;
-    unexpectedStop = false;
+    #unexpectedStop = false;
     #log;
     get log() {
         return this.#log;
@@ -161,37 +162,43 @@ export class BasicCrawler {
     requestHandler;
     errorHandler;
     failedRequestHandler;
+    // kept as TS-private: tests read it at runtime
     requestHandlerTimeoutMillis;
     internalTimeoutMillis;
     maxRequestRetries;
     maxCrawlDepth;
-    sameDomainDelayMillis;
-    domainAccessedTime;
+    #sameDomainDelayMillis;
+    #domainAccessedTime;
     maxRequestsPerCrawl;
     get handledRequestsCount() {
         return this.stats.state.requestsFinished + this.stats.state.requestsFailed;
     }
-    statusMessageLoggingInterval;
-    statusMessageCallback;
+    #statusMessageLoggingInterval;
+    #statusMessageCallback;
     blockedStatusCodes = new Set();
     additionalHttpErrorStatusCodes;
-    ignoreHttpErrorStatusCodes;
+    #ignoreHttpErrorStatusCodes;
     /**
      * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
      * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
      * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
      * settable here.
      */
+    // kept as TS-private: tests mutate it at runtime
     taskLoopOptions;
     httpClient;
     retryOnBlocked;
-    respectRobotsTxtFile;
+    #respectRobotsTxtFile;
+    /** Whether `runInStorageTransaction()` opens a transaction at all. */
+    #transactionalStorageEnabled;
+    /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
+    #storageWritePolicy;
     onSkippedRequest;
-    _closeEvents;
-    loggedPerRun = new Set();
-    robotsTxtFileCache;
+    #closeEvents;
+    #loggedPerRun = new Set();
+    #robotsTxtFileCache;
     identity;
-    contextPipelineOptions;
+    #contextPipelineOptions;
     static optionsShape = {
         contextPipelineBuilder: ow.optional.object,
         extendContext: ow.optional.function,
@@ -219,6 +226,9 @@ export class BasicCrawler {
         blockedStatusCodes: ow.optional.array.ofType(ow.number),
         retryOnBlocked: ow.optional.boolean,
         respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object),
+        transactionalStorage: ow.optional.any(ow.boolean, ow.object.exactShape({
+            requestQueue: ow.optional.string.oneOf(['deferred', 'writeThrough']),
+        })),
         onSkippedRequest: ow.optional.function,
         httpClient: ow.optional.object,
         configuration: ow.optional.object,
@@ -246,7 +256,7 @@ export class BasicCrawler {
         // Service locator options
         configuration, storageBackend, eventManager, logger, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statistics, httpClient, id, } = options;
+        minConcurrency, maxConcurrency, maxRequestsPerMinute, blockedStatusCodes: blockedStatusCodesInput, retryOnBlocked = false, respectRobotsTxtFile = false, transactionalStorage, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statistics, httpClient, id, } = options;
         // All concurrency configuration lives on the `ConcurrencySystem`, so the shortcuts have nowhere to go once
         // one is supplied - and silently dropping a `maxConcurrency` the user asked for is how crawls end up
         // hammering a site.
@@ -271,7 +281,7 @@ export class BasicCrawler {
         }
         try {
             serviceLocatorScope.enterScope();
-            this.contextPipelineOptions = {
+            this.#contextPipelineOptions = {
                 contextPipelineBuilder: options.contextPipelineBuilder,
                 extendContext: options.extendContext,
             };
@@ -302,13 +312,13 @@ export class BasicCrawler {
             }
             this.httpClient = httpClient ?? new LazyDefaultHttpClient({ logger: this.log });
             this.proxyConfiguration = proxyConfiguration;
-            this.statusMessageLoggingInterval = statusMessageLoggingInterval;
-            this.statusMessageCallback = statusMessageCallback;
-            this.domainAccessedTime = new Map();
-            this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
+            this.#statusMessageLoggingInterval = statusMessageLoggingInterval;
+            this.#statusMessageCallback = statusMessageCallback;
+            this.#domainAccessedTime = new Map();
+            this.#robotsTxtFileCache = new LruCache({ maxLength: 1000 });
             this.handleSkippedRequest = this.handleSkippedRequest.bind(this);
             this.additionalHttpErrorStatusCodes = new Set([...additionalHttpErrorStatusCodes]);
-            this.ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
+            this.#ignoreHttpErrorStatusCodes = new Set([...ignoreHttpErrorStatusCodes]);
             this.requestHandler = requestHandler ?? this.router;
             this.failedRequestHandler = failedRequestHandler;
             this.errorHandler = errorHandler;
@@ -319,7 +329,11 @@ export class BasicCrawler {
                 this.requestHandlerTimeoutMillis = 60_000;
             }
             this.retryOnBlocked = retryOnBlocked;
-            this.respectRobotsTxtFile = respectRobotsTxtFile;
+            this.#respectRobotsTxtFile = respectRobotsTxtFile;
+            // The cast undoes ow's assertion signature, which mangles `boolean | object` unions.
+            const transactionalStorageOption = transactionalStorage;
+            this.#transactionalStorageEnabled = transactionalStorageOption !== false;
+            this.#storageWritePolicy = typeof transactionalStorageOption === 'object' ? transactionalStorageOption : {};
             this.onSkippedRequest = onSkippedRequest;
             // allow at least 5min for internal timeouts
             this.internalTimeoutMillis =
@@ -327,8 +341,8 @@ export class BasicCrawler {
                     Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
-            this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
-            this.statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
+            this.#sameDomainDelayMillis = sameDomainDelaySecs * 1000;
+            this.#statsDep = OwnedOrInjected.resolve(statistics, () => new Statistics({
                 logMessage: `${this.constructor.name} request statistics:`,
                 log: this.log,
                 id: this.identity.id,
@@ -339,7 +353,7 @@ export class BasicCrawler {
                     '`proxyInfo` they were created with. Configure proxies on the pool instead, ' +
                     'e.g. via `addSession({ proxyInfo })` or a custom `createSessionFunction`.');
             }
-            this.sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
+            this.#sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
                 createSessionFunction: async (opts) => new Session({
                     ...opts?.sessionOptions,
                     proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
@@ -376,12 +390,15 @@ export class BasicCrawler {
                     this.stats.startJob(request.id || request.uniqueKey);
                     const crawlingContext = { request };
                     try {
+                        // The transaction spans the whole pipeline call, covering the navigation hooks
+                        // and `extendContext` too; `handleRequest` drives its outcome explicitly.
+                        await this.runInStorageTransaction(async () => 
                         // Navigation, the navigation hooks and the request handler are timed individually, but the
                         // phases between them are not, so a request could still get stuck indefinitely. This is the
                         // catch-all for that - see `raceWithTimeout` for why it is a bare timer, not a timeout frame.
                         await this.withRequestTimeout(crawlingContext, this.basicContextPipeline
                             .chain(this.contextPipeline)
-                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request)));
+                            .call(crawlingContext, (ctx) => this.handleRequest(ctx, source, request))));
                     }
                     catch (error) {
                         // ContextPipelineInterruptedError means the request was intentionally skipped
@@ -421,7 +438,7 @@ export class BasicCrawler {
                             `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
                         return false;
                     }
-                    if (this.unexpectedStop) {
+                    if (this.#unexpectedStop) {
                         this.logOncePerRun('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
                             'Ongoing requests will be allowed to complete.');
                         return false;
@@ -435,7 +452,7 @@ export class BasicCrawler {
                             `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
                         return true;
                     }
-                    if (this.unexpectedStop) {
+                    if (this.#unexpectedStop) {
                         this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
                         return true;
                     }
@@ -453,7 +470,7 @@ export class BasicCrawler {
                 log: this.log,
             };
             this.taskLoopOptions = { ...taskLoopOptions, ...crawlerOwnedTaskLoopConfiguration };
-            this.resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
+            this.#resolveConcurrencySystem = () => OwnedOrInjected.resolve(concurrencySystem, () => this.createDefaultConcurrencySystem({
                 minConcurrency,
                 maxConcurrency,
                 maxTasksPerMinute: maxRequestsPerMinute,
@@ -482,7 +499,7 @@ export class BasicCrawler {
      * @returns `true` if the status code is considered an error, `false` otherwise
      */
     isErrorStatusCode(status) {
-        const excludeError = this.ignoreHttpErrorStatusCodes.has(status);
+        const excludeError = this.#ignoreHttpErrorStatusCodes.has(status);
         const includeError = this.additionalHttpErrorStatusCodes.has(status);
         return (status >= 500 && !excludeError) || includeError;
     }
@@ -575,7 +592,7 @@ export class BasicCrawler {
         return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };
     }
     buildFinalContextPipeline() {
-        const subclassPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??
+        const subclassPipeline = (this.#contextPipelineOptions.contextPipelineBuilder?.() ??
             this.buildContextPipeline());
         // `extendContext` runs *before* the subclass navigation pipeline (which includes the
         // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
@@ -587,7 +604,7 @@ export class BasicCrawler {
         // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
         // casts below are sound because `buildFinalContextPipeline` is declared to return the fully
         // resolved `ExtendedContext` (= `Context & ContextExtension`).
-        const { extendContext } = this.contextPipelineOptions;
+        const { extendContext } = this.#contextPipelineOptions;
         let contextPipeline;
         if (extendContext !== undefined) {
             contextPipeline = ContextPipeline.create()
@@ -621,7 +638,7 @@ export class BasicCrawler {
      * @param error The error to check.
      */
     isProxyError(error) {
-        return ROTATE_PROXY_ERRORS.some((x) => this._getMessageFromError(error)?.includes(x));
+        return ROTATE_PROXY_ERRORS.some((x) => this.getMessageFromError(error)?.includes(x));
     }
     /**
      * Sets the status message for the current crawler run.
@@ -662,14 +679,14 @@ export class BasicCrawler {
             const { mode: operationMode, failedDelta } = getOperationMode();
             let message;
             if (operationMode === 'ERROR') {
-                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
+                message = `Experiencing problems, ${failedDelta} failed requests in the past ${this.#statusMessageLoggingInterval} seconds.`;
             }
             else {
                 const total = await this.requestManager?.getTotalCount();
                 message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
-            if (this.statusMessageCallback) {
-                await this.statusMessageCallback({
+            if (this.#statusMessageCallback) {
+                await this.#statusMessageCallback({
                     crawler: this,
                     state: this.stats.state,
                     previousState,
@@ -679,7 +696,7 @@ export class BasicCrawler {
             }
             this.setStatusMessage(message);
         };
-        const interval = setInterval(log, this.statusMessageLoggingInterval * 1e3);
+        const interval = setInterval(log, this.#statusMessageLoggingInterval * 1e3);
         return { log, stop: () => clearInterval(interval) };
     }
     /**
@@ -707,20 +724,20 @@ export class BasicCrawler {
             // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
             // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
             const shouldPurge = purgeRequestQueue !== false;
-            const managerToPurge = this.ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
+            const managerToPurge = this.#ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
             if (managerToPurge?.purge && shouldPurge) {
                 await managerToPurge.purge();
             }
             // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
-            await this.statsDep.ifOwned(async (stats) => {
+            await this.#statsDep.ifOwned(async (stats) => {
                 stats.reset();
                 await stats.resetStore();
             });
-            await this.sessionPoolDep.ifOwned((pool) => pool.resetStore());
+            await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());
         }
-        this.unexpectedStop = false;
+        this.#unexpectedStop = false;
         this.running = true;
-        this.loggedPerRun.clear();
+        this.#loggedPerRun.clear();
         await purgeDefaultStorages({
             onlyPurgeOnce: true,
             storageBackend: serviceLocator.getStorageBackend(),
@@ -730,7 +747,7 @@ export class BasicCrawler {
             await this.addRequests(requests, addRequestsOptions);
         }
         try {
-            await this._init();
+            await this.init();
             await this.stats.startCapturing();
         }
         catch (error) {
@@ -747,7 +764,7 @@ export class BasicCrawler {
         const sigintHandler = async () => {
             this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
             await this.pauseOnMigration();
-            await this.autoscaledPool.abort();
+            await this.#autoscaledPool.abort();
         };
         // Attach a listener to handle migration and aborting events gracefully.
         const boundPauseOnMigration = this.pauseOnMigration.bind(this);
@@ -757,7 +774,7 @@ export class BasicCrawler {
         eventManager.on("aborting" /* EventType.ABORTING */, boundPauseOnMigration);
         let stats = {};
         try {
-            await this.autoscaledPool.run();
+            await this.#autoscaledPool.run();
         }
         finally {
             await this.teardown();
@@ -807,11 +824,11 @@ export class BasicCrawler {
      * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
     stop(reason = 'The crawler has been gracefully stopped.') {
-        if (this.unexpectedStop) {
+        if (this.#unexpectedStop) {
             return;
         }
         this.log.info(reason);
-        this.unexpectedStop = true;
+        this.#unexpectedStop = true;
     }
     /**
      * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
@@ -822,22 +839,22 @@ export class BasicCrawler {
      * throughout, since a shared one may still be serving other crawlers.
      */
     async pause(timeoutSecs) {
-        if (!this.autoscaledPool) {
+        if (!this.#autoscaledPool) {
             this.log.warning('Cannot pause a crawler that is not running.');
             return;
         }
-        await this.autoscaledPool.pause(timeoutSecs);
+        await this.#autoscaledPool.pause(timeoutSecs);
     }
     /**
      * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
      * again. A no-op on a crawler that is not paused.
      */
     resume() {
-        if (!this.autoscaledPool) {
+        if (!this.#autoscaledPool) {
             this.log.warning('Cannot resume a crawler that is not running.');
             return;
         }
-        this.autoscaledPool.resume();
+        this.#autoscaledPool.resume();
     }
     /**
      * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
@@ -850,8 +867,8 @@ export class BasicCrawler {
         // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
         // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
         // but guard so we do not re-issue it on every call.
-        if (!this.requestManagerTimeoutsApplied) {
-            this.requestManagerTimeoutsApplied = true;
+        if (!this.#requestManagerTimeoutsApplied) {
+            this.#requestManagerTimeoutsApplied = true;
             await this.applyRequestManagerTimeouts(this.requestManager);
         }
         return this.requestManager;
@@ -873,7 +890,7 @@ export class BasicCrawler {
         // subsequent instances get their own queue via a unique alias so they don't collide.
         const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
         const requestQueue = await RequestQueue.open(identifier, { configuration: serviceLocator.getConfiguration() });
-        return this.ownedRequestQueue.set(requestQueue);
+        return this.#ownedRequestQueue.set(requestQueue);
     }
     /**
      * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
@@ -922,8 +939,8 @@ export class BasicCrawler {
             const stateKey = `${BasicCrawler.CRAWLEE_STATE_KEY}_${this.identity.id}`;
             return kvs.getAutoSavedValue(stateKey, defaultValue);
         }
-        BasicCrawler.useStateAnonymousIndices.add(this.identity.instanceIndex);
-        if (BasicCrawler.useStateAnonymousIndices.size > 1) {
+        BasicCrawler.#useStateAnonymousIndices.add(this.identity.instanceIndex);
+        if (BasicCrawler.#useStateAnonymousIndices.size > 1) {
             serviceLocator
                 .getLogger()
                 .warningOnce('Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
@@ -944,19 +961,23 @@ export class BasicCrawler {
         return Math.min(limit, explicitLimit ?? Infinity);
     }
     async handleSkippedRequest(options) {
-        if (options.reason === 'limit') {
-            this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
-                `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
-        }
-        if (options.reason === 'depth') {
-            this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
-        }
-        await this.onSkippedRequest?.(options);
+        // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
+        // transaction scope, which rolls back - so the skip bookkeeping must write directly.
+        await withDirectStorageAccess(async () => {
+            if (options.reason === 'limit') {
+                this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
+                    `${this.maxRequestsPerCrawl} requests and no further requests will be added.`);
+            }
+            if (options.reason === 'depth') {
+                this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`);
+            }
+            await this.onSkippedRequest?.(options);
+        });
     }
     logOncePerRun(key, message) {
-        if (!this.loggedPerRun.has(key)) {
+        if (!this.#loggedPerRun.has(key)) {
             this.log.info(message);
-            this.loggedPerRun.add(key);
+            this.#loggedPerRun.add(key);
         }
     }
     /**
@@ -1094,11 +1115,11 @@ export class BasicCrawler {
     /**
      * Initializes the crawler.
      */
-    async _init() {
+    async init() {
         const eventManager = serviceLocator.getEventManager();
         if (!eventManager.isInitialized()) {
             await eventManager.init();
-            this._closeEvents = true;
+            this.#closeEvents = true;
         }
         // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
         // floored per request so it will not actually cut them short, but the configured value is then effectively
@@ -1113,11 +1134,11 @@ export class BasicCrawler {
         // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
         // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
         // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
-        this.concurrencySystemDep = this.resolveConcurrencySystem();
-        await this.concurrencySystemDep.ifOwned((system) => system.start());
-        this.autoscaledPool = new AutoscaledPool({
+        this.#concurrencySystemDep = this.#resolveConcurrencySystem();
+        await this.#concurrencySystemDep.ifOwned((system) => system.start());
+        this.#autoscaledPool = new AutoscaledPool({
             ...this.taskLoopOptions,
-            concurrencySystem: this.concurrencySystemDep.value,
+            concurrencySystem: this.#concurrencySystemDep.value,
             consumer: this.identity,
         });
         await this.getRequestManager();
@@ -1168,10 +1189,44 @@ export class BasicCrawler {
         const timeoutMillis = this.resolveRequestHandlerTimeoutMillis(crawlingContext.request.label);
         await addTimeoutToPromise(async () => this.requestHandler(crawlingContext), timeoutMillis, `requestHandler timed out after ${timeoutMillis / 1000} seconds (${crawlingContext.request.id}).`);
     }
+    /**
+     * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
+     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
+     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     */
+    async runInStorageTransaction(callback) {
+        if (!this.#transactionalStorageEnabled) {
+            return callback();
+        }
+        const transaction = createStorageTransaction({
+            policy: this.#storageWritePolicy,
+            commitTimeoutMillis: this.internalTimeoutMillis,
+        });
+        let threw = true;
+        try {
+            const result = await transaction.run(callback);
+            threw = false;
+            return result;
+        }
+        finally {
+            if (transaction.state === 'open') {
+                // `handleRequest` commits or rolls back on every normal path, so an open transaction on
+                // a normal return is a wiring bug; on a propagating throw (a pipeline-level failure) it is
+                // expected. Either way, discard the unvalidated writes; only the former is worth flagging.
+                if (!threw) {
+                    this.log.error('Internal error: a storage transaction was still open after the request pipeline ' +
+                        'returned normally. Its writes are being discarded. Please report this.');
+                }
+                transaction.rollback();
+            }
+            // Unconditional: `failed` is a terminal state that the branch above never reaches.
+            transaction.dispose();
+        }
+    }
     /**
      * Handles blocked request
      */
-    _throwOnBlockedRequest(statusCode) {
+    throwOnBlockedRequest(statusCode) {
         if (this.retryOnBlocked)
             return;
         if (this.blockedStatusCodes.has(statusCode)) {
@@ -1179,25 +1234,25 @@ export class BasicCrawler {
         }
     }
     async isAllowedBasedOnRobotsTxtFile(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return true;
         }
         const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
-        const userAgent = typeof this.respectRobotsTxtFile === 'object' ? this.respectRobotsTxtFile?.userAgent : '*';
+        const userAgent = typeof this.#respectRobotsTxtFile === 'object' ? this.#respectRobotsTxtFile?.userAgent : '*';
         return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
     }
     async getRobotsTxtFileForUrl(url) {
-        if (!this.respectRobotsTxtFile) {
+        if (!this.#respectRobotsTxtFile) {
             return undefined;
         }
         try {
             const origin = new URL(url).origin;
-            const cachedRobotsTxtFile = this.robotsTxtFileCache.get(origin);
+            const cachedRobotsTxtFile = this.#robotsTxtFileCache.get(origin);
             if (cachedRobotsTxtFile) {
                 return cachedRobotsTxtFile;
             }
             const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
-            this.robotsTxtFileCache.add(origin, robotsTxtFile);
+            this.#robotsTxtFileCache.add(origin, robotsTxtFile);
             return robotsTxtFile;
         }
         catch (e) {
@@ -1206,9 +1261,9 @@ export class BasicCrawler {
         }
     }
     async pauseOnMigration() {
-        if (this.autoscaledPool) {
+        if (this.#autoscaledPool) {
             // if run wasn't called, this is going to crash
-            await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
+            await this.#autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
                 if (err.message.includes('running tasks did not finish')) {
                     this.log.error('The crawler was paused due to migration to another host, ' +
                         "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -1259,12 +1314,12 @@ export class BasicCrawler {
             return false;
         }
         const now = Date.now();
-        const lastAccessTime = this.domainAccessedTime.get(domain);
-        if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {
-            this.domainAccessedTime.set(domain, now);
+        const lastAccessTime = this.#domainAccessedTime.get(domain);
+        if (!lastAccessTime || now - lastAccessTime >= this.#sameDomainDelayMillis) {
+            this.#domainAccessedTime.set(domain, now);
             return false;
         }
-        const delay = lastAccessTime + this.sameDomainDelayMillis - now;
+        const delay = lastAccessTime + this.#sameDomainDelayMillis - now;
         this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
         setTimeout(async () => {
             this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
@@ -1281,10 +1336,15 @@ export class BasicCrawler {
             return;
         }
         const statisticsId = request.id || request.uniqueKey;
+        // Opened by `runInStorageTransaction`; absent when disabled or when the subclass opens its own.
+        const transaction = currentStorageTransaction();
         let isRequestLocked = true;
         try {
             request.state = RequestState.REQUEST_HANDLER;
             await this.runRequestHandler(crawlingContext);
+            // Commit *before* marking the request as handled, so a commit failure fails the request and
+            // it is retried. This also closes the transaction, so everything below passes through.
+            await transaction?.commit();
             await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
             isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
             this.stats.finishJob(statisticsId, request.retryCount);
@@ -1293,6 +1353,9 @@ export class BasicCrawler {
             crawlingContext.session.markGood();
         }
         catch (rawError) {
+            // Roll back *before* any error handler runs - error handlers write to real storage precisely
+            // because the transaction is already closed. A no-op when the commit above succeeded.
+            transaction?.rollback();
             const err = this.unwrapError(rawError);
             try {
                 request.state = RequestState.ERROR_HANDLER;
@@ -1374,7 +1437,7 @@ export class BasicCrawler {
         return await enqueueLinks({
             requestManager,
             robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
-            respectRobotsTxtFile: this.respectRobotsTxtFile,
+            respectRobotsTxtFile: this.#respectRobotsTxtFile,
             onSkippedRequest,
             limit: await this.calculateEnqueuedRequestLimit(options.limit),
             // Allow user options to override defaults set above ⤴
@@ -1461,7 +1524,7 @@ export class BasicCrawler {
                 const { url, retryCount, id } = request;
                 // We don't want to see the stack trace in the logs by default, when we are going to retry the request.
                 // Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
-                const message = this._getMessageFromError(error);
+                const message = this.getMessageFromError(error);
                 this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {
                     id,
                     url,
@@ -1494,7 +1557,7 @@ export class BasicCrawler {
     async handleFailedRequestHandler(crawlingContext, error) {
         // Always log the last error regardless if the user provided a failedRequestHandler
         const { id, url, method, uniqueKey } = crawlingContext.request;
-        const message = this._getMessageFromError(error, true);
+        const message = this.getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
         if (this.failedRequestHandler) {
             await this.failedRequestHandler?.(crawlingContext, // valid cast - ExtendedContext transitively extends CrawlingContext
@@ -1506,7 +1569,7 @@ export class BasicCrawler {
      * @param error The error received
      * @returns The message to be logged
      */
-    _getMessageFromError(error, forceStack = false) {
+    getMessageFromError(error, forceStack = false) {
         if ([TypeError, SyntaxError, ReferenceError].some((type) => error instanceof type)) {
             forceStack = true;
         }
@@ -1542,14 +1605,14 @@ export class BasicCrawler {
      */
     async teardown() {
         serviceLocator.getEventManager().emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
-        if (this._closeEvents) {
+        if (this.#closeEvents) {
             await serviceLocator.getEventManager().close();
         }
-        await this.sessionPoolDep.ifOwned((pool) => pool.teardown());
-        await this.autoscaledPool?.abort();
-        await this.concurrencySystemDep?.ifOwned((system) => system.stop());
+        await this.#sessionPoolDep.ifOwned((pool) => pool.teardown());
+        await this.#autoscaledPool?.abort();
+        await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
     }
-    _getCookieHeaderFromRequest(request) {
+    getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {
             this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
             return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.104",
+    "version": "4.0.0-beta.106",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.104",
-        "@crawlee/http-client": "4.0.0-beta.104",
-        "@crawlee/types": "4.0.0-beta.104",
-        "@crawlee/utils": "4.0.0-beta.104",
+        "@crawlee/core": "4.0.0-beta.106",
+        "@crawlee/http-client": "4.0.0-beta.106",
+        "@crawlee/types": "4.0.0-beta.106",
+        "@crawlee/utils": "4.0.0-beta.106",
         "csv-stringify": "^6.5.2",
         "ow": "^2.0.0",
         "tldts": "^7.0.6",
@@ -53,7 +53,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.104"
+        "@crawlee/impit-client": "^4.0.0-beta.106"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "1c761e689e56ac06b67fc4f1dc091f693e024d12"
+    "gitHead": "c622f1fc65e65221ea245817c58ecc0ffb4a5cb0"
 }