@crawlee/basic 4.0.0-beta.98 → 4.0.0-rc.0

package/index.d.ts

@@ -1,3 +1,3 @@
 export * from '@crawlee/core';
 export * from './internals/basic-crawler.js';
-export type { CheerioRoot, CheerioAPI, Cheerio, Element } from '@crawlee/utils';
+export type { CheerioRoot, CheerioAPI, Cheerio, Element } from '@crawlee/utils/internal';

package/internals/basic-crawler.d.ts

@@ -1,8 +1,10 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
-import { ConcurrencySystem, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
-import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueUrlsOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
+import { ConcurrencySystem, Configuration, ContextPipeline, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
+import { BaseHttpClient } from '@crawlee/http-client';
+import type { Awaitable, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
-import type { ReadonlyDeep, SetRequired } from 'type-fest';
+import type { ReadonlyDeep } from 'type-fest';
+import { z } from 'zod';
 import { TimeoutError } from '@apify/timeout';
 export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
 }
@@ -27,7 +29,7 @@ export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlin
 export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
     contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
 };
-export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
@@ -135,7 +137,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     maxRequestRetries?: number;
     /**
-     * Indicates how much time (in seconds) to wait before crawling another same domain request.
+     * Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
+     * paced together with the site they belong to.
+     *
+     * Wraps the crawler's request manager in a {@link ThrottlingRequestManager}; pass one as `requestManager`
+     * yourself to configure it further.
      * @default 0
      */
     sameDomainDelaySecs?: number;
@@ -236,6 +242,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     statusMessageCallback?: StatusMessageCallback;
     /**
      * HTTP status codes that indicate the session should be retired.
+     *
+     * A 429 from a domain covered by a {@link ThrottlingRequestManager} is handled as a rate limit before
+     * this is consulted, so removing 429 here only affects domains that manager does not cover.
+     *
      * @default [401, 403, 429]
      */
     blockedStatusCodes?: number[];
@@ -267,10 +277,29 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     onSkippedRequest?: SkippedRequestCallback;
     /**
-     * Customize the way statistics collecting works, such as logging interval or
-     * whether to output them to the Key-Value store.
+     * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
+     * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
+     * implementing {@link IStatistics}.
+     *
+     * Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
+     * {@link BasicCrawler.statistics|`crawler.statistics.state`}:
+     *
+     * ```ts
+     * const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
+     *
+     * const crawler = new BasicCrawler({
+     *     statistics,
+     *     requestHandler: async () => {
+     *         statistics.state.productsFound++;
+     *     },
+     * });
+     *
+     * await crawler.run();
+     * // the custom fields are typed on `crawler.statistics` too
+     * console.log(crawler.statistics.state.productsFound);
+     * ```
      */
-    statisticsOptions?: StatisticsOptions;
+    statistics?: IStatistics<StatisticStateExtension>;
     /**
      * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
      * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
@@ -313,6 +342,20 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      *
      */
     id?: string;
+    /**
+     * Makes the storage writes performed while handling a request atomic with respect to the request
+     * succeeding: they are recorded in a {@link StorageTransaction} spanning the whole request
+     * lifecycle and only applied when the request handler succeeds, so a thrown handler leaves no partial
+     * writes behind and a retry does not double-write. Reads within the handler see its own writes.
+     *
+     * `false` disables the mechanism entirely; an object overrides the per-storage-type
+     * {@link StorageWritePolicy} (e.g. `{ requestQueue: 'deferred' }` for all-or-nothing enqueues).
+     * {@link withDirectStorageAccess} is the per-call-site escape hatch; `useState()` is deliberately
+     * *not* transactional.
+     *
+     * @default true
+     */
+    transactionalStorage?: boolean | Partial<StorageWritePolicy>;
     /**
      * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
      * By default, status codes >= 500 trigger errors.
@@ -408,7 +451,7 @@ interface CrawlerIdentity {
     /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */
     readonly hasExplicitId: boolean;
 }
-export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     #private;
     protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
     /**
@@ -418,47 +461,21 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     private static instanceCount;
     /**
-     * Tracks crawler instances that accessed shared state without having an explicit id.
-     * Used to detect and warn about multiple crawlers sharing the same state.
+     * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+     * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    private static useStateAnonymousIndices;
-    /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
-     */
-    readonly stats: Statistics;
+    get statistics(): IStatistics<StatisticStateExtension>;
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
     protected requestManager?: IRequestManager;
-    /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
-    private sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool(): ISessionPool;
-    /**
-     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
-     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
-     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
-     */
-    private ownedRequestQueue;
-    /**
-     * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
-     * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
-     * so it only needs to be applied once, at the first async access of the manager.
-     */
-    private requestManagerTimeoutsApplied;
-    /**
-     * Resolves the governor for one run: either the injected
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
-     * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
-     */
-    private readonly resolveConcurrencySystem;
-    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    private concurrencySystemDep?;
     /**
      * The concurrency governor this run is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
@@ -472,14 +489,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem(): IConcurrencySystem | undefined;
-    /**
-     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
-     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
-     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
-     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
-     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
-     */
-    private autoscaledPool?;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -490,7 +499,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     readonly router: RouterHandler<Context, Routes>;
-    private _basicContextPipeline?;
     /**
      * The basic part of the context pipeline. Unlike the subclass pipeline, this
      * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
@@ -502,11 +510,9 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     get basicContextPipeline(): ContextPipeline<{
         request: Request;
     }, CrawlingContext>;
-    private _contextPipeline?;
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
     running: boolean;
     hasFinishedBefore: boolean;
-    private unexpectedStop;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
     protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
@@ -515,15 +521,10 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     protected readonly internalTimeoutMillis: number;
     protected readonly maxRequestRetries: number;
     protected readonly maxCrawlDepth?: number;
-    private sameDomainDelayMillis;
-    private domainAccessedTime;
     protected readonly maxRequestsPerCrawl?: number;
     private get handledRequestsCount();
-    private statusMessageLoggingInterval;
-    private statusMessageCallback?;
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
-    private ignoreHttpErrorStatusCodes;
     /**
      * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
      * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
@@ -533,89 +534,100 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
-    private respectRobotsTxtFile;
     protected readonly onSkippedRequest?: SkippedRequestCallback;
-    private _closeEvents?;
-    private loggedPerRun;
-    private readonly robotsTxtFileCache;
     protected readonly identity: CrawlerIdentity;
-    private readonly contextPipelineOptions;
     protected static optionsShape: {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        blockedStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;
+        configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
     };
+    protected static optionsSchema: z.ZodObject<{
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;
+        configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> & RequireContextPipeline<CrawlingContext, Context>);
+    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
      * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
@@ -743,6 +755,11 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
      * the `waitForAllRequestsToBeAdded` promise you get in the response object.
      *
+     * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
+     * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
+     * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
+     * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
+     *
      * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
      *
      * @param requests The requests to add
@@ -769,7 +786,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * Initializes the crawler.
      */
-    protected _init(): Promise<void>;
+    protected init(): Promise<void>;
     /**
      * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
      * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
@@ -797,33 +814,40 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     private getRouteTimeoutMillis;
     protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
+    /**
+     * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
+     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
+     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     */
+    private runInStorageTransaction;
     /**
      * Handles blocked request
      */
-    protected _throwOnBlockedRequest(statusCode: number): void;
+    protected throwOnBlockedRequest(statusCode: number): void;
     private isAllowedBasedOnRobotsTxtFile;
+    /**
+     * Records an HTTP 429 against the URL's domain so the request manager can pace the retry.
+     *
+     * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
+     * @returns `true` if a manager took responsibility for the delay, in which case the caller should throw
+     *  {@link RequestThrottledError} rather than treating the response as a blocked session.
+     */
+    protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
+    /**
+     * Hands a robots.txt `Crawl-delay` to the request manager, warning if nothing is able to honour it.
+     *
+     * The warning is driven by whether the delay was actually accepted rather than by the type of the manager,
+     * because a manager that does throttle still drops the delay for a domain missing from its `domains` list.
+     */
+    private applyCrawlDelay;
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
     private pauseOnMigration;
     /**
      * Fetches the next request to process from the underlying request provider.
      */
     private fetchNextRequest;
-    /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
-     */
-    private delayRequest;
     /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
     private handleRequest;
-    /**
-     * Wrapper around the crawling context's `enqueueLinks` method:
-     * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
-     * @internal
-     */
-    protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestManager: IRequestManager): Promise<BatchAddRequestsResult>;
     /**
      * Generator function that yields requests injected with the given crawl depth.
      * @internal
@@ -859,7 +883,12 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @param error The error received
      * @returns The message to be logged
      */
-    protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
+    protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
+    /**
+     * Whether the session should be spared for this error - either because it was already retired, or because the
+     * failure says nothing about the session (a rate limit is a property of the domain).
+     */
+    private errorAbsolvesSession;
     private canRequestBeRetried;
     /**
      * Stops the crawler immediately.
@@ -869,7 +898,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     teardown(): Promise<void>;
-    protected _getCookieHeaderFromRequest(request: Request): string;
+    protected getCookieHeaderFromRequest(request: Request): string;
     private requestMatchesEnqueueStrategy;
 }
 export interface CreateContextOptions {
@@ -877,7 +906,7 @@ export interface CreateContextOptions {
     session: ISession;
     proxyInfo?: ProxyInfo;
 }
-export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
+export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
 }
 export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
 }