@crawlee/basic 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/basic-crawler.d.ts

@@ -1,9 +1,19 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, Configuration, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueLinksOptions, EventManager, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, Request, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticsOptions, StatisticState, StorageIdentifier, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
-import { ConcurrencySystem, ContextPipeline, Dataset, RequestQueue, Statistics } from '@crawlee/core';
-import type { Awaitable, BaseHttpClient, BatchAddRequestsResult, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, CrawleeLogger, DatasetExportOptions, IProxyConfiguration, IRequestLoader, IRequestManager, RequestsLike, Source, StorageIdentifier, StorageWritePolicy } from '@crawlee/core';
+import { Configuration, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
+import { BaseHttpClient } from '@crawlee/http-client';
+import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
-import type { ReadonlyDeep, SetRequired } from 'type-fest';
-import { TimeoutError } from '@apify/timeout';
+import type { ReadonlyDeep } from 'type-fest';
+import { z } from 'zod';
+import type { TaskLoopOptions } from './autoscaling/autoscaled_pool.js';
+import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency_system.js';
+import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
+import type { FinalStatistics } from './autoscaling/system_status.js';
+import { ContextPipeline } from './crawlers/context_pipeline.js';
+import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler_commons.js';
+import type { IStatistics, StatisticState } from './crawlers/statistics.js';
+import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue_links/index.js';
+import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
 export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
 }
 export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
@@ -27,7 +37,7 @@ export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlin
 export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
     contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
 };
-export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     /**
      * User-provided function that performs the logic of the crawler. It is called for each URL to crawl.
      *
@@ -43,7 +53,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * To make this work, we should **always**
      * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+     * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
      */
     requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
@@ -135,7 +145,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     maxRequestRetries?: number;
     /**
-     * Indicates how much time (in seconds) to wait before crawling another same domain request.
+     * Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
+     * paced together with the site they belong to.
+     *
+     * Offered to the crawler's request manager as a `minIntervalEverywhere` {@link PacingSignal}; a manager that
+     * already paces every domain it dispatches to takes it, so no domain ends up with two clocks. Otherwise the
+     * crawler wraps its request manager in a {@link ThrottlingRequestManager} of its own.
      * @default 0
      */
     sameDomainDelaySecs?: number;
@@ -159,7 +174,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
      * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
      */
-    taskLoopOptions?: TaskLoopPredicates;
+    taskLoopOptions?: TaskLoopOptions;
     /**
      * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
      * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
@@ -170,8 +185,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
      * accounting is shared.
      *
-     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts, which configure
-     * the default system this one replaces — combining the two throws.
+     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
+     * shortcuts, which configure the default system this one replaces — combining the two throws.
      *
      * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
      * every crawler borrowing it has finished. The crawler does neither on your behalf.
@@ -191,6 +206,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * {@link ConcurrencySystem}.
      */
     maxConcurrency?: number;
+    /**
+     * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
+     * {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}. Defaults to `minConcurrency`.
+     */
+    initialConcurrency?: number;
     /**
      * The maximum number of requests per minute the crawler should run.
      * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
@@ -201,7 +222,16 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     /**
      * Allows to keep the crawler alive even if the {@link RequestQueue} gets empty.
      * By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
-     * waiting for more requests to come. Use `crawler.stop()` to exit the crawler gracefully, or `crawler.teardown()` to stop it immediately.
+     * waiting for more requests to come.
+     *
+     * `crawler.stop()` only stops the crawler from dispatching new requests — it does not wait for or drain
+     * requests already sitting in the queue. If this crawler is fed by another crawler or an external producer,
+     * calling `stop()` as soon as the producer finishes can abandon requests the producer already enqueued but
+     * this crawler had not yet picked up.
+     *
+     * To shut down only once the queue is genuinely empty and the producer has stopped, override
+     * {@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`} to check both
+     * `(await requestManager.checkReadiness()).status === 'finished'` and the producer's own running/finished flag.
      */
     keepAlive?: boolean;
     /**
@@ -236,6 +266,10 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     statusMessageCallback?: StatusMessageCallback;
     /**
      * HTTP status codes that indicate the session should be retired.
+     *
+     * A 429 from a domain covered by a {@link ThrottlingRequestManager} is handled as a rate limit before
+     * this is consulted, so removing 429 here only affects domains that manager does not cover.
+     *
      * @default [401, 403, 429]
      */
     blockedStatusCodes?: number[];
@@ -263,14 +297,34 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * 1. based on robots.txt file,
      * 2. because they don't match enqueueLinks filters,
      * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
-     * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
+     * 4. because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached,
+     * 5. or by {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}.
      */
     onSkippedRequest?: SkippedRequestCallback;
     /**
-     * Customize the way statistics collecting works, such as logging interval or
-     * whether to output them to the Key-Value store.
+     * A preconfigured statistics instance. When provided, the crawler records into it instead of building its own and
+     * will not `reset()` it between `run()` calls. Accepts the built-in {@link Statistics} or any object
+     * implementing {@link IStatistics}.
+     *
+     * Custom fields declared via {@link StatisticsOptions.stateExtension|`stateExtension`} are carried over to
+     * {@link BasicCrawler.statistics|`crawler.statistics.state`}:
+     *
+     * ```ts
+     * const statistics = new Statistics({ stateExtension: { defaultState: { productsFound: 0 } } });
+     *
+     * const crawler = new BasicCrawler({
+     *     statistics,
+     *     requestHandler: async () => {
+     *         statistics.state.productsFound++;
+     *     },
+     * });
+     *
+     * await crawler.run();
+     * // the custom fields are typed on `crawler.statistics` too
+     * console.log(crawler.statistics.state.productsFound);
+     * ```
      */
-    statisticsOptions?: StatisticsOptions;
+    statistics?: IStatistics<StatisticStateExtension>;
     /**
      * HTTP client implementation for the `sendRequest` context helper and for plain HTTP crawling.
      * Defaults to {@link ImpitHttpClient} when `@crawlee/impit-client` is installed, otherwise {@link FetchHttpClient}.
@@ -313,6 +367,20 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      *
      */
     id?: string;
+    /**
+     * Makes the storage writes performed while handling a request atomic with respect to the request
+     * succeeding: they are recorded in a {@link StorageTransaction} spanning the whole request
+     * lifecycle and only applied when the request handler succeeds, so a thrown handler leaves no partial
+     * writes behind and a retry does not double-write. Reads within the handler see its own writes.
+     *
+     * `false` disables the mechanism entirely; an object overrides the per-storage-type
+     * {@link StorageWritePolicy} (e.g. `{ requestQueue: 'deferred' }` for all-or-nothing enqueues).
+     * {@link withDirectStorageAccess} is the per-call-site escape hatch; `useState()` is deliberately
+     * *not* transactional.
+     *
+     * @default true
+     */
+    transactionalStorage?: boolean | Partial<StorageWritePolicy>;
     /**
      * An array of HTTP response [Status Codes](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status) to be excluded from error consideration.
      * By default, status codes >= 500 trigger errors.
@@ -324,162 +392,39 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     additionalHttpErrorStatusCodes?: number[];
 }
-/**
- * Provides a simple framework for parallel crawling of web pages.
- * The URLs to crawl are fed either from a static list of URLs
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
- *
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
- * download and data extraction functionality themselves.
- * If we want a crawler that already facilitates this functionality,
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
- *
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
- * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link IRequestManager|request manager} provided via the
- * {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is
- * itself a request manager). If no `requestManager` is provided, the crawler opens the default {@link RequestQueue}
- * either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called, or if the `requests`
- * parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
- *
- * To read requests from a read-only source such as a {@link RequestList} or {@link SitemapRequestLoader} while
- * still being able to enqueue new ones, combine the loader with a queue into a {@link RequestManagerTandem} using
- * {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`. The tandem
- * first processes URLs from the loader and automatically enqueues them into the queue, ensuring a single URL is not
- * crawled multiple times.
- *
- * > The legacy {@link BasicCrawlerOptions.requestList|`requestList`} and
- * > {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated. They are still accepted and
- * > folded into a single `requestManager` (combined into a tandem when both are given), but new code should use
- * > `requestManager` directly.
- *
- * The crawler finishes if there are no more {@link Request} objects to crawl.
- *
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
- * {@link ConcurrencySystem}.
- * Concurrency is tuned via the {@link BasicCrawlerOptions.minConcurrency|`minConcurrency`},
- * {@link BasicCrawlerOptions.maxConcurrency|`maxConcurrency`} and
- * {@link BasicCrawlerOptions.maxRequestsPerMinute|`maxRequestsPerMinute`} shortcuts, or, for finer control, by
- * injecting a pre-configured {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.
- *
- * **Example usage:**
- *
- * ```javascript
- * import { BasicCrawler, Dataset } from 'crawlee';
- *
- * // Create a crawler instance
- * const crawler = new BasicCrawler({
- *     async requestHandler({ request, sendRequest }) {
- *         // 'request' contains an instance of the Request class
- *         // Here we simply fetch the HTML of the page and store it to a dataset
- *         const { body } = await sendRequest({
- *             url: request.url,
- *             method: request.method,
- *             body: request.payload,
- *             headers: request.headers,
- *         });
- *
- *         await Dataset.pushData({
- *             url: request.url,
- *             html: body,
- *         })
- *     },
- * });
- *
- * // Enqueue the initial requests and run the crawler
- * await crawler.run([
- *     'http://www.example.com/page-1',
- *     'http://www.example.com/page-2',
- * ]);
- * ```
- * @category Crawlers
- */
-/**
- * Identifies a crawler instance for storage aliasing, `useState()` and status-message events.
- */
-interface CrawlerIdentity {
-    /**
-     * 0-based instantiation order across all crawlers in the process.
-     * Note that the value can be subject to race conditions between different script invocations.
-     */
-    readonly instanceIndex: number;
-    /** The user-supplied `id` option, or a fallback derived from `instanceIndex`. */
-    readonly id: string;
-    /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */
-    readonly hasExplicitId: boolean;
-}
-export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
+export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     #private;
-    protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
-    /**
-     * Tracks the number of crawler instances created. The first crawler uses the default
-     * request queue; subsequent ones get their own queue via a unique alias so they don't
-     * collide.
-     */
-    private static instanceCount;
-    /**
-     * Tracks crawler instances that accessed shared state without having an explicit id.
-     * Used to detect and warn about multiple crawlers sharing the same state.
-     */
-    private static useStateAnonymousIndices;
     /**
-     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
+     * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
      */
-    readonly stats: Statistics;
+    get statistics(): IStatistics<StatisticStateExtension>;
     /**
      * The main request-handling component of the crawler. It manages the requests that the crawler processes,
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
-    protected requestManager?: IRequestManager;
-    /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
-    private sessionPoolDep;
+    protected get requestManager(): IRequestManager | undefined;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool(): ISessionPool;
+    /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
+    get running(): boolean;
     /**
-     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
-     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
-     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
-     */
-    private ownedRequestQueue;
-    /**
-     * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
-     * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
-     * so it only needs to be applied once, at the first async access of the manager.
-     */
-    private requestManagerTimeoutsApplied;
-    /**
-     * Resolves the governor for one run: either the injected
-     * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} (borrowed) or a freshly built default with
-     * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
-     */
-    private readonly resolveConcurrencySystem;
-    /** As resolved by `_init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    private concurrencySystemDep?;
-    /**
-     * The concurrency governor this run is booking its requests against — either the
+     * The concurrency governor the run in progress is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
      * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
      *
-     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
-     * is also rebuilt for every run, so the instance is not stable across runs.
+     * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
+     * during a run rather than caching it across runs.
      *
      * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
      * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem(): IConcurrencySystem | undefined;
-    /**
-     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
-     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
-     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
-     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
-     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
-     */
-    private autoscaledPool?;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -490,135 +435,72 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     readonly router: RouterHandler<Context, Routes>;
-    private _basicContextPipeline?;
-    /**
-     * The basic part of the context pipeline. Unlike the subclass pipeline, this
-     * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
-     * pipelines expect the basic crawler fields to already be present in the context at runtime.
-     *
-     * Context built with this pipeline can be passed into multiple crawler pipelines at once.
-     * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
-     */
-    get basicContextPipeline(): ContextPipeline<{
-        request: Request;
-    }, CrawlingContext>;
-    private _contextPipeline?;
+    /** @internal */
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
-    running: boolean;
-    hasFinishedBefore: boolean;
-    private unexpectedStop;
+    /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
+    get hasFinishedBefore(): boolean;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
-    protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    private requestHandlerTimeoutMillis;
     protected readonly internalTimeoutMillis: number;
-    protected readonly maxRequestRetries: number;
-    protected readonly maxCrawlDepth?: number;
-    private sameDomainDelayMillis;
-    private domainAccessedTime;
-    protected readonly maxRequestsPerCrawl?: number;
-    private get handledRequestsCount();
-    private statusMessageLoggingInterval;
-    private statusMessageCallback?;
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
-    private ignoreHttpErrorStatusCodes;
-    /**
-     * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
-     * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
-     * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
-     * settable here.
-     */
-    private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
-    private respectRobotsTxtFile;
-    protected readonly onSkippedRequest?: SkippedRequestCallback;
-    private _closeEvents?;
-    private loggedPerRun;
-    private readonly robotsTxtFileCache;
-    protected readonly identity: CrawlerIdentity;
-    private readonly contextPipelineOptions;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        blockedStatusCodes: import("ow").ArrayPredicate<number>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
+        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
+        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
+        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
+        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
+        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
+        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
+        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
+        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
+            requestQueue: z.ZodOptional<z.ZodEnum<{
+                deferred: "deferred";
+                writeThrough: "writeThrough";
+            }>>;
+        }, z.core.$strict>]>>;
+        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        httpClient: z.ZodOptional<z.ZodInstanceOf<BaseHttpClient>>;
+        configuration: z.ZodOptional<z.ZodInstanceOf<Configuration>>;
+        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        eventManager: z.ZodOptional<z.ZodInstanceOf<EventManager>>;
+        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
+        keepAlive: z.ZodOptional<z.ZodBoolean>;
+        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        id: z.ZodOptional<z.ZodString>;
     };
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> & RequireContextPipeline<CrawlingContext, Context>);
+    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
@@ -632,22 +514,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @returns `true` if the status code is considered an error, `false` otherwise
      */
     protected isErrorStatusCode(status: number): boolean;
-    /**
-     * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
-     * This handles base context creation, session resolution, and context helpers.
-     */
-    private buildBasicContextPipeline;
-    private checkRobotsTxt;
-    /**
-     * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
-     * Subclasses should override this to add their own pipeline stages.
-     */
-    protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
-    private createBaseContext;
-    private resolveRequest;
-    private resolveSession;
-    private createContextHelpers;
-    private buildFinalContextPipeline;
     /**
      * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
      * Used for retrying requests that failed due to proxy errors.
@@ -665,7 +531,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * their status-reporting backend (e.g. the Apify platform).
      */
     setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
-    private getPeriodicLogger;
     /**
      * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
      * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
@@ -674,8 +539,12 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
      *
+     * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
+     * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
+     * if that is what you want.
+     *
      * @param [requests] The requests to add.
-     * @param [options] Options for the request queue.
+     * @param [options] Options for adding the initial requests.
      */
     run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
     /**
@@ -710,39 +579,18 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
      */
     getRequestQueue(): Promise<IRequestManager>;
-    /**
-     * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
-     * crawler-owned queue (so it gets purged between repeated `run()` calls).
-     * @private
-     */
-    private openOwnedRequestQueue;
-    /**
-     * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
-     * locking storage backend keeps it reserved for slightly longer than the request handler timeout
-     * (with some padding for overhead), but never for less than a minute. This prevents a long-running
-     * request from being handed out a second time while it is still being processed — and it works
-     * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
-     */
-    private applyRequestManagerTimeouts;
-    /**
-     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
-     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
-     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
-     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
-     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
-     */
-    private validateRequestUserData;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
-    protected getPendingRequestCountApproximation(): Promise<number>;
-    protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;
-    protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
-    private logOncePerRun;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
      * the batches via `waitBetweenBatchesMillis`. If you want to wait for all batches to be added to the queue, you can use
      * the `waitForAllRequestsToBeAdded` promise you get in the response object.
      *
+     * Optionally, the requests can be filtered using `include`/`exclude` glob or regexp patterns and an
+     * enqueue `strategy` (both AND-ed together, same as {@link CrawlingContext.enqueueLinks|`enqueueLinks`}),
+     * relative to `baseUrl`. Unlike `enqueueLinks`, there is no implicit "current page" to anchor the strategy
+     * to, so `strategy` defaults to {@link EnqueueStrategy.All|`all`} here.
+     *
      * This is an alias for calling `addRequestsBatched()` on the implicit `RequestQueue` for this crawler instance.
      *
      * @param requests The requests to add
@@ -769,84 +617,41 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     /**
      * Initializes the crawler.
      */
-    protected _init(): Promise<void>;
+    protected init(): Promise<void>;
     /**
      * The navigation timeout (pre-navigation hooks, navigation, and post-navigation hooks) in milliseconds, used
      * to size the internal request timeout. `BasicCrawler` has no navigation phase, so this is 0; the HTTP and
      * browser crawlers override it with their `navigationTimeoutSecs`.
      */
     protected getNavigationTimeoutMillis(): number;
-    /**
-     * Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
-     * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
-     * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
-     * whichever is larger: the configured internal timeout, or this request's combined phase budget.
-     */
-    private withRequestTimeout;
-    /**
-     * The request handler timeout for a request with the given route label. A router route may override the
-     * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
-     *
-     * @param label The request's route label, or `undefined` for the default route / no specific request.
-     * @param fallbackMillis Timeout to use when no route overrides it.
-     */
-    private resolveRequestHandlerTimeoutMillis;
-    /**
-     * The timeout the router route with the given label asked for, or `undefined` when it did not override one
-     * (or the request handler is not a router at all).
-     */
-    private getRouteTimeoutMillis;
     protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     /**
-     * Handles blocked request
+     * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
+     * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
+     * normal return says nothing about success. `handleRequest` commits on success.
      */
-    protected _throwOnBlockedRequest(statusCode: number): void;
-    private isAllowedBasedOnRobotsTxtFile;
-    protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
-    private pauseOnMigration;
+    private runInStorageTransaction;
     /**
-     * Fetches the next request to process from the underlying request provider.
+     * Handles blocked request
      */
-    private fetchNextRequest;
+    protected throwOnBlockedRequest(statusCode: number): void;
     /**
-     * Delays processing of the request based on the `sameDomainDelaySecs` option,
-     * adding it back to the queue after the timeout passes. Returns `true` if the request
-     * should be ignored and will be reclaimed to the queue once ready.
+     * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
+     *
+     * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
+     * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
+     *  {@link RequestThrottledError} rather than treating the response as a blocked session.
      */
-    private delayRequest;
-    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
+    /** @internal */
+    protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
+    /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
     private handleRequest;
-    /**
-     * Wrapper around the crawling context's `enqueueLinks` method:
-     * - Injects `crawlDepth` to each request being added based on the crawling context request.
-     * - Provides defaults for the `enqueueLinks` options based on the crawler configuration.
-     *      - These options can be overridden by the user.
-     * @internal
-     */
-    protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request<Dictionary>, requestManager: IRequestManager): Promise<BatchAddRequestsResult>;
     /**
      * Generator function that yields requests injected with the given crawl depth.
      * @internal
      */
     protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
-    /**
-     * Run async callback with given timeout and retry. Returns the result of the callback.
-     * @ignore
-     */
-    private timeoutAndRetry;
-    /**
-     * Returns true if either RequestList or RequestQueue have a request ready for processing.
-     */
-    private isTaskReadyFunction;
-    /**
-     * Returns true if both RequestList and RequestQueue have all requests finished.
-     */
-    private defaultIsFinishedFunction;
-    /**
-     * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
-     */
-    private unwrapError;
     /**
      * Handles errors thrown by user provided requestHandler()
      *
@@ -859,40 +664,31 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @param error The error received
      * @returns The message to be logged
      */
-    protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
-    private canRequestBeRetried;
+    protected getMessageFromError(error: Error, forceStack?: boolean): string;
     /**
-     * Stops the crawler immediately.
+     * Ends the run in progress immediately, without waiting for the requests already in flight.
      *
-     * This method doesn't wait for currently active requests to finish.
+     * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
+     * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
+     * released by {@link BasicCrawler.destroy|`destroy()`} instead.
      *
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     teardown(): Promise<void>;
-    protected _getCookieHeaderFromRequest(request: Request): string;
-    private requestMatchesEnqueueStrategy;
-}
-export interface CreateContextOptions {
-    request: Request;
-    session: ISession;
-    proxyInfo?: ProxyInfo;
+    /**
+     * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
+     * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
+     * could otherwise have run again.
+     */
+    destroy(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+    protected getCookieHeaderFromRequest(request: Request): string;
 }
-export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
+export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
 }
 export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
 }
 export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
-    /**
-     * Controls whether the request queue is purged between repeated `run()` calls on the same crawler instance.
-     * Purging clears all requests and resets internal counters, allowing the same URLs to be processed again.
-     *
-     * - **`undefined`** (default) — only the crawler's own (auto-created) queue is purged.
-     *   A user-supplied `requestQueue` is left untouched.
-     * - **`true`** — the queue is always purged, even if it was supplied by the user.
-     * - **`false`** — nothing is purged. Only genuinely new requests will be processed;
-     *   note that even a failed request is considered handled.
-     */
-    purgeRequestQueue?: boolean;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.