@crawlee/basic 4.0.0-rc.0 → 4.0.0-rc.1

package/internals/basic-crawler.d.ts

@@ -1,11 +1,19 @@
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, ConcurrencySystemOptions, CrawleeLogger, CrawlingContext, DatasetExportOptions, EnqueueUrlsOptions, FinalStatistics, GetUserDataFromRequest, IConcurrencySystem, IProxyConfiguration, IRequestLoader, IRequestManager, IStatistics, RequestsLike, RouterHandler, RouterRoutes, SkippedRequestCallback, Source, StatisticState, StorageIdentifier, StorageWritePolicy, TaskLoopPredicates, TypedRequestsLike } from '@crawlee/core';
-import { ConcurrencySystem, Configuration, ContextPipeline, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
+import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, CrawleeLogger, DatasetExportOptions, IProxyConfiguration, IRequestLoader, IRequestManager, RequestsLike, Source, StorageIdentifier, StorageWritePolicy } from '@crawlee/core';
+import { Configuration, Request, Dataset, EventManager, RequestQueue } from '@crawlee/core';
 import { BaseHttpClient } from '@crawlee/http-client';
-import type { Awaitable, Dictionary, ISession, ISessionPool, ProxyInfo, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
+import type { Awaitable, Dictionary, ISessionPool, SetStatusMessageOptions, StorageBackend } from '@crawlee/types';
 import { RobotsTxtFile } from '@crawlee/utils';
 import type { ReadonlyDeep } from 'type-fest';
 import { z } from 'zod';
-import { TimeoutError } from '@apify/timeout';
+import type { TaskLoopOptions } from './autoscaling/autoscaled_pool.js';
+import type { ConcurrencySystemOptions, IConcurrencySystem } from './autoscaling/concurrency_system.js';
+import { ConcurrencySystem } from './autoscaling/concurrency_system.js';
+import type { FinalStatistics } from './autoscaling/system_status.js';
+import { ContextPipeline } from './crawlers/context_pipeline.js';
+import type { CrawlingContext, TypedRequestsLike } from './crawlers/crawler_commons.js';
+import type { IStatistics, StatisticState } from './crawlers/statistics.js';
+import type { EnqueueUrlsOptions, SkippedRequestCallback } from './enqueue_links/index.js';
+import type { GetUserDataFromRequest, RouterHandler, RouterRoutes } from './router.js';
 export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
 }
 export { navigationDeadlineKey, remainingNavigationWindowMillis } from './request-timeout.js';
@@ -45,7 +53,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * To make this work, we should **always**
      * let our function throw exceptions rather than catch them.
      * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage|`Request.pushErrorMessage()`} function.
+     * {@link CrawlingRequest.pushErrorMessage|`request.pushErrorMessage()`} function.
      */
     requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
@@ -140,8 +148,9 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * Indicates how much time (in seconds) to wait before crawling another same domain request. Subdomains are
      * paced together with the site they belong to.
      *
-     * Wraps the crawler's request manager in a {@link ThrottlingRequestManager}; pass one as `requestManager`
-     * yourself to configure it further.
+     * Offered to the crawler's request manager as a `minIntervalEverywhere` {@link PacingSignal}; a manager that
+     * already paces every domain it dispatches to takes it, so no domain ends up with two clocks. Otherwise the
+     * crawler wraps its request manager in a {@link ThrottlingRequestManager} of its own.
      * @default 0
      */
     sameDomainDelaySecs?: number;
@@ -165,7 +174,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * Concurrency is configured elsewhere — through the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute`
      * shortcuts, or a {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} for finer control.
      */
-    taskLoopOptions?: TaskLoopPredicates;
+    taskLoopOptions?: TaskLoopOptions;
     /**
      * A pre-configured concurrency governor — the component that decides whether there is free compute for one more
      * task. Typically a {@link ConcurrencySystem}, though any {@link IConcurrencySystem} is accepted. All
@@ -176,8 +185,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * single budget. Each crawler still builds and drives its own {@link AutoscaledPool}; only the load/scaling
      * accounting is shared.
      *
-     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts, which configure
-     * the default system this one replaces — combining the two throws.
+     * Mutually exclusive with the `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute`
+     * shortcuts, which configure the default system this one replaces — combining the two throws.
      *
      * You own a supplied system's lifecycle: `start()` it before `run()` (which throws otherwise) and `stop()` it once
      * every crawler borrowing it has finished. The crawler does neither on your behalf.
@@ -197,6 +206,12 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * {@link ConcurrencySystem}.
      */
     maxConcurrency?: number;
+    /**
+     * Sets the concurrency (parallelism) the crawl starts with, before any scaling happens. Shortcut for the
+     * {@link ConcurrencySystemOptions.desiredConcurrency|`desiredConcurrency`} option of the crawler's default
+     * {@link ConcurrencySystem}. Defaults to `minConcurrency`.
+     */
+    initialConcurrency?: number;
     /**
      * The maximum number of requests per minute the crawler should run.
      * By default, this is set to `Infinity`, but we can pass any positive, non-zero integer.
@@ -207,7 +222,16 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
     /**
      * Allows to keep the crawler alive even if the {@link RequestQueue} gets empty.
      * By default, the `crawler.run()` will resolve once the queue is empty. With `keepAlive: true` it will keep running,
-     * waiting for more requests to come. Use `crawler.stop()` to exit the crawler gracefully, or `crawler.teardown()` to stop it immediately.
+     * waiting for more requests to come.
+     *
+     * `crawler.stop()` only stops the crawler from dispatching new requests — it does not wait for or drain
+     * requests already sitting in the queue. If this crawler is fed by another crawler or an external producer,
+     * calling `stop()` as soon as the producer finishes can abandon requests the producer already enqueued but
+     * this crawler had not yet picked up.
+     *
+     * To shut down only once the queue is genuinely empty and the producer has stopped, override
+     * {@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`} to check both
+     * `(await requestManager.checkReadiness()).status === 'finished'` and the producer's own running/finished flag.
      */
     keepAlive?: boolean;
     /**
@@ -273,7 +297,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * 1. based on robots.txt file,
      * 2. because they don't match enqueueLinks filters,
      * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,
-     * 4. or because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached
+     * 4. because the {@link BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached,
+     * 5. or by {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}.
      */
     onSkippedRequest?: SkippedRequestCallback;
     /**
@@ -367,99 +392,8 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     additionalHttpErrorStatusCodes?: number[];
 }
-/**
- * Provides a simple framework for parallel crawling of web pages.
- * The URLs to crawl are fed either from a static list of URLs
- * or from a dynamic queue of URLs enabling recursive crawling of websites.
- *
- * `BasicCrawler` is a low-level tool that requires the user to implement the page
- * download and data extraction functionality themselves.
- * If we want a crawler that already facilitates this functionality,
- * we should consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
- *
- * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler|`requestHandler`}
- * for each {@link Request} object, which represents a single URL to crawl.
- * The {@link Request} objects are fed from the {@link IRequestManager|request manager} provided via the
- * {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is
- * itself a request manager). If no `requestManager` is provided, the crawler opens the default {@link RequestQueue}
- * either when the {@link BasicCrawler.addRequests|`crawler.addRequests()`} function is called, or if the `requests`
- * parameter (representing the initial requests) of the {@link BasicCrawler.run|`crawler.run()`} function is provided.
- *
- * To read requests from a read-only source such as a {@link RequestList} or {@link SitemapRequestLoader} while
- * still being able to enqueue new ones, combine the loader with a queue into a {@link RequestManagerTandem} using
- * {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`. The tandem
- * first processes URLs from the loader and automatically enqueues them into the queue, ensuring a single URL is not
- * crawled multiple times.
- *
- * > The legacy {@link BasicCrawlerOptions.requestList|`requestList`} and
- * > {@link BasicCrawlerOptions.requestQueue|`requestQueue`} options are deprecated. They are still accepted and
- * > folded into a single `requestManager` (combined into a tandem when both are given), but new code should use
- * > `requestManager` directly.
- *
- * The crawler finishes if there are no more {@link Request} objects to crawl.
- *
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's
- * {@link ConcurrencySystem}.
- * Concurrency is tuned via the {@link BasicCrawlerOptions.minConcurrency|`minConcurrency`},
- * {@link BasicCrawlerOptions.maxConcurrency|`maxConcurrency`} and
- * {@link BasicCrawlerOptions.maxRequestsPerMinute|`maxRequestsPerMinute`} shortcuts, or, for finer control, by
- * injecting a pre-configured {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`}.
- *
- * **Example usage:**
- *
- * ```javascript
- * import { BasicCrawler, Dataset } from 'crawlee';
- *
- * // Create a crawler instance
- * const crawler = new BasicCrawler({
- *     async requestHandler({ request, sendRequest }) {
- *         // 'request' contains an instance of the Request class
- *         // Here we simply fetch the HTML of the page and store it to a dataset
- *         const { body } = await sendRequest({
- *             url: request.url,
- *             method: request.method,
- *             body: request.payload,
- *             headers: request.headers,
- *         });
- *
- *         await Dataset.pushData({
- *             url: request.url,
- *             html: body,
- *         })
- *     },
- * });
- *
- * // Enqueue the initial requests and run the crawler
- * await crawler.run([
- *     'http://www.example.com/page-1',
- *     'http://www.example.com/page-2',
- * ]);
- * ```
- * @category Crawlers
- */
-/**
- * Identifies a crawler instance for storage aliasing, `useState()` and status-message events.
- */
-interface CrawlerIdentity {
-    /**
-     * 0-based instantiation order across all crawlers in the process.
-     * Note that the value can be subject to race conditions between different script invocations.
-     */
-    readonly instanceIndex: number;
-    /** The user-supplied `id` option, or a fallback derived from `instanceIndex`. */
-    readonly id: string;
-    /** Whether `id` came from the user (as opposed to being derived from `instanceIndex`). */
-    readonly hasExplicitId: boolean;
-}
 export declare class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
     #private;
-    protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
-    /**
-     * Tracks the number of crawler instances created. The first crawler uses the default
-     * request queue; subsequent ones get their own queue via a unique alias so they don't
-     * collide.
-     */
-    private static instanceCount;
     /**
      * The statistics instance collecting the crawler's run statistics - either the injected `statistics` option or a
      * crawler-built default. Typed as {@link IStatistics} so custom implementations can be plugged in.
@@ -470,19 +404,21 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * combining any provided request loader and/or queue. It's initialized during the crawler startup or lazily
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
-    protected requestManager?: IRequestManager;
+    protected get requestManager(): IRequestManager | undefined;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool(): ISessionPool;
+    /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
+    get running(): boolean;
     /**
-     * The concurrency governor this run is booking its requests against — either the
+     * The concurrency governor the run in progress is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
      * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
      *
-     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
-     * is also rebuilt for every run, so the instance is not stable across runs.
+     * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
+     * during a run rather than caching it across runs.
      *
      * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
      * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
@@ -499,43 +435,20 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
      */
     readonly router: RouterHandler<Context, Routes>;
-    /**
-     * The basic part of the context pipeline. Unlike the subclass pipeline, this
-     * part has no major side effects (e.g. launching a browser). It also makes typing more explicit, as subclass
-     * pipelines expect the basic crawler fields to already be present in the context at runtime.
-     *
-     * Context built with this pipeline can be passed into multiple crawler pipelines at once.
-     * This is used e.g. in the {@link AdaptivePlaywrightCrawler|`AdaptivePlaywrightCrawler`}.
-     */
-    get basicContextPipeline(): ContextPipeline<{
-        request: Request;
-    }, CrawlingContext>;
+    /** @internal */
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
-    running: boolean;
-    hasFinishedBefore: boolean;
+    /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
+    get hasFinishedBefore(): boolean;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
-    protected readonly errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    protected readonly failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
-    private requestHandlerTimeoutMillis;
     protected readonly internalTimeoutMillis: number;
-    protected readonly maxRequestRetries: number;
-    protected readonly maxCrawlDepth?: number;
-    protected readonly maxRequestsPerCrawl?: number;
-    private get handledRequestsCount();
     protected blockedStatusCodes: Set<number>;
     protected readonly additionalHttpErrorStatusCodes: Set<number>;
-    /**
-     * The resolved options for the crawler's own task loop — the crawler-owned `runTaskFunction`, the (possibly
-     * user-overridden) ready/finished predicates and cadence/logging. Concurrency configuration lives on the
-     * {@link ConcurrencySystem} instead, and the loop's `consumer` identity is the crawler's own, so neither is
-     * settable here.
-     */
-    private taskLoopOptions;
     protected readonly httpClient: BaseHttpClient;
     protected readonly retryOnBlocked: boolean;
-    protected readonly onSkippedRequest?: SkippedRequestCallback;
-    protected readonly identity: CrawlerIdentity;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
         contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
@@ -568,69 +481,26 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
             }>>;
         }, z.core.$strict>]>>;
         onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;
-        configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;
+        httpClient: z.ZodOptional<z.ZodInstanceOf<BaseHttpClient>>;
+        configuration: z.ZodOptional<z.ZodInstanceOf<Configuration>>;
         storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;
+        eventManager: z.ZodOptional<z.ZodInstanceOf<EventManager>>;
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
     };
-    protected static optionsSchema: z.ZodObject<{
-        contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
-        errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
-        sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
-        maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
-        maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
-        taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
-        statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
-        ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
-        blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
-        retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
-        respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
-        transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
-            requestQueue: z.ZodOptional<z.ZodEnum<{
-                deferred: "deferred";
-                writeThrough: "writeThrough";
-            }>>;
-        }, z.core.$strict>]>>;
-        onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        httpClient: z.ZodOptional<z.ZodCustom<BaseHttpClient, BaseHttpClient>>;
-        configuration: z.ZodOptional<z.ZodCustom<Configuration, Configuration>>;
-        storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        eventManager: z.ZodOptional<z.ZodCustom<EventManager, EventManager>>;
-        logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
-        maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
-        maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
-        keepAlive: z.ZodOptional<z.ZodBoolean>;
-        statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        id: z.ZodOptional<z.ZodString>;
-    }, z.core.$strict>;
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
     constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
     /**
      * Builds the crawler-owned default {@link ConcurrencySystem} from the resolved
-     * `minConcurrency`/`maxConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
+     * `minConcurrency`/`maxConcurrency`/`initialConcurrency`/`maxRequestsPerMinute` shortcuts. Not called when a
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} was injected.
      *
      * Subclasses may override this to tune the default system (e.g. {@link HttpCrawler} raises the starting
@@ -644,22 +514,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @returns `true` if the status code is considered an error, `false` otherwise
      */
     protected isErrorStatusCode(status: number): boolean;
-    /**
-     * Builds the basic context pipeline that transforms `{ request }` into a full `CrawlingContext`.
-     * This handles base context creation, session resolution, and context helpers.
-     */
-    private buildBasicContextPipeline;
-    private checkRobotsTxt;
-    /**
-     * Builds the subclass-specific context pipeline that transforms a `CrawlingContext` into the crawler's target context type.
-     * Subclasses should override this to add their own pipeline stages.
-     */
-    protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
-    private createBaseContext;
-    private resolveRequest;
-    private resolveSession;
-    private createContextHelpers;
-    private buildFinalContextPipeline;
     /**
      * Checks if the given error is a proxy error by comparing its message to a list of known proxy error messages.
      * Used for retrying requests that failed due to proxy errors.
@@ -677,7 +531,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * their status-reporting backend (e.g. the Apify platform).
      */
     setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
-    private getPeriodicLogger;
     /**
      * Runs the crawler. Returns a promise that resolves once every request has been processed and the crawler's
      * finished-check ({@link BasicCrawlerOptions.taskLoopOptions|`taskLoopOptions.isFinishedFunction`}, or the
@@ -686,8 +539,12 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
      *
+     * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
+     * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
+     * if that is what you want.
+     *
      * @param [requests] The requests to add.
-     * @param [options] Options for the request queue.
+     * @param [options] Options for adding the initial requests.
      */
     run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
     /**
@@ -722,33 +579,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * crawler's request manager, which is no longer guaranteed to be a {@link RequestQueue}.
      */
     getRequestQueue(): Promise<IRequestManager>;
-    /**
-     * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
-     * crawler-owned queue (so it gets purged between repeated `run()` calls).
-     * @private
-     */
-    private openOwnedRequestQueue;
-    /**
-     * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
-     * locking storage backend keeps it reserved for slightly longer than the request handler timeout
-     * (with some padding for overhead), but never for less than a minute. This prevents a long-running
-     * request from being handed out a second time while it is still being processed — and it works
-     * regardless of whether the manager is a plain {@link RequestQueue} or a `RequestManagerTandem`.
-     */
-    private applyRequestManagerTimeouts;
-    /**
-     * Validates a request source's `userData` against the {@link RouteSchemas|Standard Schema} registered
-     * for its label on the crawler's schema-router (if any), throwing a {@link RequestValidationError} on
-     * mismatch. A no-op when the user's request handler is not a schema-router, or no schema is registered for
-     * the request's label. Applied by the crawler on the add paths it owns — `crawler.addRequests`,
-     * `crawler.run`, `context.addRequests` and `context.enqueueLinks`.
-     */
-    private validateRequestUserData;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
-    protected getPendingRequestCountApproximation(): Promise<number>;
-    protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;
-    protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
-    private logOncePerRun;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -793,84 +624,34 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * browser crawlers override it with their `navigationTimeoutSecs`.
      */
     protected getNavigationTimeoutMillis(): number;
-    /**
-     * Races the request against the internal timeout (see {@link raceWithTimeout}), sized to outlast the phases
-     * that have their own timeout - the navigation, its hooks, and the request handler - so a legitimately slow
-     * request, a per-route override, or a low `CRAWLEE_INTERNAL_TIMEOUT` is not cut short mid-phase. It takes
-     * whichever is larger: the configured internal timeout, or this request's combined phase budget.
-     */
-    private withRequestTimeout;
-    /**
-     * The request handler timeout for a request with the given route label. A router route may override the
-     * crawler's own `requestHandlerTimeoutSecs`; anything else falls back to `fallbackMillis`.
-     *
-     * @param label The request's route label, or `undefined` for the default route / no specific request.
-     * @param fallbackMillis Timeout to use when no route overrides it.
-     */
-    private resolveRequestHandlerTimeoutMillis;
-    /**
-     * The timeout the router route with the given label asked for, or `undefined` when it did not override one
-     * (or the request handler is not a router at all).
-     */
-    private getRouteTimeoutMillis;
     protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
     /**
      * Runs `callback` inside a {@link StorageTransaction}, unless transactional storage is disabled.
-     * Deliberately does **not** commit on return - `handleRequest` swallows request handler failures, so
-     * a normal return says nothing about success. `handleRequest` owns the outcome.
+     * Deliberately does **not** commit on return - the pipeline hands failures to `handleRequestFailure`, so a
+     * normal return says nothing about success. `handleRequest` commits on success.
      */
     private runInStorageTransaction;
     /**
      * Handles blocked request
      */
     protected throwOnBlockedRequest(statusCode: number): void;
-    private isAllowedBasedOnRobotsTxtFile;
     /**
-     * Records an HTTP 429 against the URL's domain so the request manager can pace the retry.
+     * Records an HTTP 429 against the URL's domain so the request manager can hold the retry back.
      *
      * @param retryAfterHeader The raw `Retry-After` response header, if the server sent one.
-     * @returns `true` if a manager took responsibility for the delay, in which case the caller should throw
+     * @returns `true` if the manager took responsibility for the delay, in which case the caller should throw
      *  {@link RequestThrottledError} rather than treating the response as a blocked session.
      */
     protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
-    /**
-     * Hands a robots.txt `Crawl-delay` to the request manager, warning if nothing is able to honour it.
-     *
-     * The warning is driven by whether the delay was actually accepted rather than by the type of the manager,
-     * because a manager that does throttle still drops the delay for a domain missing from its `domains` list.
-     */
-    private applyCrawlDelay;
+    /** @internal */
     protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
-    private pauseOnMigration;
-    /**
-     * Fetches the next request to process from the underlying request provider.
-     */
-    private fetchNextRequest;
-    /** Handles a single request - runs the request handler with retries, error handling, and lifecycle management. */
+    /** Runs the request handler and marks the request as handled. Failures are left to `handleRequestFailure`. */
     private handleRequest;
     /**
      * Generator function that yields requests injected with the given crawl depth.
      * @internal
      */
     protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
-    /**
-     * Run async callback with given timeout and retry. Returns the result of the callback.
-     * @ignore
-     */
-    private timeoutAndRetry;
-    /**
-     * Returns true if either RequestList or RequestQueue have a request ready for processing.
-     */
-    private isTaskReadyFunction;
-    /**
-     * Returns true if both RequestList and RequestQueue have all requests finished.
-     */
-    private defaultIsFinishedFunction;
-    /**
-     * Unwraps errors thrown by the context pipeline to get the actual user error.
-     * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
-     */
-    private unwrapError;
     /**
      * Handles errors thrown by user provided requestHandler()
      *
@@ -883,45 +664,31 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * @param error The error received
      * @returns The message to be logged
      */
-    protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
-    /**
-     * Whether the session should be spared for this error - either because it was already retired, or because the
-     * failure says nothing about the session (a rate limit is a property of the domain).
-     */
-    private errorAbsolvesSession;
-    private canRequestBeRetried;
+    protected getMessageFromError(error: Error, forceStack?: boolean): string;
     /**
-     * Stops the crawler immediately.
+     * Ends the run in progress immediately, without waiting for the requests already in flight.
      *
-     * This method doesn't wait for currently active requests to finish.
+     * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
+     * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
+     * released by {@link BasicCrawler.destroy|`destroy()`} instead.
      *
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     teardown(): Promise<void>;
+    /**
+     * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
+     * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
+     * could otherwise have run again.
+     */
+    destroy(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     protected getCookieHeaderFromRequest(request: Request): string;
-    private requestMatchesEnqueueStrategy;
-}
-export interface CreateContextOptions {
-    request: Request;
-    session: ISession;
-    proxyInfo?: ProxyInfo;
 }
 export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
 }
 export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
 }
 export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
-    /**
-     * Controls whether the request queue is purged between repeated `run()` calls on the same crawler instance.
-     * Purging clears all requests and resets internal counters, allowing the same URLs to be processed again.
-     *
-     * - **`undefined`** (default) — only the crawler's own (auto-created) queue is purged.
-     *   A user-supplied `requestQueue` is left untouched.
-     * - **`true`** — the queue is always purged, even if it was supplied by the user.
-     * - **`false`** — nothing is purged. Only genuinely new requests will be processed;
-     *   note that even a failed request is considered handled.
-     */
-    purgeRequestQueue?: boolean;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.